From db0cbc2eb53172b7a1d47ee2167e06805d31b180 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 26 Oct 2026 16:40:24 +0800 Subject: [PATCH 1/5] [enhancement](lance) Add cache diagnostics and scalar index auto selection --- be/src/common/config.cpp | 2 + be/src/common/config.h | 1 + .../format_v2/lance/lance_session_manager.cpp | 45 ++++++-- .../format_v2/lance/lance_session_manager.h | 16 +++ be/src/format_v2/table/lance_reader.cpp | 49 ++++++++- be/src/format_v2/table/lance_reader.h | 7 ++ .../lance/lance_session_manager_test.cpp | 23 ++++ be/test/format_v2/table/lance_reader_test.cpp | 37 +++++++ docs/lance-cache-and-index-selection.md | 71 +++++++++++++ .../java/org/apache/doris/analysis/Expr.java | 5 + .../lance/source/LanceScalarIndexPlanner.java | 43 +++++++- .../lance/source/LanceScanNode.java | 7 +- .../lance/source/LanceScanPlanner.java | 16 ++- .../translator/PhysicalPlanTranslator.java | 15 ++- .../org/apache/doris/qe/SessionVariable.java | 9 ++ .../source/LanceScalarIndexPlannerTest.java | 75 +++++++++++++ .../lance/source/LanceScanNodeTest.java | 41 +++++++ .../LanceScalarIndexSelectivityTest.java | 100 ++++++++++++++++++ 18 files changed, 544 insertions(+), 18 deletions(-) create mode 100644 docs/lance-cache-and-index-selection.md create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 8c1c0f62a01ec1..ad8f2617dfc60b 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1232,6 +1232,8 @@ DEFINE_Bool(enable_lance_data_cache, "true"); DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache"); DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB DEFINE_Int64(lance_data_cache_read_block_size_bytes, "1048576"); // 1MB +// Zero preserves the existing one-block Foyer memory tier, including custom block sizes. +DEFINE_Int64(lance_data_cache_memory_capacity_bytes, "0"); // I/O buffering budget per Lance scanner, not a cap on its total memory usage. // Runtime changes apply to newly created scanners. diff --git a/be/src/common/config.h b/be/src/common/config.h index afb0e6c58028a9..1758ac1ebd7a52 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1268,6 +1268,7 @@ DECLARE_Bool(enable_lance_data_cache); DECLARE_String(lance_data_cache_path); DECLARE_Int64(lance_data_cache_disk_capacity_bytes); DECLARE_Int64(lance_data_cache_read_block_size_bytes); +DECLARE_Int64(lance_data_cache_memory_capacity_bytes); // I/O buffering budget per Lance scanner, applied when a new scanner is created. DECLARE_mInt64(lance_io_buffer_size_bytes); diff --git a/be/src/format_v2/lance/lance_session_manager.cpp b/be/src/format_v2/lance/lance_session_manager.cpp index 884eb5db18c5e7..53b0945af7c820 100644 --- a/be/src/format_v2/lance/lance_session_manager.cpp +++ b/be/src/format_v2/lance/lance_session_manager.cpp @@ -64,6 +64,8 @@ LanceSessionManager::Config load_lance_session_config() { .lance_data_cache_disk_capacity_bytes = config::lance_data_cache_disk_capacity_bytes, .lance_data_cache_read_block_size_bytes = config::lance_data_cache_read_block_size_bytes, + .lance_data_cache_memory_capacity_bytes = + config::lance_data_cache_memory_capacity_bytes, }; } @@ -159,8 +161,8 @@ LanceSessionManager::LanceSessionManager(Config config) : _config(std::move(conf << ", lance_data_cache_disk_capacity_bytes=" << _config.lance_data_cache_disk_capacity_bytes << ", lance_data_cache_read_block_size_bytes=" - << _config.lance_data_cache_read_block_size_bytes - << ", foyer_memory_capacity_bytes=" << _config.lance_data_cache_read_block_size_bytes; + << _config.lance_data_cache_read_block_size_bytes << ", foyer_memory_capacity_bytes=" + << _config.resolved_data_cache_memory_capacity_bytes(); } LanceSessionManager::~LanceSessionManager() { @@ -170,17 +172,20 @@ LanceSessionManager::~LanceSessionManager() { Status LanceSessionManager::_initialize() { if (_config.enable_lance_data_cache) { + if (_config.lance_data_cache_memory_capacity_bytes < 0) { + return Status::InvalidArgument( + "lance_data_cache_memory_capacity_bytes must be non-negative"); + } // Treat the configured cache mode as a process-level requirement. If an enabled // data cache cannot initialize, report the failure instead of silently creating a // session without it. This keeps directory/configuration/device failures visible // to the operator. Disabling the cache is an explicit configuration change. const LanceDataCacheOptions data_cache_options { .directory = _config.lance_data_cache_path.c_str(), - // Foyer's HybridCache requires a memory tier. Keep it at the minimum useful - // capacity of exactly one range-cache block; WriteOnInsertion enqueues disk - // writes on insertion rather than waiting for memory-tier eviction. + // Keep the historical one-block tier when the memory capacity is unset. + // Its capacity is independent of the range-cache block granularity otherwise. .memory_capacity_bytes = - static_cast(_config.lance_data_cache_read_block_size_bytes), + static_cast(_config.resolved_data_cache_memory_capacity_bytes()), .disk_capacity_bytes = static_cast(_config.lance_data_cache_disk_capacity_bytes), .read_block_size_bytes = @@ -220,19 +225,37 @@ Status LanceSessionManager::open_dataset(const char* uri, const char* const* sto } *dataset = nullptr; - // Initialize lazily on the first dataset open, not at BE startup. A failed Status is a + RETURN_IF_ERROR(_ensure_initialized()); + + *dataset = lance_dataset_open_with_session(uri, storage_options, version, _session); + if (*dataset == nullptr) { + return lance_error("open Lance dataset with shared session"); + } + return Status::OK(); +} + +Status LanceSessionManager::_ensure_initialized() { + // Initialize lazily on the first use, not at BE startup. A failed Status is a // normal return from this lambda, so call_once completes and retains that failure just // like a successful initialization. All subsequent readers receive the same copied // error; fixing the cache directory alone does not trigger another attempt. This is // intentional: repair the cache configuration/storage (or disable the data cache), then // restart the BE to recreate the process-wide manager and session. std::call_once(_initialize_once, [this] { _initialize_status = _initialize(); }); - RETURN_IF_ERROR(_initialize_status); + return _initialize_status; +} - *dataset = lance_dataset_open_with_session(uri, storage_options, version, _session); - if (*dataset == nullptr) { - return lance_error("open Lance dataset with shared session"); +Status LanceSessionManager::get_cache_statistics(CacheStatistics* statistics) { + if (statistics == nullptr) { + return Status::InvalidArgument("Lance cache statistics output must not be null"); + } + RETURN_IF_ERROR(_ensure_initialized()); + LanceSessionCacheStats stats {}; + if (lance_session_get_cache_stats(_session, &stats) != 0) { + return lance_error("collect Lance session cache statistics"); } + statistics->index_cache_misses = stats.index_cache_misses; + statistics->metadata_cache_misses = stats.metadata_cache_misses; return Status::OK(); } diff --git a/be/src/format_v2/lance/lance_session_manager.h b/be/src/format_v2/lance/lance_session_manager.h index 45dc62acad4442..d1773c2a34ab6d 100644 --- a/be/src/format_v2/lance/lance_session_manager.h +++ b/be/src/format_v2/lance/lance_session_manager.h @@ -43,6 +43,18 @@ class LanceSessionManager final { std::string lance_data_cache_path; int64_t lance_data_cache_disk_capacity_bytes = 0; int64_t lance_data_cache_read_block_size_bytes = 0; + int64_t lance_data_cache_memory_capacity_bytes = 0; + + int64_t resolved_data_cache_memory_capacity_bytes() const { + return lance_data_cache_memory_capacity_bytes == 0 + ? lance_data_cache_read_block_size_bytes + : lance_data_cache_memory_capacity_bytes; + } + }; + + struct CacheStatistics { + uint64_t index_cache_misses = 0; + uint64_t metadata_cache_misses = 0; }; static LanceSessionManager& instance(); @@ -58,8 +70,12 @@ class LanceSessionManager final { Status open_dataset(const char* uri, const char* const* storage_options, uint64_t version, LanceDataset** dataset); + // A process-wide snapshot, including other readers using the shared session. + Status get_cache_statistics(CacheStatistics* statistics); + private: Status _initialize(); + Status _ensure_initialized(); Config _config; std::once_flag _initialize_once; diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index e0e2884efe7a4a..265d8a3284e1ce 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -117,6 +117,15 @@ Status LanceTableReader::init(TableReadOptions&& options) { file_scan_profile::TABLE_READER, 1); _dataset_open_time = add_lance_counter(_scanner_profile, "LanceDatasetOpenTime", TUnit::TIME_NS, LANCE_TIMING_PROFILE); + _session_index_cache_misses = add_lance_counter( + _scanner_profile, "LanceSessionIndexCacheMisses", TUnit::UNIT, "LanceSessionCache"); + _session_metadata_cache_misses = add_lance_counter( + _scanner_profile, "LanceSessionMetadataCacheMisses", TUnit::UNIT, "LanceSessionCache"); + _scanner_profile->add_info_string( + "LanceSessionCacheStatsScope", + "BE-wide changes during dataset-open windows; includes concurrent readers; " + "overlapping windows must not be summed as query-exclusive misses"); + _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); _arrow_to_doris_block_time = add_lance_counter(_scanner_profile, "LanceArrowToDorisBlockTime", TUnit::TIME_NS, LANCE_TIMING_PROFILE); _data_cache_bytes_read_from_cache = @@ -647,8 +656,15 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) { std::unique_ptr dataset; { SCOPED_TIMER(_dataset_open_time); + auto& session = LanceSessionManager::instance(); + _session_cache_statistics_valid = + session.get_cache_statistics(&_session_cache_statistics).ok(); + if (!_session_cache_statistics_valid) { + _session_cache_statistics_complete = false; + _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); + } LanceDataset* raw_dataset = nullptr; - Status status = LanceSessionManager::instance().open_dataset( + Status status = session.open_dataset( key.uri.c_str(), key.storage_options.empty() ? nullptr : storage_option_ptrs.data(), static_cast(key.version), &raw_dataset); if (!status.ok()) { @@ -1261,6 +1277,7 @@ void LanceTableReader::_close_dataset() { _fts_query_context = nullptr; } if (_dataset != nullptr) { + _collect_session_cache_statistics(); _collect_data_cache_statistics(); lance_dataset_close(_dataset); _dataset = nullptr; @@ -1271,6 +1288,36 @@ void LanceTableReader::_close_dataset() { _record_batch_converter.reset_schema(); } +void LanceTableReader::_collect_session_cache_statistics() { + if (!_session_cache_statistics_valid) { + return; + } + LanceSessionManager::CacheStatistics current; + const auto status = LanceSessionManager::instance().get_cache_statistics(¤t); + if (!status.ok()) { + _session_cache_statistics_complete = false; + _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); + LOG(WARNING) << status; + return; + } + // Session counters are monotonic and shared. Subtract the last snapshot exactly once + // per window; sampling them never resets counters used by other readers or metrics. + const auto update = [](RuntimeProfile::Counter* counter, uint64_t before, uint64_t after) { + if (counter != nullptr && after >= before) { + const auto delta = + std::min(after - before, std::numeric_limits::max()); + COUNTER_UPDATE(counter, static_cast(delta)); + } + }; + update(_session_index_cache_misses, _session_cache_statistics.index_cache_misses, + current.index_cache_misses); + update(_session_metadata_cache_misses, _session_cache_statistics.metadata_cache_misses, + current.metadata_cache_misses); + _session_cache_statistics = current; + _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", + _session_cache_statistics_complete ? "true" : "false"); +} + void LanceTableReader::_collect_data_cache_statistics() { if (_dataset == nullptr) { return; diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h index 205c082d12052d..843336f36cf14b 100644 --- a/be/src/format_v2/table/lance_reader.h +++ b/be/src/format_v2/table/lance_reader.h @@ -26,6 +26,7 @@ #include "common/status.h" #include "format_v2/lance/lance_record_batch_converter.h" +#include "format_v2/lance/lance_session_manager.h" #include "format_v2/table_reader.h" #include "runtime/runtime_profile.h" @@ -92,6 +93,7 @@ class LanceTableReader final : public TableReader { // Collect the cumulative statistics owned by this dataset handle. The Lance-C API returns an // absolute snapshot, so this method replaces rather than increments the profile counters. void _collect_data_cache_statistics(); + void _collect_session_cache_statistics(); // Keep lance-c's anonymous statistics typedef out of this header. _open_scanner installs the // strongly typed C callback adapter before forwarding the borrowed value here. static void _collect_scan_statistics(void* callback_ctx, const void* opaque_statistics); @@ -101,6 +103,11 @@ class LanceTableReader final : public TableReader { Status _dataset_key(const TFileRangeDesc& range, DatasetKey* key) const; LanceDataset* _dataset = nullptr; + LanceSessionManager::CacheStatistics _session_cache_statistics; + bool _session_cache_statistics_valid = false; + bool _session_cache_statistics_complete = true; + RuntimeProfile::Counter* _session_index_cache_misses = nullptr; + RuntimeProfile::Counter* _session_metadata_cache_misses = nullptr; std::shared_ptr _dataset_schema; LanceScanner* _scanner = nullptr; ShardedKVCache* _runtime_filter_cache = nullptr; diff --git a/be/test/format_v2/lance/lance_session_manager_test.cpp b/be/test/format_v2/lance/lance_session_manager_test.cpp index e1274219347352..dde7a68e19b4e7 100644 --- a/be/test/format_v2/lance/lance_session_manager_test.cpp +++ b/be/test/format_v2/lance/lance_session_manager_test.cpp @@ -54,6 +54,7 @@ TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) { .lance_data_cache_path = "", .lance_data_cache_disk_capacity_bytes = -1, .lance_data_cache_read_block_size_bytes = -1, + .lance_data_cache_memory_capacity_bytes = -1, }; LanceSessionManager manager(std::move(config)); LanceDataset* raw_dataset = nullptr; @@ -62,6 +63,27 @@ TEST(LanceSessionManagerTest, SessionAndDataCacheConfigurationsAreIndependent) { ASSERT_NE(dataset, nullptr); } +TEST(LanceSessionManagerTest, MemoryCapacityDefaultsToOneConfiguredBlock) { + LanceSessionManager::Config config; + config.lance_data_cache_read_block_size_bytes = 64 * 1024; + EXPECT_EQ(config.resolved_data_cache_memory_capacity_bytes(), 64 * 1024); + config.lance_data_cache_memory_capacity_bytes = 8 * 1024 * 1024; + EXPECT_EQ(config.resolved_data_cache_memory_capacity_bytes(), 8 * 1024 * 1024); + EXPECT_EQ(config.lance_data_cache_read_block_size_bytes, 64 * 1024); +} + +TEST(LanceSessionManagerTest, RejectsNegativeMemoryCapacityBeforeUnsignedConversion) { + LanceSessionManager::Config config; + config.enable_lance_data_cache = true; + config.lance_data_cache_memory_capacity_bytes = -1; + LanceSessionManager manager(std::move(config)); + LanceDataset* dataset = nullptr; + const auto status = manager.open_dataset(lance_fixture_path().c_str(), nullptr, 0, &dataset); + EXPECT_FALSE(status.ok()); + EXPECT_NE(status.to_string().find("lance_data_cache_memory_capacity_bytes"), std::string::npos); + EXPECT_EQ(dataset, nullptr); +} + TEST(LanceSessionManagerTest, PublishesSessionCacheMetrics) { constexpr int64_t INDEX_CACHE_CAPACITY = 8 * 1024 * 1024; constexpr int64_t METADATA_CACHE_CAPACITY = 4 * 1024 * 1024; @@ -114,6 +136,7 @@ TEST(LanceSessionManagerTest, CreatesFoyerBackedSession) { .lance_data_cache_path = cache_path.string(), .lance_data_cache_disk_capacity_bytes = 32 * 1024 * 1024, .lance_data_cache_read_block_size_bytes = 64 * 1024, + .lance_data_cache_memory_capacity_bytes = 8 * 1024 * 1024, }; { LanceSessionManager manager(std::move(config)); diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 6531fc21de72e8..58ffef582939c5 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -1349,6 +1349,43 @@ TEST(LanceTableReaderSnapshotTest, ProfileShowsTheSnapshotRead) { EXPECT_TRUE(latest_reader.close().ok()); } +TEST(LanceTableReaderSnapshotTest, ProfileReportsObservedSessionCacheMisses) { + LanceSessionManager::CacheStatistics before; + ASSERT_TRUE(LanceSessionManager::instance().get_cache_statistics(&before).ok()); + const std::filesystem::path dataset_uri = + "./be/test/format_v2/table/lance/data/all_types.lance"; + TQueryGlobals query_globals; + RuntimeState state(query_globals); + RuntimeProfile profile("lance_session_cache_profile"); + TFileScanRangeParams scan_params; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); + ASSERT_TRUE(reader.close().ok()); + LanceSessionManager::CacheStatistics after; + ASSERT_TRUE(LanceSessionManager::instance().get_cache_statistics(&after).ok()); + for (const char* name : {"LanceSessionIndexCacheMisses", "LanceSessionMetadataCacheMisses"}) { + auto* counter = profile.get_counter(name); + ASSERT_NE(counter, nullptr) << name; + EXPECT_GE(counter->value(), 0) << name; + } + EXPECT_EQ(profile.get_counter("LanceSessionIndexCacheMisses")->value(), + after.index_cache_misses - before.index_cache_misses); + EXPECT_EQ(profile.get_counter("LanceSessionMetadataCacheMisses")->value(), + after.metadata_cache_misses - before.metadata_cache_misses); + const auto* scope = profile.get_info_string("LanceSessionCacheStatsScope"); + ASSERT_NE(scope, nullptr); + EXPECT_NE(scope->find("concurrent"), std::string::npos); + const auto* available = profile.get_info_string("LanceSessionCacheStatsAvailable"); + ASSERT_NE(available, nullptr); + EXPECT_EQ(*available, "true"); + auto* misses = profile.get_counter("LanceSessionMetadataCacheMisses"); + const auto before_close = misses->value(); + ASSERT_TRUE(reader.close().ok()); + EXPECT_EQ(misses->value(), before_close); +} + TEST(LanceTableReaderSnapshotTest, OpenErrorsNameTheVersionWithoutTheQueryString) { const std::filesystem::path dataset_uri = "./be/test/format_v2/table/lance/data/all_types.lance"; diff --git a/docs/lance-cache-and-index-selection.md b/docs/lance-cache-and-index-selection.md new file mode 100644 index 00000000000000..d696568d808139 --- /dev/null +++ b/docs/lance-cache-and-index-selection.md @@ -0,0 +1,71 @@ +# Lance cache diagnostics and scalar index selection + +## Cache misses in query profiles + +The `LanceReader / LanceSessionCache` profile group contains: + +| Counter | Meaning | +| --- | --- | +| `LanceSessionIndexCacheMisses` | Increase in the BE-wide Lance Session index-cache miss counter during this reader's dataset-open windows. | +| `LanceSessionMetadataCacheMisses` | Increase in the BE-wide Lance Session metadata-cache miss counter during the same windows, including dataset opening. | + +The reader takes a baseline before opening its dataset and publishes the difference +when closing it. Reopening a dataset adds another observation window; closing a +reader twice does not count the same window twice. `LanceSessionCacheStatsAvailable` +is `true` only after statistics have been collected successfully for every observed +window. When it is `false`, zero counters do not establish that the caches had no misses. + +The Session is shared by all readers in one BE. These counters include activity from +concurrent readers and are **not query-exclusive miss counts**. Overlapping windows +must not be summed to obtain a query's exact misses or a cluster-wide miss total. +`LanceSessionCacheStatsScope` records this limitation in the profile. An isolated +query can still have several readers in one BE. For an exact query total, isolate +the query and compare each BE's Session metrics before and after it, rather than +summing reader windows. These counters do not measure the FE's separate metadata cache. +`LanceIndexPartitionCacheMissLoads` remains a separate, narrower execution metric. + +## Automatic scalar index selection + +```sql +SET enable_lance_scalar_index_auto_selection = true; +``` + +This session variable is enabled by default. For ordinary Lance table scans, the FE +uses Nereids predicate selectivity estimates to choose at most one eligible logical +BTree, Bitmap, or LabelList index. The estimated work is: + +```text +uncovered physical rows + covered physical rows * driver selectivity +``` + +Only indexable predicates on the selected field reduce its estimated candidates. +Estimates for independent conjuncts on that field are multiplied; they are advisory +and can be inaccurate for correlated predicates or stale/missing column statistics. +Multi-field predicates are not used to estimate a single index's candidates. Without +usable estimates, selection retains the existing largest-coverage rule and stable +index-name tie breaking. Existing eligibility, expression-budget and fragment-ownership +checks still apply. Uncovered fragments retain their non-indexed fallback scans. +`EXPLAIN` shows the chosen index as `lanceGroupingIndex`. + +Setting the variable to `false` retains the previous coverage-based selection; it +does not disable scalar indexes. The `lance_fragments_per_split` debug override still +bypasses index-segment grouping. This setting does not alter vector/FTS index selection +or move an external-search TVF's outer `WHERE` predicates before candidate selection. + +## Foyer memory capacity + +`lance_data_cache_memory_capacity_bytes` is a startup BE configuration, independent +of `lance_data_cache_read_block_size_bytes` and the disk capacity. Restart the BE +after changing it. + +| Value | Behavior | +| --- | --- | +| `0` (default) | Use one configured read block, preserving the previous behavior. With the default block size this is 1 MiB. | +| Positive | Set the Foyer memory tier's capacity in bytes. It must be at least one read block. | +| Negative | Invalid when the Lance data cache is enabled. | + +For example, `lance_data_cache_memory_capacity_bytes=10737418240` allows a 10 GiB +memory tier while retaining 1 MiB read blocks. This capacity is shared by Lance +readers in one BE and is separate from index and metadata cache capacities. Cache +bookkeeping and in-flight reads can consume additional memory. When +`enable_lance_data_cache=false`, the Foyer configuration is ignored. diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 253d45fa743ad5..a9ca5337b19b4a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -238,6 +238,11 @@ public double getSelectivity() { return selectivity; } + public void setSelectivity(double selectivity) { + this.selectivity = Double.isFinite(selectivity) && selectivity >= 0 && selectivity <= 1 + ? selectivity : -1; + } + public boolean hasSelectivity() { return selectivity >= 0; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java index 4d6319242f6984..db1aa45c667510 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java @@ -63,6 +63,11 @@ static final class Plan { static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, Map visibleFragments) { + return plan(metadata, pushedConjuncts, visibleFragments, true); + } + + static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, + Map visibleFragments, boolean autoSelection) { if (metadata.getVersion() <= 0 || !metadata.getIndexMetadataState().canPlanIndexSegments()) { return null; @@ -86,6 +91,9 @@ static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, } } Plan selected = null; + double selectedCost = Double.NaN; + double totalRows = visibleFragments.values().stream() + .mapToDouble(fragment -> Math.max(fragment.getPhysicalRows(), 0)).sum(); Map> fieldsByIndexType = new HashMap<>(); for (LanceIndexSegmentGroup logicalIndex : indices) { List segments = logicalIndex.getSegments(); @@ -103,13 +111,46 @@ static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, continue; } Plan candidate = groupFragments(metadata, segments, visibleFragments); - if (candidate != null && (selected == null || candidate.coveredRows > selected.coveredRows)) { + if (candidate == null) { + continue; + } + double cost = autoSelection ? estimateCandidateRows(metadata, pushedConjuncts, + index, candidate.coveredRows, totalRows) : Double.NaN; + int comparison = Double.isNaN(cost) && Double.isNaN(selectedCost) ? 0 + : Double.compare(Double.isNaN(cost) ? totalRows : cost, + Double.isNaN(selectedCost) ? totalRows : selectedCost); + if (selected == null || comparison < 0 + || (comparison == 0 && candidate.coveredRows > selected.coveredRows)) { selected = candidate; + selectedCost = cost; } } return selected; } + private static double estimateCandidateRows(LanceTableMetadata metadata, List conjuncts, + LanceIndexSegmentInfo index, long coveredRows, double totalRows) { + double selectivity = 1; + boolean estimated = false; + int field = index.getFieldIds().get(0); + for (Expr conjunct : conjuncts) { + double value = conjunct.getSelectivity(); + Set slots = new HashSet<>(); + conjunct.collect(SlotRef.class, slots); + boolean singleField = !slots.isEmpty() && slots.stream().allMatch(slot -> + metadata.getLanceFieldId(slot.getColumnName()).orElse(-1) == field); + if (Double.isFinite(value) && value >= 0 && value <= 1 + && singleField + && collectDriverFields(metadata, conjunct, index.getIndexType(), false).contains(field)) { + selectivity *= value; + estimated = true; + } + } + // Only predicates that can drive this index reduce its candidates. Uncovered fragments + // still need a full scan, and unrelated residuals must not make this index look cheaper. + return estimated ? Math.max(0, totalRows - coveredRows) + coveredRows * selectivity : Double.NaN; + } + private static Set collectFilterFields(LanceTableMetadata metadata, List pushedConjuncts, IndexType indexType) { Set fields = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java index ebd18437d20eaf..8d267832fa6af6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java @@ -263,10 +263,15 @@ public List getSplits(int numBackends) throws UserException { * Math.max(numBackends, 1)); } scanPlan = LanceScanPlanner.plan(plannedMetadata, searchKind, externalSearchRequest, searchFieldId, - lancePushedConjuncts, sessionVariable.lanceFragmentsPerSplit, countParallelism); + lancePushedConjuncts, sessionVariable.lanceFragmentsPerSplit, countParallelism, + isScalarIndexAutoSelectionEnabled()); return scanPlan.createSchedulingSplits(); } + public boolean isScalarIndexAutoSelectionEnabled() { + return searchKind == SearchKind.NORMAL && sessionVariable.enableLanceScalarIndexAutoSelection; + } + @Override protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { if (!(split instanceof LanceSplit)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java index 96925ff239929f..7289d86afc107c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java @@ -56,6 +56,7 @@ final class LanceScanPlanner { private final List lancePushedConjuncts; private final int fragmentsPerSplit; private final int countParallelism; + private final boolean scalarIndexAutoSelection; private int plannedFragments; private int plannedFragmentsPerSplit; private int plannedUnindexedFragments; @@ -65,7 +66,8 @@ final class LanceScanPlanner { private VectorIndexStatus vectorIndexStatus = VectorIndexStatus.NOT_PLANNED; private LanceScanPlanner(LanceTableMetadata metadata, SearchKind kind, TExternalSearchRequest request, - int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism) { + int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism, + boolean scalarIndexAutoSelection) { this.plannedMetadata = metadata; this.searchKind = kind; this.externalSearchRequest = request; @@ -73,13 +75,20 @@ private LanceScanPlanner(LanceTableMetadata metadata, SearchKind kind, TExternal this.lancePushedConjuncts = pushedConjuncts; this.fragmentsPerSplit = fragmentsPerSplit; this.countParallelism = countParallelism; + this.scalarIndexAutoSelection = scalarIndexAutoSelection; } static LanceScanPlan plan(LanceTableMetadata metadata, SearchKind kind, TExternalSearchRequest request, int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism) throws UserException { + return plan(metadata, kind, request, fieldId, pushedConjuncts, fragmentsPerSplit, countParallelism, true); + } + + static LanceScanPlan plan(LanceTableMetadata metadata, SearchKind kind, TExternalSearchRequest request, + int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism, + boolean scalarIndexAutoSelection) throws UserException { try (LanceMetadataMetrics metrics = LanceMetadataMetrics.startSplitPlanning()) { LanceScanPlanner planner = new LanceScanPlanner(metadata, kind, request, - fieldId, pushedConjuncts, fragmentsPerSplit, countParallelism); + fieldId, pushedConjuncts, fragmentsPerSplit, countParallelism, scalarIndexAutoSelection); List splits = planner.buildSplits(); LanceScanPlan plan = new LanceScanPlan(splits, planner.plannedFragments, planner.plannedFragmentsPerSplit, planner.plannedIndexSegments, planner.plannedIndexFragments, @@ -193,7 +202,8 @@ private List createNormalFragmentSplits(LanceTableMetadata metadata, return LanceSplitBuilder.buildFragmentSplits(metadata.getDatasetUri(), metadata.getVersion(), visibleFragments.values(), plannedFragmentsPerSplit); } - scalarIndexPlan = LanceScalarIndexPlanner.plan(metadata, lancePushedConjuncts, visibleFragments); + scalarIndexPlan = LanceScalarIndexPlanner.plan( + metadata, lancePushedConjuncts, visibleFragments, scalarIndexAutoSelection); LanceSplitBuilder plan; if (scalarIndexPlan != null) { plan = scalarIndexPlan.splits; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 36f8c4eb59844b..fef84610088cc4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -100,6 +100,7 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup; import org.apache.doris.nereids.rules.rewrite.MergeLimits; +import org.apache.doris.nereids.stats.FilterEstimation; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.UnaryNode; import org.apache.doris.nereids.trees.expressions.AggregateExpression; @@ -243,6 +244,7 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.StatisticConstants; +import org.apache.doris.statistics.Statistics; import org.apache.doris.tablefunction.TableValuedFunctionIf; import org.apache.doris.thrift.TExternalTableSinkHashAlgorithm; import org.apache.doris.thrift.TExternalTableSinkWriterAssignment; @@ -3069,7 +3071,18 @@ private void addConjunctsToPlanNode(PhysicalFilter filter, PlanTranslatorContext context) { for (Expression conjunct : filter.getConjuncts()) { for (Expression singleConjunct : ExpressionUtils.extractConjunctionToSet(conjunct)) { - planNode.addConjunct(ExpressionTranslator.translate(singleConjunct, context)); + Expr translated = ExpressionTranslator.translate(singleConjunct, context); + if (planNode instanceof LanceScanNode + && ((LanceScanNode) planNode).isScalarIndexAutoSelectionEnabled() + && filter.child().getStats() != null && filter.child().getStats().getRowCount() > 0) { + double inputRows = filter.child().getStats().getRowCount(); + // Estimation may normalize column statistics. Keep the optimizer's input + // unchanged, and attach only an advisory cost to the translated predicate. + double outputRows = new FilterEstimation().estimate( + singleConjunct, new Statistics(filter.child().getStats())).getRowCount(); + translated.setSelectivity(outputRows / inputRows); + } + planNode.addConjunct(translated); } } updateLegacyPlanIdToPhysicalPlan(planNode, filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 68ecf81c998c08..eaba30d5caf050 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -587,6 +587,8 @@ public String toString() { public static final String ENABLE_LANCE_LAZY_MATERIALIZATION = "enable_lance_lazy_materialization"; + public static final String ENABLE_LANCE_SCALAR_INDEX_AUTO_SELECTION = "enable_lance_scalar_index_auto_selection"; + public static final String FILE_SPLIT_SIZE_ON_FE = "file_split_size_on_fe"; public static final String FILE_SPLIT_SIZE_ON_BE = "file_split_size_on_be"; @@ -2670,6 +2672,13 @@ public Map getForceEagerAggHintMap() { + "Enabled by default for eligible columns in vector_search and full_text_search."}) public boolean enableLanceLazyMaterialization = true; + @VariableMgr.VarAttr(name = ENABLE_LANCE_SCALAR_INDEX_AUTO_SELECTION, needForward = true, + description = { + "根据过滤选择率和索引覆盖范围自动选择 Lance 标量索引，默认开启。关闭后保留按覆盖行数选择的行为。", + "Select a Lance scalar index using predicate selectivity and fragment coverage. " + + "Enabled by default; disabling retains coverage-based selection."}) + public boolean enableLanceScalarIndexAutoSelection = true; + @VariableMgr.VarAttr(name = FILE_SPLIT_SIZE_ON_FE, needForward = true, description = { "支持 BE 细粒度切分时，FE 粗粒度文件分片的目标大小，单位为字节，默认为 512MB", "Target size in bytes for FE coarse-grained file splits when BE refinement is supported. " diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java index d94e2f554740ef..12a8f578d31958 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java @@ -53,6 +53,81 @@ import java.util.UUID; public class LanceScalarIndexPlannerTest { + @Test + public void testSelectsLowerEstimatedCandidateCount() { + Expr broad = estimatedEqual("key", 0.8); + Expr selective = estimatedEqual("other", 0.01); + Assertions.assertEquals("other_idx", competingPlan(broad, selective, true).indexName); + } + + @Test + public void testUncoveredRowsContributeToIndexCost() { + Expr broad = estimatedEqual("key", 0.2); + Expr selective = estimatedEqual("other", 0.01); + // Reading the uncovered half costs more than using the fully covered index. + Assertions.assertEquals("key_idx", competingPlan(broad, selective, false).indexName); + } + + @Test + public void testUnknownEstimatesRetainCoverageAndStableNameOrder() { + Assertions.assertEquals("key_idx", competingPlan(equal("key", 1), equal("other", 1), true).indexName); + Assertions.assertEquals("key_idx", competingPlan(estimatedEqual("key", Double.NaN), + estimatedEqual("other", Double.POSITIVE_INFINITY), true).indexName); + } + + private static Expr estimatedEqual(String column, double selectivity) { + Expr predicate = equal(column, 1); + predicate.setSelectivity(selectivity); + return predicate; + } + + private static LanceScalarIndexPlanner.Plan competingPlan(Expr first, Expr second, boolean fullCoverage) { + return competingPlan(first, second, fullCoverage, true); + } + + private static LanceScalarIndexPlanner.Plan competingPlan(Expr first, Expr second, + boolean fullCoverage, boolean autoSelection) { + List fragments = Arrays.asList( + new LanceFragmentInfo(1, 100, 100), new LanceFragmentInfo(2, 100, 100)); + Map visible = new HashMap<>(); + fragments.forEach(fragment -> visible.put(fragment.getId(), fragment)); + Map fields = new HashMap<>(); + fields.put("key", 9); + fields.put("other", 10); + LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes( + new LanceTableAccess("s3://bucket/table.lance", Collections.emptyMap()), 42, + new Schema(Arrays.asList(Field.nullable("key", new ArrowType.Int(64, true)), + Field.nullable("other", new ArrowType.Int(64, true)))), fragments, fields, + Arrays.asList(new LanceIndexSegmentInfo(UUID.randomUUID(), "key_idx", Collections.singletonList(9), + Arrays.asList(1L, 2L), IndexType.BTREE, null), + new LanceIndexSegmentInfo(UUID.randomUUID(), "other_idx", Collections.singletonList(10), + fullCoverage ? Arrays.asList(1L, 2L) : Collections.singletonList(1L), + IndexType.BTREE, null))); + return LanceScalarIndexPlanner.plan(metadata, Arrays.asList(first, second), visible, autoSelection); + } + + @Test + public void testDisablingAutoSelectionRetainsCoveragePlan() { + Assertions.assertEquals("key_idx", competingPlan(estimatedEqual("key", 0.8), + estimatedEqual("other", 0.01), true, false).indexName); + } + + @Test + public void testZeroSelectivityAndInvalidEstimates() { + Assertions.assertEquals("other_idx", competingPlan(estimatedEqual("key", 0.8), + estimatedEqual("other", 0), true).indexName); + Assertions.assertEquals("key_idx", competingPlan(estimatedEqual("key", -1), + estimatedEqual("other", 2), true).indexName); + } + + @Test + public void testResidualOnAnotherFieldDoesNotReduceDriverCost() { + Expr mixed = and(equal("key", 1), new BinaryPredicate(BinaryPredicate.Operator.NE, + new SlotRef(null, "other"), new IntLiteral(0))); + mixed.setSelectivity(0.0001); + Assertions.assertEquals("other_idx", competingPlan(mixed, estimatedEqual("other", 0.5), true).indexName); + } + @Test public void testBooleanDriverSelectionPreservesAllBranches() { Expr key = equal("key", 1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java index 32173603c8f0f3..b42dd42c8cec1a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java @@ -80,6 +80,47 @@ public class LanceScanNodeTest { + @Test + public void testScalarAutoSelectionSwitchSurvivesPushdownAndForwarding() throws Exception { + UUID keyIndex = UUID.randomUUID(); + UUID otherIndex = UUID.randomUUID(); + Map fields = new LinkedHashMap<>(); + fields.put("key", 9); + fields.put("other", 10); + LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes( + new LanceTableAccess("s3://bucket/table.lance", Collections.emptyMap()), 42, + new Schema(Arrays.asList(Field.nullable("key", new ArrowType.Int(64, true)), + Field.nullable("other", new ArrowType.Int(64, true)))), + Arrays.asList(new LanceFragmentInfo(1, 100, 100), new LanceFragmentInfo(2, 100, 100)), fields, + Arrays.asList(new LanceIndexSegmentInfo(keyIndex, "key_idx", Collections.singletonList(9), + Arrays.asList(1L, 2L), IndexType.BTREE, null), + new LanceIndexSegmentInfo(otherIndex, "other_idx", Collections.singletonList(10), + Arrays.asList(1L, 2L), IndexType.BTREE, null))); + for (boolean enabled : Arrays.asList(true, false)) { + SessionVariable variables = new SessionVariable(); + Assert.assertTrue(variables.enableLanceScalarIndexAutoSelection); + variables.enableLanceScalarIndexAutoSelection = enabled; + Assert.assertEquals(Boolean.toString(enabled), variables.getForwardVariables().get( + SessionVariable.ENABLE_LANCE_SCALAR_INDEX_AUTO_SELECTION)); + LanceScanNode node = newNode(variables); + setMetadata(node, metadata); + Expr broad = scalarPredicate(); + broad.setSelectivity(0.8); + Expr selective = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(null, "other"), new IntLiteral(1)); + selective.setSelectivity(0.01); + node.getConjuncts().addAll(Arrays.asList(broad, selective)); + node.convertPredicate(); + Assert.assertTrue(node.getConjuncts().isEmpty()); + List splits = node.getSplits(1); + Assert.assertEquals(1, splits.size()); + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, splits.get(0)); + ByteBuffer uuid = range.getTableFormatParams().getLanceParams().getIndexSegmentUuids().get(0).duplicate(); + Assert.assertEquals(enabled ? otherIndex : keyIndex, new UUID(uuid.getLong(), uuid.getLong())); + } + } + @Test public void testSplitPlanningReportsSuccessAndFailureToQueryProfile() throws Exception { ConnectContext previous = ConnectContext.get(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java new file mode 100644 index 00000000000000..e66ebc98eebf2c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.datasource.lance.source.LanceScanNode; +import org.apache.doris.nereids.properties.DataTrait; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.ScanContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.statistics.ColumnStatistic; +import org.apache.doris.statistics.ColumnStatisticBuilder; +import org.apache.doris.statistics.Statistics; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +public class LanceScalarIndexSelectivityTest { + @Test + public void testTranslationUsesIndependentColumnEstimatesAndHonorsSwitch() throws Exception { + for (boolean enabled : Arrays.asList(true, false)) { + SlotReference broad = new SlotReference("broad", BigIntType.INSTANCE); + SlotReference selective = new SlotReference("selective", BigIntType.INSTANCE); + Map columns = new HashMap<>(); + columns.put(broad, columnStats(10)); + columns.put(selective, columnStats(1000)); + Statistics input = new Statistics(1000, columns); + LogicalProperties properties = new LogicalProperties( + () -> Arrays.asList(broad, selective), () -> DataTrait.EMPTY_TRAIT); + PhysicalEmptyRelation child = new PhysicalEmptyRelation(new RelationId(0), + Arrays.asList(broad, selective), properties); + child.setStatistics(input); + PhysicalFilter filter = new PhysicalFilter<>(new HashSet<>(Arrays.asList( + new EqualTo(broad, new BigIntLiteral(1)), + new EqualTo(selective, new BigIntLiteral(1)))), properties, child); + PlanTranslatorContext context = new PlanTranslatorContext(); + TupleDescriptor tuple = context.generateTupleDesc(); + context.createSlotDesc(tuple, broad); + context.createSlotDesc(tuple, selective); + SessionVariable variables = new SessionVariable(); + variables.enableLanceScalarIndexAutoSelection = enabled; + LanceScanNode node = new LanceScanNode(new PlanNodeId(0), tuple, false, variables, ScanContext.EMPTY); + Method method = PhysicalPlanTranslator.class.getDeclaredMethod("addConjunctsToPlanNode", + PhysicalFilter.class, PlanNode.class, PlanTranslatorContext.class); + method.setAccessible(true); + method.invoke(new PhysicalPlanTranslator(context), filter, node, context); + Assertions.assertEquals(2, node.getConjuncts().size()); + for (Expr predicate : node.getConjuncts()) { + if (enabled) { + String column = ((SlotRef) predicate.getChild(0)).getDesc().getLabel(); + Assertions.assertEquals("broad".equals(column) ? 0.1 : 0.001, + predicate.getSelectivity(), 1e-9, column + ": " + predicate.toSql()); + } else { + Assertions.assertFalse(predicate.hasSelectivity()); + } + } + Assertions.assertEquals(1000, input.getRowCount()); + Assertions.assertEquals(10, input.findColumnStatistics(broad).ndv); + Assertions.assertEquals(1000, input.findColumnStatistics(selective).ndv); + } + } + + private static ColumnStatistic columnStats(double ndv) { + return new ColumnStatisticBuilder(1000).setNdv(ndv).setNumNulls(0) + .setMinValue(0).setMaxValue(1000).setAvgSizeByte(8).setDataSize(8000).build(); + } +} From a4e757b9a7dea0b114c27988d87951a83f910dc7 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 26 Oct 2026 17:33:09 +0800 Subject: [PATCH 2/5] Move Lance scalar index cost selection to BE runtime --- be/src/common/config.cpp | 1 + be/src/common/config.h | 2 + be/src/format_v2/lance/lance_reader_helper.h | 20 + be/src/format_v2/table/lance_reader.cpp | 9 + be/test/format_v2/table/lance_reader_test.cpp | 171 +++-- docs/lance-cache-and-index-selection.md | 55 +- .../java/org/apache/doris/analysis/Expr.java | 5 - .../lance/source/LanceScalarIndexPlanner.java | 43 +- .../lance/source/LanceScanNode.java | 7 +- .../lance/source/LanceScanPlanner.java | 16 +- .../translator/PhysicalPlanTranslator.java | 15 +- .../org/apache/doris/qe/SessionVariable.java | 9 - .../source/LanceScalarIndexPlannerTest.java | 81 +- .../lance/source/LanceScanNodeTest.java | 41 - .../LanceScalarIndexSelectivityTest.java | 100 --- thirdparty/download-thirdparty.sh | 8 +- .../lance-c-runtime-index-selection.patch | 723 ++++++++++++++++++ 17 files changed, 902 insertions(+), 404 deletions(-) delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java create mode 100644 thirdparty/patches/lance-c-runtime-index-selection.patch diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ad8f2617dfc60b..cbb48a518d14f4 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1228,6 +1228,7 @@ DEFINE_Validator(variant_storage_parse_mode, // can be shared by all Lance dataset readers. DEFINE_Int64(lance_index_cache_size_bytes, "10737418240"); // 10 GiB DEFINE_Int64(lance_metadata_cache_size_bytes, "1073741824"); // 1GB +DEFINE_Bool(enable_lance_scalar_index_auto_selection, "true"); DEFINE_Bool(enable_lance_data_cache, "true"); DEFINE_String(lance_data_cache_path, "${DORIS_HOME}/lance_data_cache"); DEFINE_Int64(lance_data_cache_disk_capacity_bytes, "107374182400"); // 100GB diff --git a/be/src/common/config.h b/be/src/common/config.h index 1758ac1ebd7a52..5c5b11e7748b42 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1264,6 +1264,8 @@ DECLARE_Int32(blocking_pipeline_executor_size); // Lance shared session and optional Foyer data-file cache. DECLARE_Int64(lance_index_cache_size_bytes); DECLARE_Int64(lance_metadata_cache_size_bytes); +// Runtime BTree selection uses page metadata and preserves the explicit fragment domain. +DECLARE_Bool(enable_lance_scalar_index_auto_selection); DECLARE_Bool(enable_lance_data_cache); DECLARE_String(lance_data_cache_path); DECLARE_Int64(lance_data_cache_disk_capacity_bytes); diff --git a/be/src/format_v2/lance/lance_reader_helper.h b/be/src/format_v2/lance/lance_reader_helper.h index 1f07fbd4bcb66d..d5fa76ede15d9a 100644 --- a/be/src/format_v2/lance/lance_reader_helper.h +++ b/be/src/format_v2/lance/lance_reader_helper.h @@ -141,6 +141,26 @@ inline constexpr LanceProfileMetric LANCE_SCAN_METRICS[] = { .profile_name = "LanceIVFPartitionSelectionTime", .group = LANCE_VECTOR_INDEX_PROFILE, .unit = TUnit::TIME_NS}, + {.native_name = "scalar_index_auto_candidates", + .profile_name = "LanceScalarIndexAutoCandidates", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_index_auto_changed", + .profile_name = "LanceScalarIndexAutoChanged", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_index_auto_estimated_pages", + .profile_name = "LanceScalarIndexAutoEstimatedPages", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_index_auto_estimated_rows", + .profile_name = "LanceScalarIndexAutoEstimatedRows", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, + {.native_name = "scalar_index_auto_full_scans", + .profile_name = "LanceScalarIndexAutoFullScans", + .group = LANCE_SCALAR_INDEX_PROFILE, + .unit = TUnit::UNIT}, {.native_name = "scalar_segments_requested", .profile_name = "LanceScalarIndexSegmentsRequested", .group = LANCE_SCALAR_INDEX_PROFILE, diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index 265d8a3284e1ce..462549f76c398c 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -111,6 +111,11 @@ Status LanceTableReader::init(TableReadOptions&& options) { DORIS_CHECK(_scanner_profile != nullptr); DORIS_CHECK(_scan_params != nullptr); RETURN_IF_ERROR(_resolve_search_kind()); + if (_search_kind == SearchKind::NORMAL) { + _scanner_profile->add_info_string( + "LanceScalarIndexAutoSelection", + config::enable_lance_scalar_index_auto_selection ? "true" : "false"); + } const auto& lance_scan_params = _scan_params->lance_scan_params; ADD_CHILD_COUNTER_WITH_LEVEL(_scanner_profile, LANCE_READER_PROFILE, TUnit::NONE, @@ -959,6 +964,10 @@ Status LanceTableReader::_configure_normal_scan(LanceScanner* scanner, return Status::InvalidArgument( "Lance scalar index segment cannot be combined with use_scalar_index=false"); } + if (lance_scanner_set_scalar_index_auto_selection( + scanner, config::enable_lance_scalar_index_auto_selection) != 0) { + return lance_error("set Lance runtime scalar index selection"); + } if (lance_scanner_set_scalar_index_segment(scanner, segment_uuids.data()) != 0) { return lance_error("set Lance scanner scalar index segment"); } diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 58ffef582939c5..c6f3cc879f2c16 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -2272,87 +2272,100 @@ TEST(LanceTableReaderFilterTest, CombinesStaticSubstraitFilterWithRuntimeFilter) } TEST(LanceTableReaderScalarSegmentTest, FiltersIndexedAndUncoveredDomainsWithoutLosingRows) { - const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); - for (const auto index_type : {LANCE_SCALAR_BTREE, LANCE_SCALAR_BITMAP}) { - SCOPED_TRACE(index_type == LANCE_SCALAR_BTREE ? "BTREE" : "BITMAP"); - const auto dataset_uri = std::filesystem::temp_directory_path() / - ("doris_lance_scalar_segment_" + std::to_string(unique_suffix) + - "_" + std::to_string(index_type) + ".lance"); - Defer cleanup {[&] { - std::error_code error; - std::filesystem::remove_all(dataset_uri, error); - }}; - const auto schema = arrow::schema({arrow::field("row_id", arrow::int64(), false)}); - for (int64_t first_id : {1, 3}) { - arrow::Int64Builder values; - ASSERT_TRUE(values.AppendValues({first_id, first_id + 1}).ok()); - auto array = values.Finish(); - ASSERT_TRUE(array.ok()); - auto batch = arrow::RecordBatch::Make(schema, 2, {std::move(array).ValueUnsafe()}); - auto batches = arrow::RecordBatchReader::Make({batch}, schema); - ASSERT_TRUE(batches.ok()); - ArrowArrayStream stream {}; - ASSERT_TRUE( - arrow::ExportRecordBatchReader(std::move(batches).ValueUnsafe(), &stream).ok()); - auto dataset = ::lance::Dataset::write( - dataset_uri.string(), &stream, - first_id == 1 ? ::lance::WriteMode::Create : ::lance::WriteMode::Append); - if (first_id == 1) { - // The second append remains uncovered by this index segment. - dataset.create_scalar_index("row_id", index_type, "row_id_idx"); - } - } - LanceFixtureInfo fixture; - ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); - ASSERT_EQ(2, fixture.fragment_ids.size()); - std::vector segments; - ASSERT_TRUE(get_index_segment_uuids(dataset_uri, "row_id_idx", &segments).ok()); - ASSERT_EQ(1, segments.size()); - - const auto read_domain = [&](const std::vector& fragments, bool use_segment, - const std::vector& expected) { - SCOPED_TRACE(::testing::PrintToString(fragments)); - SCOPED_TRACE(use_segment ? "selected segment" : "scalar index disabled"); - TQueryGlobals globals; - RuntimeState state(globals); - RuntimeProfile profile("lance_scalar_segment"); - TFileScanRangeParams scan_params; - const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; - LanceTableReader reader; - auto filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 41); - ASSERT_TRUE( - init_reader(&reader, columns, &state, &profile, &scan_params, {filter}).ok()); - auto range = make_lance_range(dataset_uri, fixture.version, fragments); - if (use_segment) { - range.table_format_params.lance_params.__set_index_segment_uuids(segments); - } else { - range.table_format_params.lance_params.__set_use_scalar_index(false); - } - ASSERT_TRUE(prepare_range(&reader, std::move(range)).ok()); - Block block; - add_output_columns(&block, columns); - std::vector actual; - bool eos = false; - while (!eos) { - auto status = reader.get_block(&block, &eos); - ASSERT_TRUE(status.ok()) << status.to_string(); - if (!eos) { - const auto& ids = - assert_cast(*block.get_by_position(0).column); - actual.insert(actual.end(), ids.get_data().begin(), ids.get_data().end()); + const auto previous_auto_selection = config::enable_lance_scalar_index_auto_selection; + Defer restore_config { + [&] { config::enable_lance_scalar_index_auto_selection = previous_auto_selection; }}; + for (const bool auto_selection : {false, true}) { + config::enable_lance_scalar_index_auto_selection = auto_selection; + const auto unique_suffix = std::chrono::steady_clock::now().time_since_epoch().count(); + for (const auto index_type : {LANCE_SCALAR_BTREE, LANCE_SCALAR_BITMAP}) { + SCOPED_TRACE(index_type == LANCE_SCALAR_BTREE ? "BTREE" : "BITMAP"); + const auto dataset_uri = + std::filesystem::temp_directory_path() / + ("doris_lance_scalar_segment_" + std::to_string(unique_suffix) + "_" + + std::to_string(index_type) + ".lance"); + Defer cleanup {[&] { + std::error_code error; + std::filesystem::remove_all(dataset_uri, error); + }}; + const auto schema = arrow::schema({arrow::field("row_id", arrow::int64(), false)}); + for (int64_t first_id : {1, 3}) { + arrow::Int64Builder values; + ASSERT_TRUE(values.AppendValues({first_id, first_id + 1}).ok()); + auto array = values.Finish(); + ASSERT_TRUE(array.ok()); + auto batch = arrow::RecordBatch::Make(schema, 2, {std::move(array).ValueUnsafe()}); + auto batches = arrow::RecordBatchReader::Make({batch}, schema); + ASSERT_TRUE(batches.ok()); + ArrowArrayStream stream {}; + ASSERT_TRUE( + arrow::ExportRecordBatchReader(std::move(batches).ValueUnsafe(), &stream) + .ok()); + auto dataset = ::lance::Dataset::write( + dataset_uri.string(), &stream, + first_id == 1 ? ::lance::WriteMode::Create : ::lance::WriteMode::Append); + if (first_id == 1) { + // The second append remains uncovered by this index segment. + dataset.create_scalar_index("row_id", index_type, "row_id_idx"); } } - std::ranges::sort(actual); - EXPECT_EQ(expected, actual); - EXPECT_TRUE(reader.close().ok()); - }; - read_domain({fixture.fragment_ids[0]}, true, {2}); - read_domain({fixture.fragment_ids[1]}, false, {4}); - // A segment which cannot cover the whole task must filter the entire explicit domain. - read_domain(fixture.fragment_ids, true, {2, 4}); - // Disabling the index must produce the same rows in each read domain. - read_domain({fixture.fragment_ids[0]}, false, {2}); - read_domain(fixture.fragment_ids, false, {2, 4}); + LanceFixtureInfo fixture; + ASSERT_TRUE(get_fixture_info(dataset_uri, &fixture).ok()); + ASSERT_EQ(2, fixture.fragment_ids.size()); + std::vector segments; + ASSERT_TRUE(get_index_segment_uuids(dataset_uri, "row_id_idx", &segments).ok()); + ASSERT_EQ(1, segments.size()); + + const auto read_domain = [&](const std::vector& fragments, bool use_segment, + const std::vector& expected) { + SCOPED_TRACE(::testing::PrintToString(fragments)); + SCOPED_TRACE(use_segment ? "selected segment" : "scalar index disabled"); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_scalar_segment"); + TFileScanRangeParams scan_params; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + LanceTableReader reader; + auto filter = create_int64_runtime_in_conjunct("row_id", {2, 4}, 41); + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params, {filter}) + .ok()); + auto range = make_lance_range(dataset_uri, fixture.version, fragments); + if (use_segment) { + range.table_format_params.lance_params.__set_index_segment_uuids(segments); + } else { + range.table_format_params.lance_params.__set_use_scalar_index(false); + } + ASSERT_TRUE(prepare_range(&reader, std::move(range)).ok()); + Block block; + add_output_columns(&block, columns); + std::vector actual; + bool eos = false; + while (!eos) { + auto status = reader.get_block(&block, &eos); + ASSERT_TRUE(status.ok()) << status.to_string(); + if (!eos) { + const auto& ids = + assert_cast(*block.get_by_position(0).column); + actual.insert(actual.end(), ids.get_data().begin(), ids.get_data().end()); + } + } + std::ranges::sort(actual); + EXPECT_EQ(expected, actual); + EXPECT_TRUE(reader.close().ok()); + auto* candidates = profile.get_counter("LanceScalarIndexAutoCandidates"); + ASSERT_NE(nullptr, candidates); + const bool estimable = auto_selection && use_segment && + index_type == LANCE_SCALAR_BTREE && fragments.size() == 1; + EXPECT_EQ(estimable ? 1 : 0, candidates->value()); + }; + read_domain({fixture.fragment_ids[0]}, true, {2}); + read_domain({fixture.fragment_ids[1]}, false, {4}); + // A segment which cannot cover the whole task must filter the entire explicit domain. + read_domain(fixture.fragment_ids, true, {2, 4}); + // Disabling the index must produce the same rows in each read domain. + read_domain({fixture.fragment_ids[0]}, false, {2}); + read_domain(fixture.fragment_ids, false, {2, 4}); + } } } diff --git a/docs/lance-cache-and-index-selection.md b/docs/lance-cache-and-index-selection.md index d696568d808139..16891fbd4171e3 100644 --- a/docs/lance-cache-and-index-selection.md +++ b/docs/lance-cache-and-index-selection.md @@ -26,31 +26,46 @@ summing reader windows. These counters do not measure the FE's separate metadata ## Automatic scalar index selection -```sql -SET enable_lance_scalar_index_auto_selection = true; -``` +`enable_lance_scalar_index_auto_selection` is a startup BE configuration, enabled +by default. Restart the BE after changing it. There is no FE session variable. -This session variable is enabled by default. For ordinary Lance table scans, the FE -uses Nereids predicate selectivity estimates to choose at most one eligible logical -BTree, Bitmap, or LabelList index. The estimated work is: +FE retains its existing coverage-based grouping and assigns each task an explicit, +disjoint fragment domain. In BE, lance-c can replace the grouping index with a +BTree segment whose known coverage contains the **entire** task domain. It uses +the current snapshot's BTree page lookup (page min/max and null counts), without +Doris column statistics or `ANALYZE`. It estimates matching pages for equality, +range, small IN lists, IS NULL, and supported AND/OR combinations: ```text -uncovered physical rows + covered physical rows * driver selectivity +estimated work = matching index pages * rows per index page ``` -Only indexable predicates on the selected field reduce its estimated candidates. -Estimates for independent conjuncts on that field are multiplied; they are advisory -and can be inaccurate for correlated predicates or stale/missing column statistics. -Multi-field predicates are not used to estimate a single index's candidates. Without -usable estimates, selection retains the existing largest-coverage rule and stable -index-name tie breaking. Existing eligibility, expression-budget and fragment-ownership -checks still apply. Uncovered fragments retain their non-indexed fallback scans. -`EXPLAIN` shows the chosen index as `lanceGroupingIndex`. - -Setting the variable to `false` retains the previous coverage-based selection; it -does not disable scalar indexes. The `lance_fragments_per_split` debug override still -bypasses index-segment grouping. This setting does not alter vector/FTS index selection -or move an external-search TVF's outer `WHERE` predicates before candidate selection. +This is a conservative page-based estimate, not an exact candidate count or a +latency measurement. Intersected AND predicates may still match different values +inside the same page. Cache residency and the cost of fetching projected columns +are not modeled. Only a strictly cheaper estimate replaces the grouping index; +ties retain it. If the best estimate is at least the task's physical row count, +BE scans that domain with the complete filter instead of searching an index. + +Each selection examines at most eight segments and an aggregate 8 MiB of lookup +files; a lookup has at most 65,536 pages, an IN list at most 256 values, and +one estimate at most 1,000,000 page/predicate comparisons. Lookup +batches share the bounded Session metadata cache. Probing does not search index +leaf pages or materialize candidate row IDs. Unknown file sizes, unsupported +expressions/formats, probe errors, or an unestimable grouping index retain the +existing segment path. Bitmap and LabelList remain supported on that path, but +are not cost-ranked. Uncovered fragments keep their existing fallback scans. + +`LanceScalarIndexAutoCandidates`, `LanceScalarIndexAutoChanged`, +`LanceScalarIndexAutoEstimatedPages`, `LanceScalarIndexAutoEstimatedRows`, and +`LanceScalarIndexAutoFullScans` expose runtime decisions in Profile. Estimates +are accumulated over tasks. `EXPLAIN`'s `lanceGroupingIndex` identifies FE's task +grouping, which can differ from the runtime index. + +Setting the BE option to `false` retains the previous segment behavior and avoids +these probes. The `lance_fragments_per_split` debug override still bypasses +index-segment grouping. Vector/FTS index selection and external-search TVF filter +semantics are unchanged. ## Foyer memory capacity diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index a9ca5337b19b4a..253d45fa743ad5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -238,11 +238,6 @@ public double getSelectivity() { return selectivity; } - public void setSelectivity(double selectivity) { - this.selectivity = Double.isFinite(selectivity) && selectivity >= 0 && selectivity <= 1 - ? selectivity : -1; - } - public boolean hasSelectivity() { return selectivity >= 0; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java index db1aa45c667510..4d6319242f6984 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlanner.java @@ -63,11 +63,6 @@ static final class Plan { static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, Map visibleFragments) { - return plan(metadata, pushedConjuncts, visibleFragments, true); - } - - static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, - Map visibleFragments, boolean autoSelection) { if (metadata.getVersion() <= 0 || !metadata.getIndexMetadataState().canPlanIndexSegments()) { return null; @@ -91,9 +86,6 @@ static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, } } Plan selected = null; - double selectedCost = Double.NaN; - double totalRows = visibleFragments.values().stream() - .mapToDouble(fragment -> Math.max(fragment.getPhysicalRows(), 0)).sum(); Map> fieldsByIndexType = new HashMap<>(); for (LanceIndexSegmentGroup logicalIndex : indices) { List segments = logicalIndex.getSegments(); @@ -111,46 +103,13 @@ static Plan plan(LanceTableMetadata metadata, List pushedConjuncts, continue; } Plan candidate = groupFragments(metadata, segments, visibleFragments); - if (candidate == null) { - continue; - } - double cost = autoSelection ? estimateCandidateRows(metadata, pushedConjuncts, - index, candidate.coveredRows, totalRows) : Double.NaN; - int comparison = Double.isNaN(cost) && Double.isNaN(selectedCost) ? 0 - : Double.compare(Double.isNaN(cost) ? totalRows : cost, - Double.isNaN(selectedCost) ? totalRows : selectedCost); - if (selected == null || comparison < 0 - || (comparison == 0 && candidate.coveredRows > selected.coveredRows)) { + if (candidate != null && (selected == null || candidate.coveredRows > selected.coveredRows)) { selected = candidate; - selectedCost = cost; } } return selected; } - private static double estimateCandidateRows(LanceTableMetadata metadata, List conjuncts, - LanceIndexSegmentInfo index, long coveredRows, double totalRows) { - double selectivity = 1; - boolean estimated = false; - int field = index.getFieldIds().get(0); - for (Expr conjunct : conjuncts) { - double value = conjunct.getSelectivity(); - Set slots = new HashSet<>(); - conjunct.collect(SlotRef.class, slots); - boolean singleField = !slots.isEmpty() && slots.stream().allMatch(slot -> - metadata.getLanceFieldId(slot.getColumnName()).orElse(-1) == field); - if (Double.isFinite(value) && value >= 0 && value <= 1 - && singleField - && collectDriverFields(metadata, conjunct, index.getIndexType(), false).contains(field)) { - selectivity *= value; - estimated = true; - } - } - // Only predicates that can drive this index reduce its candidates. Uncovered fragments - // still need a full scan, and unrelated residuals must not make this index look cheaper. - return estimated ? Math.max(0, totalRows - coveredRows) + coveredRows * selectivity : Double.NaN; - } - private static Set collectFilterFields(LanceTableMetadata metadata, List pushedConjuncts, IndexType indexType) { Set fields = new HashSet<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java index 8d267832fa6af6..ebd18437d20eaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanNode.java @@ -263,15 +263,10 @@ public List getSplits(int numBackends) throws UserException { * Math.max(numBackends, 1)); } scanPlan = LanceScanPlanner.plan(plannedMetadata, searchKind, externalSearchRequest, searchFieldId, - lancePushedConjuncts, sessionVariable.lanceFragmentsPerSplit, countParallelism, - isScalarIndexAutoSelectionEnabled()); + lancePushedConjuncts, sessionVariable.lanceFragmentsPerSplit, countParallelism); return scanPlan.createSchedulingSplits(); } - public boolean isScalarIndexAutoSelectionEnabled() { - return searchKind == SearchKind.NORMAL && sessionVariable.enableLanceScalarIndexAutoSelection; - } - @Override protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { if (!(split instanceof LanceSplit)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java index 7289d86afc107c..96925ff239929f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/source/LanceScanPlanner.java @@ -56,7 +56,6 @@ final class LanceScanPlanner { private final List lancePushedConjuncts; private final int fragmentsPerSplit; private final int countParallelism; - private final boolean scalarIndexAutoSelection; private int plannedFragments; private int plannedFragmentsPerSplit; private int plannedUnindexedFragments; @@ -66,8 +65,7 @@ final class LanceScanPlanner { private VectorIndexStatus vectorIndexStatus = VectorIndexStatus.NOT_PLANNED; private LanceScanPlanner(LanceTableMetadata metadata, SearchKind kind, TExternalSearchRequest request, - int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism, - boolean scalarIndexAutoSelection) { + int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism) { this.plannedMetadata = metadata; this.searchKind = kind; this.externalSearchRequest = request; @@ -75,20 +73,13 @@ private LanceScanPlanner(LanceTableMetadata metadata, SearchKind kind, TExternal this.lancePushedConjuncts = pushedConjuncts; this.fragmentsPerSplit = fragmentsPerSplit; this.countParallelism = countParallelism; - this.scalarIndexAutoSelection = scalarIndexAutoSelection; } static LanceScanPlan plan(LanceTableMetadata metadata, SearchKind kind, TExternalSearchRequest request, int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism) throws UserException { - return plan(metadata, kind, request, fieldId, pushedConjuncts, fragmentsPerSplit, countParallelism, true); - } - - static LanceScanPlan plan(LanceTableMetadata metadata, SearchKind kind, TExternalSearchRequest request, - int fieldId, List pushedConjuncts, int fragmentsPerSplit, int countParallelism, - boolean scalarIndexAutoSelection) throws UserException { try (LanceMetadataMetrics metrics = LanceMetadataMetrics.startSplitPlanning()) { LanceScanPlanner planner = new LanceScanPlanner(metadata, kind, request, - fieldId, pushedConjuncts, fragmentsPerSplit, countParallelism, scalarIndexAutoSelection); + fieldId, pushedConjuncts, fragmentsPerSplit, countParallelism); List splits = planner.buildSplits(); LanceScanPlan plan = new LanceScanPlan(splits, planner.plannedFragments, planner.plannedFragmentsPerSplit, planner.plannedIndexSegments, planner.plannedIndexFragments, @@ -202,8 +193,7 @@ private List createNormalFragmentSplits(LanceTableMetadata metadata, return LanceSplitBuilder.buildFragmentSplits(metadata.getDatasetUri(), metadata.getVersion(), visibleFragments.values(), plannedFragmentsPerSplit); } - scalarIndexPlan = LanceScalarIndexPlanner.plan( - metadata, lancePushedConjuncts, visibleFragments, scalarIndexAutoSelection); + scalarIndexPlan = LanceScalarIndexPlanner.plan(metadata, lancePushedConjuncts, visibleFragments); LanceSplitBuilder plan; if (scalarIndexPlan != null) { plan = scalarIndexPlan.splits; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index fef84610088cc4..36f8c4eb59844b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -100,7 +100,6 @@ import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.implementation.LogicalWindowToPhysicalWindow.WindowFrameGroup; import org.apache.doris.nereids.rules.rewrite.MergeLimits; -import org.apache.doris.nereids.stats.FilterEstimation; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.UnaryNode; import org.apache.doris.nereids.trees.expressions.AggregateExpression; @@ -244,7 +243,6 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import org.apache.doris.statistics.StatisticConstants; -import org.apache.doris.statistics.Statistics; import org.apache.doris.tablefunction.TableValuedFunctionIf; import org.apache.doris.thrift.TExternalTableSinkHashAlgorithm; import org.apache.doris.thrift.TExternalTableSinkWriterAssignment; @@ -3071,18 +3069,7 @@ private void addConjunctsToPlanNode(PhysicalFilter filter, PlanTranslatorContext context) { for (Expression conjunct : filter.getConjuncts()) { for (Expression singleConjunct : ExpressionUtils.extractConjunctionToSet(conjunct)) { - Expr translated = ExpressionTranslator.translate(singleConjunct, context); - if (planNode instanceof LanceScanNode - && ((LanceScanNode) planNode).isScalarIndexAutoSelectionEnabled() - && filter.child().getStats() != null && filter.child().getStats().getRowCount() > 0) { - double inputRows = filter.child().getStats().getRowCount(); - // Estimation may normalize column statistics. Keep the optimizer's input - // unchanged, and attach only an advisory cost to the translated predicate. - double outputRows = new FilterEstimation().estimate( - singleConjunct, new Statistics(filter.child().getStats())).getRowCount(); - translated.setSelectivity(outputRows / inputRows); - } - planNode.addConjunct(translated); + planNode.addConjunct(ExpressionTranslator.translate(singleConjunct, context)); } } updateLegacyPlanIdToPhysicalPlan(planNode, filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index eaba30d5caf050..68ecf81c998c08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -587,8 +587,6 @@ public String toString() { public static final String ENABLE_LANCE_LAZY_MATERIALIZATION = "enable_lance_lazy_materialization"; - public static final String ENABLE_LANCE_SCALAR_INDEX_AUTO_SELECTION = "enable_lance_scalar_index_auto_selection"; - public static final String FILE_SPLIT_SIZE_ON_FE = "file_split_size_on_fe"; public static final String FILE_SPLIT_SIZE_ON_BE = "file_split_size_on_be"; @@ -2672,13 +2670,6 @@ public Map getForceEagerAggHintMap() { + "Enabled by default for eligible columns in vector_search and full_text_search."}) public boolean enableLanceLazyMaterialization = true; - @VariableMgr.VarAttr(name = ENABLE_LANCE_SCALAR_INDEX_AUTO_SELECTION, needForward = true, - description = { - "根据过滤选择率和索引覆盖范围自动选择 Lance 标量索引，默认开启。关闭后保留按覆盖行数选择的行为。", - "Select a Lance scalar index using predicate selectivity and fragment coverage. " - + "Enabled by default; disabling retains coverage-based selection."}) - public boolean enableLanceScalarIndexAutoSelection = true; - @VariableMgr.VarAttr(name = FILE_SPLIT_SIZE_ON_FE, needForward = true, description = { "支持 BE 细粒度切分时，FE 粗粒度文件分片的目标大小，单位为字节，默认为 512MB", "Target size in bytes for FE coarse-grained file splits when BE refinement is supported. " diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java index 12a8f578d31958..4f568b62f45410 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScalarIndexPlannerTest.java @@ -54,78 +54,15 @@ public class LanceScalarIndexPlannerTest { @Test - public void testSelectsLowerEstimatedCandidateCount() { - Expr broad = estimatedEqual("key", 0.8); - Expr selective = estimatedEqual("other", 0.01); - Assertions.assertEquals("other_idx", competingPlan(broad, selective, true).indexName); - } - - @Test - public void testUncoveredRowsContributeToIndexCost() { - Expr broad = estimatedEqual("key", 0.2); - Expr selective = estimatedEqual("other", 0.01); - // Reading the uncovered half costs more than using the fully covered index. - Assertions.assertEquals("key_idx", competingPlan(broad, selective, false).indexName); - } - - @Test - public void testUnknownEstimatesRetainCoverageAndStableNameOrder() { - Assertions.assertEquals("key_idx", competingPlan(equal("key", 1), equal("other", 1), true).indexName); - Assertions.assertEquals("key_idx", competingPlan(estimatedEqual("key", Double.NaN), - estimatedEqual("other", Double.POSITIVE_INFINITY), true).indexName); - } - - private static Expr estimatedEqual(String column, double selectivity) { - Expr predicate = equal(column, 1); - predicate.setSelectivity(selectivity); - return predicate; - } - - private static LanceScalarIndexPlanner.Plan competingPlan(Expr first, Expr second, boolean fullCoverage) { - return competingPlan(first, second, fullCoverage, true); - } - - private static LanceScalarIndexPlanner.Plan competingPlan(Expr first, Expr second, - boolean fullCoverage, boolean autoSelection) { - List fragments = Arrays.asList( - new LanceFragmentInfo(1, 100, 100), new LanceFragmentInfo(2, 100, 100)); - Map visible = new HashMap<>(); - fragments.forEach(fragment -> visible.put(fragment.getId(), fragment)); - Map fields = new HashMap<>(); - fields.put("key", 9); - fields.put("other", 10); - LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes( - new LanceTableAccess("s3://bucket/table.lance", Collections.emptyMap()), 42, - new Schema(Arrays.asList(Field.nullable("key", new ArrowType.Int(64, true)), - Field.nullable("other", new ArrowType.Int(64, true)))), fragments, fields, - Arrays.asList(new LanceIndexSegmentInfo(UUID.randomUUID(), "key_idx", Collections.singletonList(9), - Arrays.asList(1L, 2L), IndexType.BTREE, null), - new LanceIndexSegmentInfo(UUID.randomUUID(), "other_idx", Collections.singletonList(10), - fullCoverage ? Arrays.asList(1L, 2L) : Collections.singletonList(1L), - IndexType.BTREE, null))); - return LanceScalarIndexPlanner.plan(metadata, Arrays.asList(first, second), visible, autoSelection); - } - - @Test - public void testDisablingAutoSelectionRetainsCoveragePlan() { - Assertions.assertEquals("key_idx", competingPlan(estimatedEqual("key", 0.8), - estimatedEqual("other", 0.01), true, false).indexName); - } - - @Test - public void testZeroSelectivityAndInvalidEstimates() { - Assertions.assertEquals("other_idx", competingPlan(estimatedEqual("key", 0.8), - estimatedEqual("other", 0), true).indexName); - Assertions.assertEquals("key_idx", competingPlan(estimatedEqual("key", -1), - estimatedEqual("other", 2), true).indexName); - } - - @Test - public void testResidualOnAnotherFieldDoesNotReduceDriverCost() { - Expr mixed = and(equal("key", 1), new BinaryPredicate(BinaryPredicate.Operator.NE, - new SlotRef(null, "other"), new IntLiteral(0))); - mixed.setSelectivity(0.0001); - Assertions.assertEquals("other_idx", competingPlan(mixed, estimatedEqual("other", 0.5), true).indexName); + public void testGroupingDoesNotDependOnPredicateSelectivity() { + Expr predicate = new BinaryPredicate(BinaryPredicate.Operator.EQ, + new SlotRef(null, "key"), new IntLiteral(1)) { + @Override + public double getSelectivity() { + throw new AssertionError("FE grouping must not estimate runtime index costs"); + } + }; + Assertions.assertEquals("key_idx", plan(predicate).indexName); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java index b42dd42c8cec1a..32173603c8f0f3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/source/LanceScanNodeTest.java @@ -80,47 +80,6 @@ public class LanceScanNodeTest { - @Test - public void testScalarAutoSelectionSwitchSurvivesPushdownAndForwarding() throws Exception { - UUID keyIndex = UUID.randomUUID(); - UUID otherIndex = UUID.randomUUID(); - Map fields = new LinkedHashMap<>(); - fields.put("key", 9); - fields.put("other", 10); - LanceTableMetadata metadata = LanceTableMetadata.createSnapshotWithIndexes( - new LanceTableAccess("s3://bucket/table.lance", Collections.emptyMap()), 42, - new Schema(Arrays.asList(Field.nullable("key", new ArrowType.Int(64, true)), - Field.nullable("other", new ArrowType.Int(64, true)))), - Arrays.asList(new LanceFragmentInfo(1, 100, 100), new LanceFragmentInfo(2, 100, 100)), fields, - Arrays.asList(new LanceIndexSegmentInfo(keyIndex, "key_idx", Collections.singletonList(9), - Arrays.asList(1L, 2L), IndexType.BTREE, null), - new LanceIndexSegmentInfo(otherIndex, "other_idx", Collections.singletonList(10), - Arrays.asList(1L, 2L), IndexType.BTREE, null))); - for (boolean enabled : Arrays.asList(true, false)) { - SessionVariable variables = new SessionVariable(); - Assert.assertTrue(variables.enableLanceScalarIndexAutoSelection); - variables.enableLanceScalarIndexAutoSelection = enabled; - Assert.assertEquals(Boolean.toString(enabled), variables.getForwardVariables().get( - SessionVariable.ENABLE_LANCE_SCALAR_INDEX_AUTO_SELECTION)); - LanceScanNode node = newNode(variables); - setMetadata(node, metadata); - Expr broad = scalarPredicate(); - broad.setSelectivity(0.8); - Expr selective = new BinaryPredicate(BinaryPredicate.Operator.EQ, - new SlotRef(null, "other"), new IntLiteral(1)); - selective.setSelectivity(0.01); - node.getConjuncts().addAll(Arrays.asList(broad, selective)); - node.convertPredicate(); - Assert.assertTrue(node.getConjuncts().isEmpty()); - List splits = node.getSplits(1); - Assert.assertEquals(1, splits.size()); - TFileRangeDesc range = new TFileRangeDesc(); - node.setScanParams(range, splits.get(0)); - ByteBuffer uuid = range.getTableFormatParams().getLanceParams().getIndexSegmentUuids().get(0).duplicate(); - Assert.assertEquals(enabled ? otherIndex : keyIndex, new UUID(uuid.getLong(), uuid.getLong())); - } - } - @Test public void testSplitPlanningReportsSuccessAndFailureToQueryProfile() throws Exception { ConnectContext previous = ConnectContext.get(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java deleted file mode 100644 index e66ebc98eebf2c..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/LanceScalarIndexSelectivityTest.java +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.glue.translator; - -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.TupleDescriptor; -import org.apache.doris.datasource.lance.source.LanceScanNode; -import org.apache.doris.nereids.properties.DataTrait; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.EqualTo; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; -import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; -import org.apache.doris.nereids.types.BigIntType; -import org.apache.doris.planner.PlanNode; -import org.apache.doris.planner.PlanNodeId; -import org.apache.doris.planner.ScanContext; -import org.apache.doris.qe.SessionVariable; -import org.apache.doris.statistics.ColumnStatistic; -import org.apache.doris.statistics.ColumnStatisticBuilder; -import org.apache.doris.statistics.Statistics; - -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.lang.reflect.Method; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; - -public class LanceScalarIndexSelectivityTest { - @Test - public void testTranslationUsesIndependentColumnEstimatesAndHonorsSwitch() throws Exception { - for (boolean enabled : Arrays.asList(true, false)) { - SlotReference broad = new SlotReference("broad", BigIntType.INSTANCE); - SlotReference selective = new SlotReference("selective", BigIntType.INSTANCE); - Map columns = new HashMap<>(); - columns.put(broad, columnStats(10)); - columns.put(selective, columnStats(1000)); - Statistics input = new Statistics(1000, columns); - LogicalProperties properties = new LogicalProperties( - () -> Arrays.asList(broad, selective), () -> DataTrait.EMPTY_TRAIT); - PhysicalEmptyRelation child = new PhysicalEmptyRelation(new RelationId(0), - Arrays.asList(broad, selective), properties); - child.setStatistics(input); - PhysicalFilter filter = new PhysicalFilter<>(new HashSet<>(Arrays.asList( - new EqualTo(broad, new BigIntLiteral(1)), - new EqualTo(selective, new BigIntLiteral(1)))), properties, child); - PlanTranslatorContext context = new PlanTranslatorContext(); - TupleDescriptor tuple = context.generateTupleDesc(); - context.createSlotDesc(tuple, broad); - context.createSlotDesc(tuple, selective); - SessionVariable variables = new SessionVariable(); - variables.enableLanceScalarIndexAutoSelection = enabled; - LanceScanNode node = new LanceScanNode(new PlanNodeId(0), tuple, false, variables, ScanContext.EMPTY); - Method method = PhysicalPlanTranslator.class.getDeclaredMethod("addConjunctsToPlanNode", - PhysicalFilter.class, PlanNode.class, PlanTranslatorContext.class); - method.setAccessible(true); - method.invoke(new PhysicalPlanTranslator(context), filter, node, context); - Assertions.assertEquals(2, node.getConjuncts().size()); - for (Expr predicate : node.getConjuncts()) { - if (enabled) { - String column = ((SlotRef) predicate.getChild(0)).getDesc().getLabel(); - Assertions.assertEquals("broad".equals(column) ? 0.1 : 0.001, - predicate.getSelectivity(), 1e-9, column + ": " + predicate.toSql()); - } else { - Assertions.assertFalse(predicate.hasSelectivity()); - } - } - Assertions.assertEquals(1000, input.getRowCount()); - Assertions.assertEquals(10, input.findColumnStatistics(broad).ndv); - Assertions.assertEquals(1000, input.findColumnStatistics(selective).ndv); - } - } - - private static ColumnStatistic columnStats(double ndv) { - return new ColumnStatisticBuilder(1000).setNdv(ndv).setNumNulls(0) - .setMinValue(0).setMaxValue(1000).setAvgSizeByte(8).setDataSize(8000).build(); - } -} diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 198e328c79cdd2..89829ddaf3f5b8 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -718,10 +718,10 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# Foyer remains a local patch until its cache interface is accepted upstream. -# All search fixes are supplied by the immutable lance-c dependency revision. +# Local cache and runtime-selection extensions are applied to the pinned lance-c. +# Bind the source marker to both patches so cached builds cannot retain an older ABI. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then - foyer_patch_checksum="$(cksum < "${TP_PATCH_DIR}/lance-c-foyer.patch")" + foyer_patch_checksum="$(cksum < "${TP_PATCH_DIR}/lance-c-foyer.patch") $(cksum < "${TP_PATCH_DIR}/lance-c-runtime-index-selection.patch")" foyer_patch_marker="${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/${PATCHED_MARK}_foyer" # A new local patch must also replace previously patched cached sources. # Empty markers from older builds cannot identify the applied patch version. @@ -734,6 +734,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ ! -f "${PATCHED_MARK}_foyer" ]]; then patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ -p1 <"${TP_PATCH_DIR}/lance-c-foyer.patch" + patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ + -p1 <"${TP_PATCH_DIR}/lance-c-runtime-index-selection.patch" printf '%s\n' "${foyer_patch_checksum}" > "${PATCHED_MARK}_foyer" fi cd - diff --git a/thirdparty/patches/lance-c-runtime-index-selection.patch b/thirdparty/patches/lance-c-runtime-index-selection.patch new file mode 100644 index 00000000000000..6a9de65d364f27 --- /dev/null +++ b/thirdparty/patches/lance-c-runtime-index-selection.patch @@ -0,0 +1,723 @@ +diff --git a/include/lance/lance.h b/include/lance/lance.h +index b592396..052a0fc 100644 +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -2439,6 +2439,11 @@ int32_t lance_scanner_set_index_segments( + int32_t lance_scanner_set_scalar_index_segment( + LanceScanner* scanner, const uint8_t* segment_uuid); + ++/* Opt in to bounded runtime BTree page-cost selection for a scalar segment scan. ++ * Defaults to false. The explicit fragment domain and complete filter are preserved. ++ * Unsupported/unknown estimates retain the selected segment. Returns 0 or -1. */ ++int32_t lance_scanner_set_scalar_index_auto_selection(LanceScanner* scanner, bool enabled); ++ + /* ─── Full-text search (Phase 2) ─── */ + + /** +diff --git a/src/lib.rs b/src/lib.rs +index 4a96bb5..267e63b 100644 +--- a/src/lib.rs ++++ b/src/lib.rs +@@ -43,6 +43,7 @@ mod merge_insert; + mod multivector; + mod restore; + pub mod runtime; ++mod scalar_index_selection; + mod scalar_segment; + mod scanner; + mod session; +diff --git a/src/scalar_index_selection.rs b/src/scalar_index_selection.rs +new file mode 100644 +index 0000000..87270ff +--- /dev/null ++++ b/src/scalar_index_selection.rs +@@ -0,0 +1,408 @@ ++// SPDX-License-Identifier: Apache-2.0 ++// SPDX-FileCopyrightText: Copyright The Lance Authors ++ ++//! Bounded runtime BTree cost comparison. Estimates affect only acceleration; ++//! the caller always retains its explicit fragment domain and complete filter. ++ ++use std::{borrow::Cow, cmp::Ordering, ops::Bound}; ++ ++use arrow_array::{Array, RecordBatch, UInt32Array}; ++use datafusion::common::ScalarValue; ++use lance::{ ++ Dataset, ++ dataset::{index::LanceIndexStoreExt, scanner::ExecutionSummaryCounts}, ++}; ++use lance_core::{ ++ Error, ++ cache::CacheKey, ++ deepsize::{Context, DeepSizeOf}, ++}; ++use lance_index::scalar::{ ++ IndexStore, SargableQuery, ++ btree::{BTREE_LOOKUP_NAME, DEFAULT_BTREE_BATCH_SIZE}, ++ expression::ScalarIndexExpr, ++ lance_format::LanceIndexStore, ++}; ++use lance_table::format::IndexMetadata; ++use uuid::Uuid; ++ ++use crate::scalar_segment::scoped_expression; ++ ++const MAX_CANDIDATES: usize = 8; ++const MAX_LOOKUP_BYTES: u64 = 8 * 1024 * 1024; ++const MAX_LOOKUP_PAGES: usize = 65_536; ++const MAX_IN_VALUES: usize = 256; ++const MAX_ESTIMATE_COMPARISONS: usize = 1_000_000; ++ ++struct Lookup { ++ batch: RecordBatch, ++ rows_per_page: u64, ++} ++ ++impl DeepSizeOf for Lookup { ++ fn deep_size_of_children(&self, _context: &mut Context) -> usize { ++ self.batch.get_array_memory_size() ++ } ++} ++ ++struct LookupKey(String); ++impl CacheKey for LookupKey { ++ type ValueType = Lookup; ++ fn key(&self) -> Cow<'_, str> { ++ self.0.as_str().into() ++ } ++ fn type_name() -> &'static str { ++ "DorisBTreeCostLookup" ++ } ++} ++ ++fn is_btree_expression(expr: &ScalarIndexExpr) -> bool { ++ match expr { ++ ScalarIndexExpr::Query(search) => search.index_type.eq_ignore_ascii_case("btree"), ++ ScalarIndexExpr::And(a, b) | ScalarIndexExpr::Or(a, b) => { ++ is_btree_expression(a) && is_btree_expression(b) ++ } ++ ScalarIndexExpr::Not(_) => false, ++ } ++} ++ ++fn overlaps( ++ min: &ScalarValue, ++ max: &ScalarValue, ++ query: &SargableQuery, ++ nulls: u32, ++) -> Option { ++ let contains = |value: &ScalarValue| -> Option { ++ if value.is_null() { ++ return Some(nulls != 0); ++ } ++ if min.is_null() || max.is_null() { ++ return Some(false); ++ } ++ Some( ++ min.partial_cmp(value)? != Ordering::Greater ++ && max.partial_cmp(value)? != Ordering::Less, ++ ) ++ }; ++ match query { ++ SargableQuery::Equals(value) => contains(value), ++ SargableQuery::IsNull() => Some(nulls != 0), ++ SargableQuery::IsIn(values) if values.len() <= MAX_IN_VALUES => { ++ for value in values { ++ if contains(value)? { ++ return Some(true); ++ } ++ } ++ Some(false) ++ } ++ SargableQuery::Range(start, end) => { ++ if min.is_null() || max.is_null() { ++ return Some(false); ++ } ++ let above_lower = match start { ++ Bound::Included(v) if !v.is_null() => max.partial_cmp(v)? != Ordering::Less, ++ Bound::Excluded(v) if !v.is_null() => max.partial_cmp(v)? == Ordering::Greater, ++ Bound::Unbounded => true, ++ _ => return None, ++ }; ++ let below_upper = match end { ++ Bound::Included(v) if !v.is_null() => min.partial_cmp(v)? != Ordering::Greater, ++ Bound::Excluded(v) if !v.is_null() => min.partial_cmp(v)? == Ordering::Less, ++ Bound::Unbounded => true, ++ _ => return None, ++ }; ++ Some(above_lower && below_upper) ++ } ++ _ => None, ++ } ++} ++ ++fn matches_page( ++ expr: &ScalarIndexExpr, ++ min: &ScalarValue, ++ max: &ScalarValue, ++ nulls: u32, ++) -> Option { ++ match expr { ++ ScalarIndexExpr::Query(search) => overlaps( ++ min, ++ max, ++ search.query.as_any().downcast_ref::()?, ++ nulls, ++ ), ++ ScalarIndexExpr::And(a, b) => { ++ Some(matches_page(a, min, max, nulls)? && matches_page(b, min, max, nulls)?) ++ } ++ ScalarIndexExpr::Or(a, b) => { ++ Some(matches_page(a, min, max, nulls)? || matches_page(b, min, max, nulls)?) ++ } ++ // A page's min/max cannot determine which rows survive a complement. ++ ScalarIndexExpr::Not(_) => None, ++ } ++} ++ ++fn comparisons_per_page(expr: &ScalarIndexExpr) -> Option { ++ match expr { ++ ScalarIndexExpr::Query(search) => { ++ match search.query.as_any().downcast_ref::()? { ++ SargableQuery::IsIn(values) if values.len() <= MAX_IN_VALUES => { ++ Some(values.len().max(1)) ++ } ++ SargableQuery::Equals(_) | SargableQuery::Range(_, _) | SargableQuery::IsNull() => { ++ Some(1) ++ } ++ _ => None, ++ } ++ } ++ ScalarIndexExpr::And(a, b) | ScalarIndexExpr::Or(a, b) => { ++ comparisons_per_page(a)?.checked_add(comparisons_per_page(b)?) ++ } ++ ScalarIndexExpr::Not(_) => None, ++ } ++} ++ ++fn estimate_pages(batch: &RecordBatch, expr: &ScalarIndexExpr) -> Option { ++ if batch.num_columns() < 3 || batch.num_rows() > MAX_LOOKUP_PAGES { ++ return None; ++ } ++ // Node and byte limits alone still allow pages * IN-values * leaves work. ++ if batch.num_rows().checked_mul(comparisons_per_page(expr)?)? > MAX_ESTIMATE_COMPARISONS { ++ return None; ++ } ++ let nulls = batch.column(2).as_any().downcast_ref::()?; ++ let mut pages = 0; ++ for row in 0..batch.num_rows() { ++ let min = ScalarValue::try_from_array(batch.column(0), row).ok()?; ++ let max = ScalarValue::try_from_array(batch.column(1), row).ok()?; ++ pages += usize::from(matches_page(expr, &min, &max, nulls.value(row))?); ++ } ++ Some(pages) ++} ++ ++async fn estimate_cost( ++ dataset: &Dataset, ++ index: &IndexMetadata, ++ expr: &ScalarIndexExpr, ++ remaining_bytes: &mut u64, ++) -> Option<(u64, usize)> { ++ if !is_btree_expression(expr) { ++ return None; ++ } ++ // Inspect immutable manifest sizes before opening a file. Older/partition-only ++ // layouts without this root are deliberately left to the existing search path. ++ let size = index ++ .files ++ .as_ref()? ++ .iter() ++ .find(|f| f.path == BTREE_LOOKUP_NAME)? ++ .size_bytes; ++ if size > *remaining_bytes { ++ return None; ++ } ++ *remaining_bytes -= size; ++ let key = LookupKey(format!("{}/{}", dataset.uri(), index.uuid)); ++ let session = dataset.session(); ++ let lookup = session ++ .file_metadata_cache() ++ .get_or_insert_with_key(key, || async { ++ let store = LanceIndexStore::from_dataset_for_existing(dataset, index).await?; ++ let reader = store.open_index_file(BTREE_LOOKUP_NAME).await?; ++ if reader.num_rows() > MAX_LOOKUP_PAGES { ++ return Err(Error::not_supported( ++ "BTree cost lookup exceeds the page budget", ++ )); ++ } ++ let rows_per_page = reader ++ .schema() ++ .metadata ++ .get("batch_size") ++ .map(|s| s.parse::()) ++ .transpose() ++ .map_err(|_| Error::not_supported("invalid BTree page size"))? ++ .unwrap_or(DEFAULT_BTREE_BATCH_SIZE); ++ if rows_per_page == 0 { ++ return Err(Error::not_supported("zero BTree page size")); ++ } ++ let batch = reader.read_range(0..reader.num_rows(), None).await?; ++ Ok(Lookup { ++ batch, ++ rows_per_page, ++ }) ++ }) ++ .await ++ .ok()?; ++ let pages = estimate_pages(&lookup.batch, expr)?; ++ Some((lookup.rows_per_page.saturating_mul(pages as u64), pages)) ++} ++ ++/// None selects a scoped full scan. Unknown estimates preserve the grouping index. ++pub(crate) async fn choose( ++ dataset: &Dataset, ++ indices: &[IndexMetadata], ++ grouping: &IndexMetadata, ++ grouping_expr: ScalarIndexExpr, ++ index_query: &ScalarIndexExpr, ++ fragment_ids: &[u64], ++ physical_rows: u64, ++ stats: &mut ExecutionSummaryCounts, ++) -> Option<(Uuid, ScalarIndexExpr)> { ++ if !is_btree_expression(&grouping_expr) { ++ return Some((grouping.uuid, grouping_expr)); ++ } ++ let mut remaining_bytes = MAX_LOOKUP_BYTES; ++ stats ++ .all_counts ++ .insert("scalar_index_auto_candidates".into(), 1); ++ let Some((mut cost, mut pages)) = ++ estimate_cost(dataset, grouping, &grouping_expr, &mut remaining_bytes).await ++ else { ++ return Some((grouping.uuid, grouping_expr)); ++ }; ++ let mut selected = (grouping.uuid, grouping_expr); ++ let mut candidates = 1; ++ let mut alternatives: Vec<_> = indices.iter().filter(|i| i.uuid != grouping.uuid).collect(); ++ alternatives.sort_by(|a, b| (&a.name, a.uuid).cmp(&(&b.name, b.uuid))); ++ for candidate in alternatives { ++ if candidates >= MAX_CANDIDATES { ++ break; ++ } ++ let Some(field_id) = candidate.keyed_field() else { ++ continue; ++ }; ++ let Some(field) = dataset.schema().fields.iter().find(|f| f.id == field_id) else { ++ continue; ++ }; ++ let Some(coverage) = &candidate.fragment_bitmap else { ++ continue; ++ }; ++ if fragment_ids ++ .iter() ++ .any(|id| u32::try_from(*id).map_or(true, |id| !coverage.contains(id))) ++ { ++ continue; ++ } ++ let Ok(Some(expr)) = scoped_expression( ++ index_query, ++ &candidate.name, ++ &field.name, ++ false, ++ 0, ++ &mut 128, ++ ) else { ++ continue; ++ }; ++ if !is_btree_expression(&expr) { ++ continue; ++ } ++ candidates += 1; ++ if let Some((candidate_cost, candidate_pages)) = ++ estimate_cost(dataset, candidate, &expr, &mut remaining_bytes).await ++ { ++ if candidate_cost < cost { ++ cost = candidate_cost; ++ pages = candidate_pages; ++ selected = (candidate.uuid, expr); ++ } ++ } ++ } ++ stats ++ .all_counts ++ .insert("scalar_index_auto_candidates".into(), candidates); ++ stats ++ .all_counts ++ .insert("scalar_index_auto_estimated_pages".into(), pages); ++ stats.all_counts.insert( ++ "scalar_index_auto_estimated_rows".into(), ++ cost.min(usize::MAX as u64) as usize, ++ ); ++ if cost >= physical_rows { ++ stats ++ .all_counts ++ .insert("scalar_index_auto_full_scans".into(), 1); ++ return None; ++ } ++ stats.all_counts.insert( ++ "scalar_index_auto_changed".into(), ++ usize::from(selected.0 != grouping.uuid), ++ ); ++ Some(selected) ++} ++ ++#[cfg(test)] ++mod tests { ++ use super::*; ++ use arrow_array::{Int32Array, UInt32Array}; ++ use arrow_schema::{DataType, Field, Schema}; ++ use datafusion::common::ScalarValue; ++ use lance_index::scalar::{SargableQuery, expression::ScalarIndexSearch}; ++ use std::sync::Arc; ++ ++ fn lookup() -> RecordBatch { ++ RecordBatch::try_new( ++ Arc::new(Schema::new(vec![ ++ Field::new("min", DataType::Int32, true), ++ Field::new("max", DataType::Int32, true), ++ Field::new("null_count", DataType::UInt32, false), ++ ])), ++ vec![ ++ Arc::new(Int32Array::from(vec![None, Some(0), Some(10), Some(20)])), ++ Arc::new(Int32Array::from(vec![None, Some(9), Some(19), Some(29)])), ++ Arc::new(UInt32Array::from(vec![4, 0, 1, 0])), ++ ], ++ ) ++ .unwrap() ++ } ++ fn eq(value: i32) -> ScalarIndexExpr { ++ ScalarIndexExpr::Query(ScalarIndexSearch { ++ column: "key".into(), ++ index_name: "key_idx".into(), ++ index_type: "BTree".into(), ++ query: Arc::new(SargableQuery::Equals(ScalarValue::Int32(Some(value)))), ++ needs_recheck: false, ++ fragment_bitmap: None, ++ }) ++ } ++ #[test] ++ fn runtime_lookup_distinguishes_narrow_and_broad_predicates() { ++ assert_eq!(estimate_pages(&lookup(), &eq(15)), Some(1)); ++ assert_eq!(estimate_pages(&lookup(), &eq(99)), Some(0)); ++ let broad = ScalarIndexExpr::Or(Box::new(eq(5)), Box::new(eq(15))); ++ assert_eq!(estimate_pages(&lookup(), &broad), Some(2)); ++ } ++ #[test] ++ fn intersect_page_sets_without_counting_or_duplicates_twice() { ++ let repeated = ScalarIndexExpr::Or(Box::new(eq(15)), Box::new(eq(15))); ++ assert_eq!(estimate_pages(&lookup(), &repeated), Some(1)); ++ let disjoint = ScalarIndexExpr::And(Box::new(eq(5)), Box::new(eq(15))); ++ assert_eq!(estimate_pages(&lookup(), &disjoint), Some(0)); ++ assert_eq!( ++ estimate_pages(&lookup(), &ScalarIndexExpr::Not(Box::new(eq(15)))), ++ None ++ ); ++ } ++ #[test] ++ fn runtime_lookup_respects_nulls_and_exclusive_range_bounds() { ++ let mut expression = eq(0); ++ if let ScalarIndexExpr::Query(search) = &mut expression { ++ search.query = Arc::new(SargableQuery::IsNull()); ++ } ++ assert_eq!(estimate_pages(&lookup(), &expression), Some(2)); ++ if let ScalarIndexExpr::Query(search) = &mut expression { ++ search.query = Arc::new(SargableQuery::Range( ++ std::ops::Bound::Excluded(ScalarValue::Int32(Some(9))), ++ std::ops::Bound::Excluded(ScalarValue::Int32(Some(20))), ++ )); ++ } ++ assert_eq!(estimate_pages(&lookup(), &expression), Some(1)); ++ } ++ #[test] ++ fn expensive_in_cost_probes_fall_back_without_searching() { ++ let mut expression = eq(0); ++ if let ScalarIndexExpr::Query(search) = &mut expression { ++ search.query = Arc::new(SargableQuery::IsIn( ++ (0..257).map(|v| ScalarValue::Int32(Some(v))).collect(), ++ )); ++ } ++ assert_eq!(estimate_pages(&lookup(), &expression), None); ++ } ++} +diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs +index 6515fbf..6354006 100644 +--- a/src/scalar_segment.rs ++++ b/src/scalar_segment.rs +@@ -26,6 +26,7 @@ use uuid::Uuid; + pub(crate) struct PreparedScalarSegment { + pub dataset: Arc, + pub segment_uuid: Uuid, ++ pub auto_selection: bool, + pub fragment_ids: Vec, + pub use_scalar_index: bool, + pub callback: Option, +@@ -39,7 +40,7 @@ fn invalid(message: impl Into) -> Error { + const MAX_EXPRESSION_NODES: usize = 128; + const MAX_EXPRESSION_DEPTH: usize = 32; + +-fn scoped_expression( ++pub(crate) fn scoped_expression( + expr: &ScalarIndexExpr, + index_name: &str, + column: &str, +@@ -304,9 +305,45 @@ impl PreparedScalarSegment { + Ok(None) => return Ok(Some("no_driver")), + Err(reason) => return Ok(Some(reason)), + }; ++ let (segment_uuid, expr) = if self.auto_selection { ++ let physical_rows = fragments ++ .iter() ++ .filter(|f| scope.contains(&(f.id() as u64))) ++ .fold(0_u64, |sum, f| { ++ sum.saturating_add(f.metadata().physical_rows.unwrap_or(0) as u64) ++ }); ++ let Some(selected) = crate::scalar_index_selection::choose( ++ &self.dataset, ++ &indices, ++ index_meta, ++ expr, ++ filter_plan.index_query.as_ref().unwrap(), ++ &self.fragment_ids, ++ physical_rows, ++ stats, ++ ) ++ .await ++ else { ++ return Ok(Some("runtime_cost")); ++ }; ++ selected ++ } else { ++ (self.segment_uuid, expr) ++ }; ++ let index_meta = indices ++ .iter() ++ .find(|i| i.uuid == segment_uuid) ++ .ok_or_else(|| invalid("runtime scalar index is absent from the dataset snapshot"))?; ++ let field = self ++ .dataset ++ .schema() ++ .field_by_id(index_meta.keyed_field().unwrap()) ++ .ok_or_else(|| { ++ invalid("runtime scalar index field is absent from the dataset schema") ++ })?; + let index = self + .dataset +- .open_scalar_index(&field.name, &self.segment_uuid, metrics) ++ .open_scalar_index(&field.name, &segment_uuid, metrics) + .await?; + // These implementations can return exact candidates. Keep the runtime + // Exact check below: a type alone is not a guarantee for every query. +diff --git a/src/scanner.rs b/src/scanner.rs +index e5ff5ca..f2f1a00 100644 +--- a/src/scanner.rs ++++ b/src/scanner.rs +@@ -117,6 +117,7 @@ pub struct LanceScanner { + row_addr_blocklist: Option, + index_segments: Option>, + scalar_index_segment: Option, ++ scalar_index_auto_selection: bool, + nearest: Option, + nprobes: NprobesRange, + approx_mode: Option, +@@ -317,6 +318,7 @@ impl LanceScanner { + row_addr_blocklist: None, + index_segments: None, + scalar_index_segment: None, ++ scalar_index_auto_selection: false, + nearest: None, + nprobes: NprobesRange::default(), + approx_mode: None, +@@ -637,6 +639,7 @@ impl LanceScanner { + Some(PreparedScalarSegment { + dataset: Arc::clone(&self.dataset), + segment_uuid, ++ auto_selection: self.scalar_index_auto_selection, + fragment_ids: fragment_ids.clone(), + use_scalar_index: self.use_scalar_index.unwrap_or(true), + callback: self.scan_statistics_callback.clone(), +@@ -1079,6 +1082,22 @@ macro_rules! scanner_ffi_try { + }}; + } + ++/// Enable bounded runtime BTree selection for an explicitly scoped scalar segment scan. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_scanner_set_scalar_index_auto_selection( ++ scanner: *mut LanceScanner, ++ enabled: bool, ++) -> i32 { ++ scanner_poison_check!(scanner, -1); ++ scanner_ffi_try!(scanner, { ++ let scanner = unsafe { scanner.as_mut() } ++ .ok_or_else(|| lance_core::Error::invalid_input_source("scanner is NULL".into()))?; ++ scanner.ensure_scan_not_started("scalar_index_auto_selection")?; ++ scanner.scalar_index_auto_selection = enabled; ++ Ok(0) ++ }) ++} ++ + /// Select one physical scalar index segment. NULL clears the selection. + /// Requires explicit fragment_ids and an ordinary live-row scan. See the C header. + /// include_deleted_rows=true is rejected when preparing the scan, even if +diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs +index d9af02a..0042745 100644 +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -15704,6 +15704,18 @@ fn scalar_segment_ids_with_filter( + filter: ScalarSegmentFilter<'_>, + limit: Option, + offset: i64, ++) -> (Vec, CapturedScanStatistics) { ++ scalar_segment_ids_with_auto(uri, uuid, fragments, filter, limit, offset, false) ++} ++ ++fn scalar_segment_ids_with_auto( ++ uri: &str, ++ uuid: &[u8; 16], ++ fragments: &[u64], ++ filter: ScalarSegmentFilter<'_>, ++ limit: Option, ++ offset: i64, ++ auto_selection: bool, + ) -> (Vec, CapturedScanStatistics) { + let uri = c_str(uri); + let sql = match filter { +@@ -15737,6 +15749,10 @@ fn scalar_segment_ids_with_filter( + lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), + 0 + ); ++ assert_eq!( ++ lance_scanner_set_scalar_index_auto_selection(scanner, auto_selection), ++ 0 ++ ); + if let Some(limit) = limit { + assert_eq!(lance_scanner_set_limit(scanner, limit), 0); + } +@@ -15788,6 +15804,127 @@ fn scalar_segment_ids_with_filter( + (ids, captured) + } + ++#[test] ++fn test_runtime_scalar_index_selection_uses_page_cost_and_honors_switch() { ++ use lance::dataset::WriteParams; ++ use lance::index::DatasetIndexExt; ++ use lance_index::{IndexType, scalar::ScalarIndexParams}; ++ let tmp = tempfile::tempdir().unwrap(); ++ let uri = tmp ++ .path() ++ .join("runtime_selection") ++ .to_str() ++ .unwrap() ++ .to_owned(); ++ let (grouping, fragments) = lance_c::runtime::block_on(async { ++ let schema = Arc::new(Schema::new(vec![ ++ Field::new("id", DataType::Int32, false), ++ Field::new("key", DataType::Int32, false), ++ ])); ++ let batch = RecordBatch::try_new( ++ schema.clone(), ++ vec![ ++ Arc::new(Int32Array::from_iter_values(0..16384)), ++ Arc::new(Int32Array::from_iter_values(0..16384)), ++ ], ++ ) ++ .unwrap(); ++ let mut ds = Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), ++ &uri, ++ Some(WriteParams { ++ max_rows_per_file: 4096, ++ ..Default::default() ++ }), ++ ) ++ .await ++ .unwrap(); ++ ds.create_index( ++ &["key"], ++ IndexType::BTree, ++ Some("key_idx".into()), ++ &ScalarIndexParams::default(), ++ false, ++ ) ++ .await ++ .unwrap(); ++ ds.create_index( ++ &["id"], ++ IndexType::BTree, ++ Some("id_idx".into()), ++ &ScalarIndexParams::default(), ++ false, ++ ) ++ .await ++ .unwrap(); ++ let indices = ds.load_indices().await.unwrap(); ++ let uuid = *indices ++ .iter() ++ .find(|i| i.name == "key_idx") ++ .unwrap() ++ .uuid ++ .as_bytes(); ++ let fragments: Vec<_> = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); ++ (uuid, fragments) ++ }); ++ let metric = |stats: &CapturedScanStatistics, key: &str| { ++ stats ++ .metrics ++ .iter() ++ .filter(|(name, _, _)| name == key) ++ .map(|(_, _, n)| n) ++ .sum::() ++ }; ++ for enabled in [false, true] { ++ let (ids, stats) = scalar_segment_ids_with_auto( ++ &uri, ++ &grouping, ++ &fragments, ++ ScalarSegmentFilter::Sql("key >= 0 AND id = 123"), ++ None, ++ 0, ++ enabled, ++ ); ++ assert_eq!(ids, vec![123]); ++ assert_eq!( ++ metric(&stats, "scalar_index_auto_changed"), ++ u64::from(enabled) ++ ); ++ assert_eq!( ++ metric(&stats, "scalar_index_auto_candidates"), ++ if enabled { 2 } else { 0 } ++ ); ++ if enabled { ++ assert_eq!(metric(&stats, "scalar_index_auto_estimated_pages"), 1); ++ assert_eq!(metric(&stats, "scalar_index_auto_estimated_rows"), 4096); ++ assert_eq!(metric(&stats, "scalar_segments_searched"), 1); ++ } ++ } ++ let (ids, stats) = scalar_segment_ids_with_auto( ++ &uri, ++ &grouping, ++ &fragments, ++ ScalarSegmentFilter::Sql("key >= 0"), ++ None, ++ 0, ++ true, ++ ); ++ assert_eq!(ids.len(), 16384); ++ assert_eq!(metric(&stats, "scalar_index_auto_full_scans"), 1); ++ assert_eq!(metric(&stats, "scalar_segments_searched"), 0); ++ // A switch cannot broaden an explicitly assigned fragment domain. ++ let (ids, _) = scalar_segment_ids_with_auto( ++ &uri, ++ &grouping, ++ &[fragments[1]], ++ ScalarSegmentFilter::Sql("key >= 0 AND id >= 0"), ++ Some(2), ++ 1, ++ true, ++ ); ++ assert_eq!(ids, vec![4097, 4098]); ++} ++ + #[test] + fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { + for kind in [ From f1d1441169ca046338e4cf8edeca406a493043f3 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 26 Oct 2026 21:59:43 +0800 Subject: [PATCH 3/5] [chore](build) Use merged upstream lance-c runtime selection ### What problem does this PR solve? Lance profiles do not expose Session cache-miss observation windows, scalar-index task grouping cannot compare runtime costs, and Foyer memory capacity is tied to the data-cache read block size. This PR targets **branch-4.2** and adds: - `LanceSessionIndexCacheMisses` and `LanceSessionMetadataCacheMisses`, with availability and scope information. They report BE-wide Session deltas observed during a reader's dataset-open windows, including opening. Concurrent readers can overlap; these are not query-exclusive counts and must not be summed as an exact query total. - The startup BE option `enable_lance_scalar_index_auto_selection` (default `true`). FE keeps coverage-based task grouping and does not estimate selectivity or add a session variable. In BE, lance-c compares current BTree page min/max and null-count metadata for segments covering the entire task. Estimated work is matching pages times rows per page; a broad estimate can select a scoped full scan. Serialized lookup bytes, decoded buffers and probe work are bounded. Unknown estimates retain the existing segment path. The current native API cannot bound variable-width expansion, so string/binary costs remain unknown: retain such a grouping index or skip such an alternative before opening its lookup. Bitmap/LabelList remain supported on their existing path and are not cost-ranked; vector/FTS semantics are unchanged. - `lance_data_cache_memory_capacity_bytes`. Default `0` preserves the previous one-configured-block memory tier, including nondefault block sizes. Explicit capacities configure Foyer independently of block and disk capacities; negative values are rejected before unsigned conversion. The native selector and decoded-memory fix are now merged upstream in [lance-format/lance-c#98](https://github.com/lance-format/lance-c/pull/98). This PR pins upstream main commit `a51ab47a9c814a746cb1714c99fad93dd3a188b8`, refreshes the source archive checksum, and removes the local runtime-selection patch together with its application/checksum handling. The pre-existing Foyer patch is unchanged and applies with zero fuzz. Runtime decisions are exposed through Profile counters. Usage, cost-model limits and cache-counter scope are documented in `docs/lance-cache-and-index-selection.md`. ### Release note Add Lance Session cache-miss diagnostics, bounded BE runtime BTree selection controlled by a BE configuration, and independently configurable Foyer memory capacity. ### Validation - On the newly downloaded upstream archive plus the unchanged Foyer patch, `cargo test --offline --locked` passed all 515 tests, including 82 unit tests and 385 C API tests. The normal command ignores three C/C++ executable compilation tests. Coverage includes the compressed long-string allocation regression, decoded-budget limits, competing indices, scan fallback, fragment scope, NULL/deletion/stable-ID behavior and LIMIT/OFFSET. - C and C++ API sources passed syntax-only compilation against the newly patched headers. - Verified archive MD5, clean zero-fuzz patch application, cached no-op, stale-checksum source replacement, and equality of the resulting tested source. Selector/scanner/segment sources match merged upstream exactly. - All nine affected C++ files passed the repository formatting runner with clang-format 16.0.6. Shell syntax and staged whitespace checks passed. - Existing FE validation passed 74 focused tests and FE Checkstyle; this dependency synchronization changes no FE code. - Full Doris BE build and unit-suite validation remain for CI and have not been verified locally. ### Check List (For Author) - Test - [x] Unit Test - [x] Manual test: native source/patch lifecycle and header compatibility checks above. - [ ] Regression test - Behavior changed: - [x] Yes. BE runtime BTree selection is enabled by default and can be disabled. Foyer's default memory behavior is preserved. - Does this need documentation? - [x] Yes. Usage and native memory limits are documented. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- docs/lance-cache-and-index-selection.md | 16 +- thirdparty/download-thirdparty.sh | 8 +- .../lance-c-runtime-index-selection.patch | 723 ------------------ thirdparty/vars.sh | 10 +- 4 files changed, 22 insertions(+), 735 deletions(-) delete mode 100644 thirdparty/patches/lance-c-runtime-index-selection.patch diff --git a/docs/lance-cache-and-index-selection.md b/docs/lance-cache-and-index-selection.md index 16891fbd4171e3..111d0a0612a62e 100644 --- a/docs/lance-cache-and-index-selection.md +++ b/docs/lance-cache-and-index-selection.md @@ -47,8 +47,16 @@ are not modeled. Only a strictly cheaper estimate replaces the grouping index; ties retain it. If the best estimate is at least the task's physical row count, BE scans that domain with the complete filter instead of searching an index. -Each selection examines at most eight segments and an aggregate 8 MiB of lookup -files; a lookup has at most 65,536 pages, an IN list at most 256 values, and +Each selection examines at most eight segments, an aggregate 8 MiB of serialized +lookup files, and a separate aggregate 8 MiB allowance for decoded lookup buffers. +The decoded allowance is checked from schema and row count before materialization +and charged for both cache hits and new loads. With the current native reader API, +only fixed-width scalar lookup columns have a known decoded-size bound. String +and variable-width binary costs remain unknown: retain such a grouping index or +skip such an alternative without opening its lookup for estimation. These indices +remain usable on the existing segment path. + +A lookup has at most 65,536 pages, an IN list at most 256 values, and one estimate at most 1,000,000 page/predicate comparisons. Lookup batches share the bounded Session metadata cache. Probing does not search index leaf pages or materialize candidate row IDs. Unknown file sizes, unsupported @@ -67,6 +75,10 @@ these probes. The `lance_fragments_per_split` debug override still bypasses index-segment grouping. Vector/FTS index selection and external-search TVF filter semantics are unchanged. +The native selection implementation and decoded-memory guards are supplied by +upstream lance-c, pinned to an immutable commit in `thirdparty/vars.sh`. Doris +does not carry a local runtime-selection patch. + ## Foyer memory capacity `lance_data_cache_memory_capacity_bytes` is a startup BE configuration, independent diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 89829ddaf3f5b8..175c9b0df05220 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -718,10 +718,10 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# Local cache and runtime-selection extensions are applied to the pinned lance-c. -# Bind the source marker to both patches so cached builds cannot retain an older ABI. +# Foyer remains a local patch until its cache interface is accepted upstream. +# Runtime selection and its memory guards come from the pinned upstream revision. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then - foyer_patch_checksum="$(cksum < "${TP_PATCH_DIR}/lance-c-foyer.patch") $(cksum < "${TP_PATCH_DIR}/lance-c-runtime-index-selection.patch")" + foyer_patch_checksum="$(cksum < "${TP_PATCH_DIR}/lance-c-foyer.patch")" foyer_patch_marker="${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/${PATCHED_MARK}_foyer" # A new local patch must also replace previously patched cached sources. # Empty markers from older builds cannot identify the applied patch version. @@ -734,8 +734,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ ! -f "${PATCHED_MARK}_foyer" ]]; then patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ -p1 <"${TP_PATCH_DIR}/lance-c-foyer.patch" - patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ - -p1 <"${TP_PATCH_DIR}/lance-c-runtime-index-selection.patch" printf '%s\n' "${foyer_patch_checksum}" > "${PATCHED_MARK}_foyer" fi cd - diff --git a/thirdparty/patches/lance-c-runtime-index-selection.patch b/thirdparty/patches/lance-c-runtime-index-selection.patch deleted file mode 100644 index 6a9de65d364f27..00000000000000 --- a/thirdparty/patches/lance-c-runtime-index-selection.patch +++ /dev/null @@ -1,723 +0,0 @@ -diff --git a/include/lance/lance.h b/include/lance/lance.h -index b592396..052a0fc 100644 ---- a/include/lance/lance.h -+++ b/include/lance/lance.h -@@ -2439,6 +2439,11 @@ int32_t lance_scanner_set_index_segments( - int32_t lance_scanner_set_scalar_index_segment( - LanceScanner* scanner, const uint8_t* segment_uuid); - -+/* Opt in to bounded runtime BTree page-cost selection for a scalar segment scan. -+ * Defaults to false. The explicit fragment domain and complete filter are preserved. -+ * Unsupported/unknown estimates retain the selected segment. Returns 0 or -1. */ -+int32_t lance_scanner_set_scalar_index_auto_selection(LanceScanner* scanner, bool enabled); -+ - /* ─── Full-text search (Phase 2) ─── */ - - /** -diff --git a/src/lib.rs b/src/lib.rs -index 4a96bb5..267e63b 100644 ---- a/src/lib.rs -+++ b/src/lib.rs -@@ -43,6 +43,7 @@ mod merge_insert; - mod multivector; - mod restore; - pub mod runtime; -+mod scalar_index_selection; - mod scalar_segment; - mod scanner; - mod session; -diff --git a/src/scalar_index_selection.rs b/src/scalar_index_selection.rs -new file mode 100644 -index 0000000..87270ff ---- /dev/null -+++ b/src/scalar_index_selection.rs -@@ -0,0 +1,408 @@ -+// SPDX-License-Identifier: Apache-2.0 -+// SPDX-FileCopyrightText: Copyright The Lance Authors -+ -+//! Bounded runtime BTree cost comparison. Estimates affect only acceleration; -+//! the caller always retains its explicit fragment domain and complete filter. -+ -+use std::{borrow::Cow, cmp::Ordering, ops::Bound}; -+ -+use arrow_array::{Array, RecordBatch, UInt32Array}; -+use datafusion::common::ScalarValue; -+use lance::{ -+ Dataset, -+ dataset::{index::LanceIndexStoreExt, scanner::ExecutionSummaryCounts}, -+}; -+use lance_core::{ -+ Error, -+ cache::CacheKey, -+ deepsize::{Context, DeepSizeOf}, -+}; -+use lance_index::scalar::{ -+ IndexStore, SargableQuery, -+ btree::{BTREE_LOOKUP_NAME, DEFAULT_BTREE_BATCH_SIZE}, -+ expression::ScalarIndexExpr, -+ lance_format::LanceIndexStore, -+}; -+use lance_table::format::IndexMetadata; -+use uuid::Uuid; -+ -+use crate::scalar_segment::scoped_expression; -+ -+const MAX_CANDIDATES: usize = 8; -+const MAX_LOOKUP_BYTES: u64 = 8 * 1024 * 1024; -+const MAX_LOOKUP_PAGES: usize = 65_536; -+const MAX_IN_VALUES: usize = 256; -+const MAX_ESTIMATE_COMPARISONS: usize = 1_000_000; -+ -+struct Lookup { -+ batch: RecordBatch, -+ rows_per_page: u64, -+} -+ -+impl DeepSizeOf for Lookup { -+ fn deep_size_of_children(&self, _context: &mut Context) -> usize { -+ self.batch.get_array_memory_size() -+ } -+} -+ -+struct LookupKey(String); -+impl CacheKey for LookupKey { -+ type ValueType = Lookup; -+ fn key(&self) -> Cow<'_, str> { -+ self.0.as_str().into() -+ } -+ fn type_name() -> &'static str { -+ "DorisBTreeCostLookup" -+ } -+} -+ -+fn is_btree_expression(expr: &ScalarIndexExpr) -> bool { -+ match expr { -+ ScalarIndexExpr::Query(search) => search.index_type.eq_ignore_ascii_case("btree"), -+ ScalarIndexExpr::And(a, b) | ScalarIndexExpr::Or(a, b) => { -+ is_btree_expression(a) && is_btree_expression(b) -+ } -+ ScalarIndexExpr::Not(_) => false, -+ } -+} -+ -+fn overlaps( -+ min: &ScalarValue, -+ max: &ScalarValue, -+ query: &SargableQuery, -+ nulls: u32, -+) -> Option { -+ let contains = |value: &ScalarValue| -> Option { -+ if value.is_null() { -+ return Some(nulls != 0); -+ } -+ if min.is_null() || max.is_null() { -+ return Some(false); -+ } -+ Some( -+ min.partial_cmp(value)? != Ordering::Greater -+ && max.partial_cmp(value)? != Ordering::Less, -+ ) -+ }; -+ match query { -+ SargableQuery::Equals(value) => contains(value), -+ SargableQuery::IsNull() => Some(nulls != 0), -+ SargableQuery::IsIn(values) if values.len() <= MAX_IN_VALUES => { -+ for value in values { -+ if contains(value)? { -+ return Some(true); -+ } -+ } -+ Some(false) -+ } -+ SargableQuery::Range(start, end) => { -+ if min.is_null() || max.is_null() { -+ return Some(false); -+ } -+ let above_lower = match start { -+ Bound::Included(v) if !v.is_null() => max.partial_cmp(v)? != Ordering::Less, -+ Bound::Excluded(v) if !v.is_null() => max.partial_cmp(v)? == Ordering::Greater, -+ Bound::Unbounded => true, -+ _ => return None, -+ }; -+ let below_upper = match end { -+ Bound::Included(v) if !v.is_null() => min.partial_cmp(v)? != Ordering::Greater, -+ Bound::Excluded(v) if !v.is_null() => min.partial_cmp(v)? == Ordering::Less, -+ Bound::Unbounded => true, -+ _ => return None, -+ }; -+ Some(above_lower && below_upper) -+ } -+ _ => None, -+ } -+} -+ -+fn matches_page( -+ expr: &ScalarIndexExpr, -+ min: &ScalarValue, -+ max: &ScalarValue, -+ nulls: u32, -+) -> Option { -+ match expr { -+ ScalarIndexExpr::Query(search) => overlaps( -+ min, -+ max, -+ search.query.as_any().downcast_ref::()?, -+ nulls, -+ ), -+ ScalarIndexExpr::And(a, b) => { -+ Some(matches_page(a, min, max, nulls)? && matches_page(b, min, max, nulls)?) -+ } -+ ScalarIndexExpr::Or(a, b) => { -+ Some(matches_page(a, min, max, nulls)? || matches_page(b, min, max, nulls)?) -+ } -+ // A page's min/max cannot determine which rows survive a complement. -+ ScalarIndexExpr::Not(_) => None, -+ } -+} -+ -+fn comparisons_per_page(expr: &ScalarIndexExpr) -> Option { -+ match expr { -+ ScalarIndexExpr::Query(search) => { -+ match search.query.as_any().downcast_ref::()? { -+ SargableQuery::IsIn(values) if values.len() <= MAX_IN_VALUES => { -+ Some(values.len().max(1)) -+ } -+ SargableQuery::Equals(_) | SargableQuery::Range(_, _) | SargableQuery::IsNull() => { -+ Some(1) -+ } -+ _ => None, -+ } -+ } -+ ScalarIndexExpr::And(a, b) | ScalarIndexExpr::Or(a, b) => { -+ comparisons_per_page(a)?.checked_add(comparisons_per_page(b)?) -+ } -+ ScalarIndexExpr::Not(_) => None, -+ } -+} -+ -+fn estimate_pages(batch: &RecordBatch, expr: &ScalarIndexExpr) -> Option { -+ if batch.num_columns() < 3 || batch.num_rows() > MAX_LOOKUP_PAGES { -+ return None; -+ } -+ // Node and byte limits alone still allow pages * IN-values * leaves work. -+ if batch.num_rows().checked_mul(comparisons_per_page(expr)?)? > MAX_ESTIMATE_COMPARISONS { -+ return None; -+ } -+ let nulls = batch.column(2).as_any().downcast_ref::()?; -+ let mut pages = 0; -+ for row in 0..batch.num_rows() { -+ let min = ScalarValue::try_from_array(batch.column(0), row).ok()?; -+ let max = ScalarValue::try_from_array(batch.column(1), row).ok()?; -+ pages += usize::from(matches_page(expr, &min, &max, nulls.value(row))?); -+ } -+ Some(pages) -+} -+ -+async fn estimate_cost( -+ dataset: &Dataset, -+ index: &IndexMetadata, -+ expr: &ScalarIndexExpr, -+ remaining_bytes: &mut u64, -+) -> Option<(u64, usize)> { -+ if !is_btree_expression(expr) { -+ return None; -+ } -+ // Inspect immutable manifest sizes before opening a file. Older/partition-only -+ // layouts without this root are deliberately left to the existing search path. -+ let size = index -+ .files -+ .as_ref()? -+ .iter() -+ .find(|f| f.path == BTREE_LOOKUP_NAME)? -+ .size_bytes; -+ if size > *remaining_bytes { -+ return None; -+ } -+ *remaining_bytes -= size; -+ let key = LookupKey(format!("{}/{}", dataset.uri(), index.uuid)); -+ let session = dataset.session(); -+ let lookup = session -+ .file_metadata_cache() -+ .get_or_insert_with_key(key, || async { -+ let store = LanceIndexStore::from_dataset_for_existing(dataset, index).await?; -+ let reader = store.open_index_file(BTREE_LOOKUP_NAME).await?; -+ if reader.num_rows() > MAX_LOOKUP_PAGES { -+ return Err(Error::not_supported( -+ "BTree cost lookup exceeds the page budget", -+ )); -+ } -+ let rows_per_page = reader -+ .schema() -+ .metadata -+ .get("batch_size") -+ .map(|s| s.parse::()) -+ .transpose() -+ .map_err(|_| Error::not_supported("invalid BTree page size"))? -+ .unwrap_or(DEFAULT_BTREE_BATCH_SIZE); -+ if rows_per_page == 0 { -+ return Err(Error::not_supported("zero BTree page size")); -+ } -+ let batch = reader.read_range(0..reader.num_rows(), None).await?; -+ Ok(Lookup { -+ batch, -+ rows_per_page, -+ }) -+ }) -+ .await -+ .ok()?; -+ let pages = estimate_pages(&lookup.batch, expr)?; -+ Some((lookup.rows_per_page.saturating_mul(pages as u64), pages)) -+} -+ -+/// None selects a scoped full scan. Unknown estimates preserve the grouping index. -+pub(crate) async fn choose( -+ dataset: &Dataset, -+ indices: &[IndexMetadata], -+ grouping: &IndexMetadata, -+ grouping_expr: ScalarIndexExpr, -+ index_query: &ScalarIndexExpr, -+ fragment_ids: &[u64], -+ physical_rows: u64, -+ stats: &mut ExecutionSummaryCounts, -+) -> Option<(Uuid, ScalarIndexExpr)> { -+ if !is_btree_expression(&grouping_expr) { -+ return Some((grouping.uuid, grouping_expr)); -+ } -+ let mut remaining_bytes = MAX_LOOKUP_BYTES; -+ stats -+ .all_counts -+ .insert("scalar_index_auto_candidates".into(), 1); -+ let Some((mut cost, mut pages)) = -+ estimate_cost(dataset, grouping, &grouping_expr, &mut remaining_bytes).await -+ else { -+ return Some((grouping.uuid, grouping_expr)); -+ }; -+ let mut selected = (grouping.uuid, grouping_expr); -+ let mut candidates = 1; -+ let mut alternatives: Vec<_> = indices.iter().filter(|i| i.uuid != grouping.uuid).collect(); -+ alternatives.sort_by(|a, b| (&a.name, a.uuid).cmp(&(&b.name, b.uuid))); -+ for candidate in alternatives { -+ if candidates >= MAX_CANDIDATES { -+ break; -+ } -+ let Some(field_id) = candidate.keyed_field() else { -+ continue; -+ }; -+ let Some(field) = dataset.schema().fields.iter().find(|f| f.id == field_id) else { -+ continue; -+ }; -+ let Some(coverage) = &candidate.fragment_bitmap else { -+ continue; -+ }; -+ if fragment_ids -+ .iter() -+ .any(|id| u32::try_from(*id).map_or(true, |id| !coverage.contains(id))) -+ { -+ continue; -+ } -+ let Ok(Some(expr)) = scoped_expression( -+ index_query, -+ &candidate.name, -+ &field.name, -+ false, -+ 0, -+ &mut 128, -+ ) else { -+ continue; -+ }; -+ if !is_btree_expression(&expr) { -+ continue; -+ } -+ candidates += 1; -+ if let Some((candidate_cost, candidate_pages)) = -+ estimate_cost(dataset, candidate, &expr, &mut remaining_bytes).await -+ { -+ if candidate_cost < cost { -+ cost = candidate_cost; -+ pages = candidate_pages; -+ selected = (candidate.uuid, expr); -+ } -+ } -+ } -+ stats -+ .all_counts -+ .insert("scalar_index_auto_candidates".into(), candidates); -+ stats -+ .all_counts -+ .insert("scalar_index_auto_estimated_pages".into(), pages); -+ stats.all_counts.insert( -+ "scalar_index_auto_estimated_rows".into(), -+ cost.min(usize::MAX as u64) as usize, -+ ); -+ if cost >= physical_rows { -+ stats -+ .all_counts -+ .insert("scalar_index_auto_full_scans".into(), 1); -+ return None; -+ } -+ stats.all_counts.insert( -+ "scalar_index_auto_changed".into(), -+ usize::from(selected.0 != grouping.uuid), -+ ); -+ Some(selected) -+} -+ -+#[cfg(test)] -+mod tests { -+ use super::*; -+ use arrow_array::{Int32Array, UInt32Array}; -+ use arrow_schema::{DataType, Field, Schema}; -+ use datafusion::common::ScalarValue; -+ use lance_index::scalar::{SargableQuery, expression::ScalarIndexSearch}; -+ use std::sync::Arc; -+ -+ fn lookup() -> RecordBatch { -+ RecordBatch::try_new( -+ Arc::new(Schema::new(vec![ -+ Field::new("min", DataType::Int32, true), -+ Field::new("max", DataType::Int32, true), -+ Field::new("null_count", DataType::UInt32, false), -+ ])), -+ vec![ -+ Arc::new(Int32Array::from(vec![None, Some(0), Some(10), Some(20)])), -+ Arc::new(Int32Array::from(vec![None, Some(9), Some(19), Some(29)])), -+ Arc::new(UInt32Array::from(vec![4, 0, 1, 0])), -+ ], -+ ) -+ .unwrap() -+ } -+ fn eq(value: i32) -> ScalarIndexExpr { -+ ScalarIndexExpr::Query(ScalarIndexSearch { -+ column: "key".into(), -+ index_name: "key_idx".into(), -+ index_type: "BTree".into(), -+ query: Arc::new(SargableQuery::Equals(ScalarValue::Int32(Some(value)))), -+ needs_recheck: false, -+ fragment_bitmap: None, -+ }) -+ } -+ #[test] -+ fn runtime_lookup_distinguishes_narrow_and_broad_predicates() { -+ assert_eq!(estimate_pages(&lookup(), &eq(15)), Some(1)); -+ assert_eq!(estimate_pages(&lookup(), &eq(99)), Some(0)); -+ let broad = ScalarIndexExpr::Or(Box::new(eq(5)), Box::new(eq(15))); -+ assert_eq!(estimate_pages(&lookup(), &broad), Some(2)); -+ } -+ #[test] -+ fn intersect_page_sets_without_counting_or_duplicates_twice() { -+ let repeated = ScalarIndexExpr::Or(Box::new(eq(15)), Box::new(eq(15))); -+ assert_eq!(estimate_pages(&lookup(), &repeated), Some(1)); -+ let disjoint = ScalarIndexExpr::And(Box::new(eq(5)), Box::new(eq(15))); -+ assert_eq!(estimate_pages(&lookup(), &disjoint), Some(0)); -+ assert_eq!( -+ estimate_pages(&lookup(), &ScalarIndexExpr::Not(Box::new(eq(15)))), -+ None -+ ); -+ } -+ #[test] -+ fn runtime_lookup_respects_nulls_and_exclusive_range_bounds() { -+ let mut expression = eq(0); -+ if let ScalarIndexExpr::Query(search) = &mut expression { -+ search.query = Arc::new(SargableQuery::IsNull()); -+ } -+ assert_eq!(estimate_pages(&lookup(), &expression), Some(2)); -+ if let ScalarIndexExpr::Query(search) = &mut expression { -+ search.query = Arc::new(SargableQuery::Range( -+ std::ops::Bound::Excluded(ScalarValue::Int32(Some(9))), -+ std::ops::Bound::Excluded(ScalarValue::Int32(Some(20))), -+ )); -+ } -+ assert_eq!(estimate_pages(&lookup(), &expression), Some(1)); -+ } -+ #[test] -+ fn expensive_in_cost_probes_fall_back_without_searching() { -+ let mut expression = eq(0); -+ if let ScalarIndexExpr::Query(search) = &mut expression { -+ search.query = Arc::new(SargableQuery::IsIn( -+ (0..257).map(|v| ScalarValue::Int32(Some(v))).collect(), -+ )); -+ } -+ assert_eq!(estimate_pages(&lookup(), &expression), None); -+ } -+} -diff --git a/src/scalar_segment.rs b/src/scalar_segment.rs -index 6515fbf..6354006 100644 ---- a/src/scalar_segment.rs -+++ b/src/scalar_segment.rs -@@ -26,6 +26,7 @@ use uuid::Uuid; - pub(crate) struct PreparedScalarSegment { - pub dataset: Arc, - pub segment_uuid: Uuid, -+ pub auto_selection: bool, - pub fragment_ids: Vec, - pub use_scalar_index: bool, - pub callback: Option, -@@ -39,7 +40,7 @@ fn invalid(message: impl Into) -> Error { - const MAX_EXPRESSION_NODES: usize = 128; - const MAX_EXPRESSION_DEPTH: usize = 32; - --fn scoped_expression( -+pub(crate) fn scoped_expression( - expr: &ScalarIndexExpr, - index_name: &str, - column: &str, -@@ -304,9 +305,45 @@ impl PreparedScalarSegment { - Ok(None) => return Ok(Some("no_driver")), - Err(reason) => return Ok(Some(reason)), - }; -+ let (segment_uuid, expr) = if self.auto_selection { -+ let physical_rows = fragments -+ .iter() -+ .filter(|f| scope.contains(&(f.id() as u64))) -+ .fold(0_u64, |sum, f| { -+ sum.saturating_add(f.metadata().physical_rows.unwrap_or(0) as u64) -+ }); -+ let Some(selected) = crate::scalar_index_selection::choose( -+ &self.dataset, -+ &indices, -+ index_meta, -+ expr, -+ filter_plan.index_query.as_ref().unwrap(), -+ &self.fragment_ids, -+ physical_rows, -+ stats, -+ ) -+ .await -+ else { -+ return Ok(Some("runtime_cost")); -+ }; -+ selected -+ } else { -+ (self.segment_uuid, expr) -+ }; -+ let index_meta = indices -+ .iter() -+ .find(|i| i.uuid == segment_uuid) -+ .ok_or_else(|| invalid("runtime scalar index is absent from the dataset snapshot"))?; -+ let field = self -+ .dataset -+ .schema() -+ .field_by_id(index_meta.keyed_field().unwrap()) -+ .ok_or_else(|| { -+ invalid("runtime scalar index field is absent from the dataset schema") -+ })?; - let index = self - .dataset -- .open_scalar_index(&field.name, &self.segment_uuid, metrics) -+ .open_scalar_index(&field.name, &segment_uuid, metrics) - .await?; - // These implementations can return exact candidates. Keep the runtime - // Exact check below: a type alone is not a guarantee for every query. -diff --git a/src/scanner.rs b/src/scanner.rs -index e5ff5ca..f2f1a00 100644 ---- a/src/scanner.rs -+++ b/src/scanner.rs -@@ -117,6 +117,7 @@ pub struct LanceScanner { - row_addr_blocklist: Option, - index_segments: Option>, - scalar_index_segment: Option, -+ scalar_index_auto_selection: bool, - nearest: Option, - nprobes: NprobesRange, - approx_mode: Option, -@@ -317,6 +318,7 @@ impl LanceScanner { - row_addr_blocklist: None, - index_segments: None, - scalar_index_segment: None, -+ scalar_index_auto_selection: false, - nearest: None, - nprobes: NprobesRange::default(), - approx_mode: None, -@@ -637,6 +639,7 @@ impl LanceScanner { - Some(PreparedScalarSegment { - dataset: Arc::clone(&self.dataset), - segment_uuid, -+ auto_selection: self.scalar_index_auto_selection, - fragment_ids: fragment_ids.clone(), - use_scalar_index: self.use_scalar_index.unwrap_or(true), - callback: self.scan_statistics_callback.clone(), -@@ -1079,6 +1082,22 @@ macro_rules! scanner_ffi_try { - }}; - } - -+/// Enable bounded runtime BTree selection for an explicitly scoped scalar segment scan. -+#[unsafe(no_mangle)] -+pub unsafe extern "C" fn lance_scanner_set_scalar_index_auto_selection( -+ scanner: *mut LanceScanner, -+ enabled: bool, -+) -> i32 { -+ scanner_poison_check!(scanner, -1); -+ scanner_ffi_try!(scanner, { -+ let scanner = unsafe { scanner.as_mut() } -+ .ok_or_else(|| lance_core::Error::invalid_input_source("scanner is NULL".into()))?; -+ scanner.ensure_scan_not_started("scalar_index_auto_selection")?; -+ scanner.scalar_index_auto_selection = enabled; -+ Ok(0) -+ }) -+} -+ - /// Select one physical scalar index segment. NULL clears the selection. - /// Requires explicit fragment_ids and an ordinary live-row scan. See the C header. - /// include_deleted_rows=true is rejected when preparing the scan, even if -diff --git a/tests/c_api_test.rs b/tests/c_api_test.rs -index d9af02a..0042745 100644 ---- a/tests/c_api_test.rs -+++ b/tests/c_api_test.rs -@@ -15704,6 +15704,18 @@ fn scalar_segment_ids_with_filter( - filter: ScalarSegmentFilter<'_>, - limit: Option, - offset: i64, -+) -> (Vec, CapturedScanStatistics) { -+ scalar_segment_ids_with_auto(uri, uuid, fragments, filter, limit, offset, false) -+} -+ -+fn scalar_segment_ids_with_auto( -+ uri: &str, -+ uuid: &[u8; 16], -+ fragments: &[u64], -+ filter: ScalarSegmentFilter<'_>, -+ limit: Option, -+ offset: i64, -+ auto_selection: bool, - ) -> (Vec, CapturedScanStatistics) { - let uri = c_str(uri); - let sql = match filter { -@@ -15737,6 +15749,10 @@ fn scalar_segment_ids_with_filter( - lance_scanner_set_scalar_index_segment(scanner, uuid.as_ptr()), - 0 - ); -+ assert_eq!( -+ lance_scanner_set_scalar_index_auto_selection(scanner, auto_selection), -+ 0 -+ ); - if let Some(limit) = limit { - assert_eq!(lance_scanner_set_limit(scanner, limit), 0); - } -@@ -15788,6 +15804,127 @@ fn scalar_segment_ids_with_filter( - (ids, captured) - } - -+#[test] -+fn test_runtime_scalar_index_selection_uses_page_cost_and_honors_switch() { -+ use lance::dataset::WriteParams; -+ use lance::index::DatasetIndexExt; -+ use lance_index::{IndexType, scalar::ScalarIndexParams}; -+ let tmp = tempfile::tempdir().unwrap(); -+ let uri = tmp -+ .path() -+ .join("runtime_selection") -+ .to_str() -+ .unwrap() -+ .to_owned(); -+ let (grouping, fragments) = lance_c::runtime::block_on(async { -+ let schema = Arc::new(Schema::new(vec![ -+ Field::new("id", DataType::Int32, false), -+ Field::new("key", DataType::Int32, false), -+ ])); -+ let batch = RecordBatch::try_new( -+ schema.clone(), -+ vec![ -+ Arc::new(Int32Array::from_iter_values(0..16384)), -+ Arc::new(Int32Array::from_iter_values(0..16384)), -+ ], -+ ) -+ .unwrap(); -+ let mut ds = Dataset::write( -+ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), -+ &uri, -+ Some(WriteParams { -+ max_rows_per_file: 4096, -+ ..Default::default() -+ }), -+ ) -+ .await -+ .unwrap(); -+ ds.create_index( -+ &["key"], -+ IndexType::BTree, -+ Some("key_idx".into()), -+ &ScalarIndexParams::default(), -+ false, -+ ) -+ .await -+ .unwrap(); -+ ds.create_index( -+ &["id"], -+ IndexType::BTree, -+ Some("id_idx".into()), -+ &ScalarIndexParams::default(), -+ false, -+ ) -+ .await -+ .unwrap(); -+ let indices = ds.load_indices().await.unwrap(); -+ let uuid = *indices -+ .iter() -+ .find(|i| i.name == "key_idx") -+ .unwrap() -+ .uuid -+ .as_bytes(); -+ let fragments: Vec<_> = ds.get_fragments().iter().map(|f| f.id() as u64).collect(); -+ (uuid, fragments) -+ }); -+ let metric = |stats: &CapturedScanStatistics, key: &str| { -+ stats -+ .metrics -+ .iter() -+ .filter(|(name, _, _)| name == key) -+ .map(|(_, _, n)| n) -+ .sum::() -+ }; -+ for enabled in [false, true] { -+ let (ids, stats) = scalar_segment_ids_with_auto( -+ &uri, -+ &grouping, -+ &fragments, -+ ScalarSegmentFilter::Sql("key >= 0 AND id = 123"), -+ None, -+ 0, -+ enabled, -+ ); -+ assert_eq!(ids, vec![123]); -+ assert_eq!( -+ metric(&stats, "scalar_index_auto_changed"), -+ u64::from(enabled) -+ ); -+ assert_eq!( -+ metric(&stats, "scalar_index_auto_candidates"), -+ if enabled { 2 } else { 0 } -+ ); -+ if enabled { -+ assert_eq!(metric(&stats, "scalar_index_auto_estimated_pages"), 1); -+ assert_eq!(metric(&stats, "scalar_index_auto_estimated_rows"), 4096); -+ assert_eq!(metric(&stats, "scalar_segments_searched"), 1); -+ } -+ } -+ let (ids, stats) = scalar_segment_ids_with_auto( -+ &uri, -+ &grouping, -+ &fragments, -+ ScalarSegmentFilter::Sql("key >= 0"), -+ None, -+ 0, -+ true, -+ ); -+ assert_eq!(ids.len(), 16384); -+ assert_eq!(metric(&stats, "scalar_index_auto_full_scans"), 1); -+ assert_eq!(metric(&stats, "scalar_segments_searched"), 0); -+ // A switch cannot broaden an explicitly assigned fragment domain. -+ let (ids, _) = scalar_segment_ids_with_auto( -+ &uri, -+ &grouping, -+ &[fragments[1]], -+ ScalarSegmentFilter::Sql("key >= 0 AND id >= 0"), -+ Some(2), -+ 1, -+ true, -+ ); -+ assert_eq!(ids, vec![4097, 4098]); -+} -+ - #[test] - fn test_scalar_segment_scope_residual_limit_and_unindexed_fallback() { - for kind in [ diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 55083e0983eec9..26b83f7dcc234d 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -580,11 +580,11 @@ PUGIXML_SOURCE=pugixml-1.15 PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895" # lance-c -# Includes lance-c #97: resolve index/column name collisions without serializing prewarm. -LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/4fafcd64525b274b113208390bbcca86bbbc093b" -LANCE_C_NAME="lance-c-4fafcd64525b274b113208390bbcca86bbbc093b.tar.gz" -LANCE_C_SOURCE="lance-c-4fafcd64525b274b113208390bbcca86bbbc093b" -LANCE_C_MD5SUM="65e924fb62137c5dd4d58af3fb33923a" +# Includes lance-c #98: bounded runtime BTree segment selection with decoded-memory guards. +LANCE_C_DOWNLOAD="https://codeload.github.com/lance-format/lance-c/tar.gz/a51ab47a9c814a746cb1714c99fad93dd3a188b8" +LANCE_C_NAME="lance-c-a51ab47a9c814a746cb1714c99fad93dd3a188b8.tar.gz" +LANCE_C_SOURCE="lance-c-a51ab47a9c814a746cb1714c99fad93dd3a188b8" +LANCE_C_MD5SUM="d1282cff3d63acfc0450db1f00d6ab6f" # paimon-rust PAIMON_RUST_DOWNLOAD="https://github.com/apache/paimon-rust/archive/refs/tags/v0.4.0-rc1.tar.gz" From 501aa2543862a760e2f7c7e0682226c33e02e534 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 26 Oct 2026 22:25:08 +0800 Subject: [PATCH 4/5] [fix](regression) Account for Lance runtime scan choices ### What problem does this PR solve? Problem Summary: The array-predicate suite required every segment-grouped query to search its index. The runtime selector can deliberately scan a tiny BTree read domain at lower estimated cost, so correct rows were followed by an expected-one/actual-zero search assertion failure. Validate requested segments against index searches plus explicit automatic full scans. Preserve exact candidate counts for index searches, require zero candidates for cost-selected scans, and reject unrelated segment fallbacks. Label-only predicates must still search LabelList. ### Release note None. Regression assertion changes only. ### Check List (For Author) - Test: actual suite assertion block passed six valid profile paths and rejected five invalid profiles; the old assertion reproduced the failure. Native runtime-selection C API test, full-suite Groovy 4.0.19 parsing, whitespace and clang-format 16 checks passed. Full external execution remains for CI. - Behavior changed: No product behavior changes. - Does this need documentation: No. --- ...test_lance_array_predicate_pushdown.groovy | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy b/regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy index 7c2d21a8e029ef..efb550153cd22b 100644 --- a/regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy +++ b/regression-test/suites/external_table_p0/lance/test_lance_array_predicate_pushdown.groovy @@ -86,26 +86,34 @@ suite("test_lance_array_predicate_pushdown", "p0,external") { } } assertEquals(c[1], sql(query).collect { (it[0] as Number).intValue() }) - // A pushed predicate is not necessarily indexed: also verify runtime searches - // and candidate counts, so a non-indexed fallback cannot satisfy this test. if (table != "unindexed") { String profile = profiles.getProfileBySql(token, - ["LanceScalarIndexSegmentsSearched", "LanceScalarIndexCandidateRows", - "LanceScalarIndexSegmentFallbacks"]) + ["LanceScalarIndexSegmentsRequested", "LanceScalarIndexSegmentsSearched", + "LanceScalarIndexCandidateRows", "LanceScalarIndexSegmentFallbacks", + "LanceScalarIndexAutoFullScans"]) def counter = { String name -> def matches = profile =~ /${name}: (?:sum )?(\d+)\b/ assertTrue(matches.find(), "Missing counter ${name}: ${profile}") return matches.group(1).toLong() } - // Broad complements stay as parallel scans without repeated segment searches. - long expectedSearches = c[2] == -1 ? 0L : 1L - assertEquals(expectedSearches, counter("LanceScalarIndexSegmentsSearched")) - long expectedCandidates = c[2] == null ? -1L : + // The cost selector may scan a small indexed domain instead of searching. + // Account for that decision without accepting unrelated index fallbacks. + long expectedRequests = c[2] == -1 ? 0L : 1L + long runtimeFullScans = counter("LanceScalarIndexAutoFullScans") + // Only category has a cost-ranked BTree; label-only predicates + // must continue to exercise their LabelList index. + if (!c[0].contains("category")) { + assertEquals(0L, runtimeFullScans) + } + assertEquals(expectedRequests, counter("LanceScalarIndexSegmentsRequested")) + assertEquals(expectedRequests, + counter("LanceScalarIndexSegmentsSearched") + runtimeFullScans) + long expectedCandidates = runtimeFullScans > 0 ? 0L : c[2] == null ? -1L : (c[table == "partial" ? 3 : 2] as Number).longValue() if (expectedCandidates >= 0) { assertEquals(expectedCandidates, counter("LanceScalarIndexCandidateRows")) } - assertEquals(0L, counter("LanceScalarIndexSegmentFallbacks")) + assertEquals(runtimeFullScans, counter("LanceScalarIndexSegmentFallbacks")) } } // NULL needle matching and ordered subsequence matching have Doris-specific semantics. From 9e52ffd3b1c345fbd07de268451240624a3cee70 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 27 Oct 2026 09:44:58 +0800 Subject: [PATCH 5/5] [fix](lance) Publish Session cache misses before profile snapshots Session cache windows collected only at dataset close can arrive after the final profile has already been copied when LIMIT stops a scan with a native reader still running. Refresh active windows before Thrift/protobuf profile serialization. Token-owned callbacks capture independent observation state, and sampling and close share a synchronized baseline so repeated snapshots and late close do not count a delta twice. Initial Session initialization stays outside the sampling lock. Add focused tests for a held native read with concurrent LIMIT 1, overlapping windows, concurrent close, failed snapshots and callback expiration. Update the reader test to inspect the profile before close and document the snapshot observation boundary. Validation: 14 focused C++ tests passed with ASAN/UBSAN and leak detection; disabling the snapshot hook reproduces both missing-window regressions. The complete reader/test syntax checks, clang-format 16 checks on all affected PR C++ files and whitespace checks passed. Full Doris build and external regression remain for CI. --- .../lance/lance_session_cache_profile.cpp | 114 +++++++++ .../lance/lance_session_cache_profile.h | 53 +++++ be/src/format_v2/table/lance_reader.cpp | 49 +--- be/src/format_v2/table/lance_reader.h | 9 +- be/src/runtime/runtime_profile.cpp | 29 +++ be/src/runtime/runtime_profile.h | 11 + .../lance_session_cache_profile_test.cpp | 219 ++++++++++++++++++ be/test/format_v2/table/lance_reader_test.cpp | 5 +- be/test/runtime/runtime_profile_test.cpp | 25 ++ docs/lance-cache-and-index-selection.md | 10 +- 10 files changed, 474 insertions(+), 50 deletions(-) create mode 100644 be/src/format_v2/lance/lance_session_cache_profile.cpp create mode 100644 be/src/format_v2/lance/lance_session_cache_profile.h create mode 100644 be/test/format_v2/lance/lance_session_cache_profile_test.cpp diff --git a/be/src/format_v2/lance/lance_session_cache_profile.cpp b/be/src/format_v2/lance/lance_session_cache_profile.cpp new file mode 100644 index 00000000000000..82278d37a63157 --- /dev/null +++ b/be/src/format_v2/lance/lance_session_cache_profile.cpp @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_cache_profile.h" + +#include +#include +#include +#include + +#include "common/logging.h" + +namespace doris::format::lance { + +struct LanceSessionCacheProfile::State { + State(RuntimeProfile* profile, RuntimeProfile::Counter* index_misses, + RuntimeProfile::Counter* metadata_misses, Snapshot snapshot) + : profile(profile), + index_misses(index_misses), + metadata_misses(metadata_misses), + snapshot(std::move(snapshot)) {} + + void collect() { + std::lock_guard guard(mutex); + collect_locked(); + } + + void collect_locked() { + if (!active || !valid) { + return; + } + LanceSessionManager::CacheStatistics current; + const auto status = snapshot(¤t); + if (!status.ok()) { + complete = false; + profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); + LOG(WARNING) << status; + return; + } + // Serialization and dataset close share a baseline under this lock. Neither can + // count a delta twice, or extend a closed window with unrelated Session activity. + const auto update = [](RuntimeProfile::Counter* counter, uint64_t before, uint64_t after) { + if (after >= before) { + COUNTER_UPDATE(counter, + static_cast(std::min( + after - before, std::numeric_limits::max()))); + } + }; + update(index_misses, previous.index_cache_misses, current.index_cache_misses); + update(metadata_misses, previous.metadata_cache_misses, current.metadata_cache_misses); + previous = current; + profile->add_info_string("LanceSessionCacheStatsAvailable", complete ? "true" : "false"); + } + + std::mutex mutex; + RuntimeProfile* profile; + RuntimeProfile::Counter* index_misses; + RuntimeProfile::Counter* metadata_misses; + Snapshot snapshot; + LanceSessionManager::CacheStatistics previous; + bool active = false; + bool valid = false; + bool complete = true; +}; + +LanceSessionCacheProfile::LanceSessionCacheProfile(RuntimeProfile* profile, + RuntimeProfile::Counter* index_misses, + RuntimeProfile::Counter* metadata_misses, + Snapshot snapshot) + : _state(std::make_shared(profile, index_misses, metadata_misses, + std::move(snapshot))), + _snapshot_callback( + profile->add_snapshot_callback([state = _state] { state->collect(); })) {} + +LanceSessionCacheProfile::~LanceSessionCacheProfile() { + stop(); +} + +void LanceSessionCacheProfile::start() { + // First-use Session initialization may perform I/O. A profile snapshot must not wait + // for it; the dataset-open window starts only after its baseline is ready. + LanceSessionManager::CacheStatistics previous; + const bool valid = _state->snapshot(&previous).ok(); + std::lock_guard guard(_state->mutex); + _state->active = true; + _state->valid = valid; + _state->previous = previous; + if (!_state->valid) { + _state->complete = false; + _state->profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); + } +} + +void LanceSessionCacheProfile::stop() { + std::lock_guard guard(_state->mutex); + _state->collect_locked(); + _state->active = false; +} + +} // namespace doris::format::lance diff --git a/be/src/format_v2/lance/lance_session_cache_profile.h b/be/src/format_v2/lance/lance_session_cache_profile.h new file mode 100644 index 00000000000000..f4d301d739986e --- /dev/null +++ b/be/src/format_v2/lance/lance_session_cache_profile.h @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +#include "format_v2/lance/lance_session_manager.h" +#include "runtime/runtime_profile.h" + +namespace doris::format::lance { + +// Observes one reader's dataset-open windows without accessing its native dataset or scanner. +class LanceSessionCacheProfile { +public: + using Snapshot = std::function; + + LanceSessionCacheProfile( + RuntimeProfile* profile, RuntimeProfile::Counter* index_misses, + RuntimeProfile::Counter* metadata_misses, Snapshot snapshot = [](auto* stats) { + return LanceSessionManager::instance().get_cache_statistics(stats); + }); + ~LanceSessionCacheProfile(); + + LanceSessionCacheProfile(const LanceSessionCacheProfile&) = delete; + LanceSessionCacheProfile& operator=(const LanceSessionCacheProfile&) = delete; + + // The reader serializes start/stop; profile serialization may sample concurrently. + void start(); + void stop(); + +private: + struct State; + std::shared_ptr _state; + std::shared_ptr _snapshot_callback; +}; + +} // namespace doris::format::lance diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index 462549f76c398c..752ce7460c29e9 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -34,6 +34,7 @@ #include "exec/common/endian.h" #include "format_v2/lance/lance_reader_helper.h" #include "format_v2/lance/lance_runtime_filter_helper.h" +#include "format_v2/lance/lance_session_cache_profile.h" #include "format_v2/lance/lance_session_manager.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" @@ -122,15 +123,17 @@ Status LanceTableReader::init(TableReadOptions&& options) { file_scan_profile::TABLE_READER, 1); _dataset_open_time = add_lance_counter(_scanner_profile, "LanceDatasetOpenTime", TUnit::TIME_NS, LANCE_TIMING_PROFILE); - _session_index_cache_misses = add_lance_counter( + auto* session_index_cache_misses = add_lance_counter( _scanner_profile, "LanceSessionIndexCacheMisses", TUnit::UNIT, "LanceSessionCache"); - _session_metadata_cache_misses = add_lance_counter( + auto* session_metadata_cache_misses = add_lance_counter( _scanner_profile, "LanceSessionMetadataCacheMisses", TUnit::UNIT, "LanceSessionCache"); _scanner_profile->add_info_string( "LanceSessionCacheStatsScope", "BE-wide changes during dataset-open windows; includes concurrent readers; " "overlapping windows must not be summed as query-exclusive misses"); _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); + _session_cache_profile = std::make_unique( + _scanner_profile, session_index_cache_misses, session_metadata_cache_misses); _arrow_to_doris_block_time = add_lance_counter(_scanner_profile, "LanceArrowToDorisBlockTime", TUnit::TIME_NS, LANCE_TIMING_PROFILE); _data_cache_bytes_read_from_cache = @@ -662,17 +665,13 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) { { SCOPED_TIMER(_dataset_open_time); auto& session = LanceSessionManager::instance(); - _session_cache_statistics_valid = - session.get_cache_statistics(&_session_cache_statistics).ok(); - if (!_session_cache_statistics_valid) { - _session_cache_statistics_complete = false; - _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); - } + _session_cache_profile->start(); LanceDataset* raw_dataset = nullptr; Status status = session.open_dataset( key.uri.c_str(), key.storage_options.empty() ? nullptr : storage_option_ptrs.data(), static_cast(key.version), &raw_dataset); if (!status.ok()) { + _session_cache_profile->stop(); return status.prepend(fmt::format("Lance dataset version {} at {}: ", key.version, display_uri(key.uri))); } @@ -1285,8 +1284,10 @@ void LanceTableReader::_close_dataset() { lance_fts_query_context_close(_fts_query_context); _fts_query_context = nullptr; } + if (_session_cache_profile != nullptr) { + _session_cache_profile->stop(); + } if (_dataset != nullptr) { - _collect_session_cache_statistics(); _collect_data_cache_statistics(); lance_dataset_close(_dataset); _dataset = nullptr; @@ -1297,36 +1298,6 @@ void LanceTableReader::_close_dataset() { _record_batch_converter.reset_schema(); } -void LanceTableReader::_collect_session_cache_statistics() { - if (!_session_cache_statistics_valid) { - return; - } - LanceSessionManager::CacheStatistics current; - const auto status = LanceSessionManager::instance().get_cache_statistics(¤t); - if (!status.ok()) { - _session_cache_statistics_complete = false; - _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", "false"); - LOG(WARNING) << status; - return; - } - // Session counters are monotonic and shared. Subtract the last snapshot exactly once - // per window; sampling them never resets counters used by other readers or metrics. - const auto update = [](RuntimeProfile::Counter* counter, uint64_t before, uint64_t after) { - if (counter != nullptr && after >= before) { - const auto delta = - std::min(after - before, std::numeric_limits::max()); - COUNTER_UPDATE(counter, static_cast(delta)); - } - }; - update(_session_index_cache_misses, _session_cache_statistics.index_cache_misses, - current.index_cache_misses); - update(_session_metadata_cache_misses, _session_cache_statistics.metadata_cache_misses, - current.metadata_cache_misses); - _session_cache_statistics = current; - _scanner_profile->add_info_string("LanceSessionCacheStatsAvailable", - _session_cache_statistics_complete ? "true" : "false"); -} - void LanceTableReader::_collect_data_cache_statistics() { if (_dataset == nullptr) { return; diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h index 843336f36cf14b..ba09954afb57d8 100644 --- a/be/src/format_v2/table/lance_reader.h +++ b/be/src/format_v2/table/lance_reader.h @@ -26,7 +26,7 @@ #include "common/status.h" #include "format_v2/lance/lance_record_batch_converter.h" -#include "format_v2/lance/lance_session_manager.h" +#include "format_v2/lance/lance_session_cache_profile.h" #include "format_v2/table_reader.h" #include "runtime/runtime_profile.h" @@ -93,7 +93,6 @@ class LanceTableReader final : public TableReader { // Collect the cumulative statistics owned by this dataset handle. The Lance-C API returns an // absolute snapshot, so this method replaces rather than increments the profile counters. void _collect_data_cache_statistics(); - void _collect_session_cache_statistics(); // Keep lance-c's anonymous statistics typedef out of this header. _open_scanner installs the // strongly typed C callback adapter before forwarding the borrowed value here. static void _collect_scan_statistics(void* callback_ctx, const void* opaque_statistics); @@ -103,11 +102,7 @@ class LanceTableReader final : public TableReader { Status _dataset_key(const TFileRangeDesc& range, DatasetKey* key) const; LanceDataset* _dataset = nullptr; - LanceSessionManager::CacheStatistics _session_cache_statistics; - bool _session_cache_statistics_valid = false; - bool _session_cache_statistics_complete = true; - RuntimeProfile::Counter* _session_index_cache_misses = nullptr; - RuntimeProfile::Counter* _session_metadata_cache_misses = nullptr; + std::unique_ptr _session_cache_profile; std::shared_ptr _dataset_schema; LanceScanner* _scanner = nullptr; ShardedKVCache* _runtime_filter_cache = nullptr; diff --git a/be/src/runtime/runtime_profile.cpp b/be/src/runtime/runtime_profile.cpp index 311d434f6bcff9..bbd1fc85d85a61 100644 --- a/be/src/runtime/runtime_profile.cpp +++ b/be/src/runtime/runtime_profile.cpp @@ -703,12 +703,40 @@ void RuntimeProfile::pretty_print(std::ostream* s, const std::string& prefix, } } +std::shared_ptr RuntimeProfile::add_snapshot_callback( + SnapshotCallback callback) { + auto registration = std::make_shared(std::move(callback)); + std::lock_guard l(_counter_map_lock); + _snapshot_callbacks.emplace_back(registration); + return registration; +} + +void RuntimeProfile::_run_snapshot_callbacks() { + std::vector> callbacks; + { + std::lock_guard l(_counter_map_lock); + std::erase_if(_snapshot_callbacks, [&](auto& weak) { + if (auto callback = weak.lock()) { + callbacks.emplace_back(std::move(callback)); + return false; + } + return true; + }); + } + // A stopped scanner can remain alive in a native read after pipeline completion. Sample + // before the immutable final copy, without holding locks needed by counter publication. + for (const auto& callback : callbacks) { + (*callback)(); + } +} + void RuntimeProfile::to_thrift(TRuntimeProfileTree* tree, int64_t profile_level) { tree->nodes.clear(); to_thrift(&tree->nodes, profile_level); } void RuntimeProfile::to_thrift(std::vector* nodes, int64_t profile_level) { + _run_snapshot_callbacks(); size_t index = nodes->size(); nodes->push_back(TRuntimeProfileNode()); TRuntimeProfileNode& node = (*nodes)[index]; @@ -756,6 +784,7 @@ void RuntimeProfile::to_proto(PRuntimeProfileTree* tree, int64_t profile_level) void RuntimeProfile::to_proto(google::protobuf::RepeatedPtrField* nodes, int64_t profile_level) { + _run_snapshot_callbacks(); PRuntimeProfileNode* node = nodes->Add(); // allocate new node node->set_name(_name); node->set_metadata(_metadata); diff --git a/be/src/runtime/runtime_profile.h b/be/src/runtime/runtime_profile.h index e63480711ac397..0873cc6ed9b880 100644 --- a/be/src/runtime/runtime_profile.h +++ b/be/src/runtime/runtime_profile.h @@ -584,6 +584,13 @@ class RuntimeProfile { // the key does not exist. const std::string* get_info_string(const std::string& key); + using SnapshotCallback = std::function; + // Refresh observations before serializing counters and info strings, without profile locks. + // Keep the returned token alive to stay registered. In-flight callbacks can outlive the + // token, so callbacks must own their state and synchronize with its producer's shutdown. + // Callbacks must not throw or recursively serialize this profile. + std::shared_ptr add_snapshot_callback(SnapshotCallback callback); + // Returns the counter for the total elapsed time. Counter* total_time_counter() { return &_counter_total_time; } @@ -698,6 +705,10 @@ class RuntimeProfile { // protects _counter_map, _counter_child_map and _bucketing_counters mutable std::mutex _counter_map_lock; + // Protected by _counter_map_lock. Expired registrations are pruned during snapshots. + std::vector> _snapshot_callbacks; + void _run_snapshot_callbacks(); + // Child profiles. Does not own memory. // We record children in both a map (to facilitate updates) and a vector // (to print things in the order they were registered) diff --git a/be/test/format_v2/lance/lance_session_cache_profile_test.cpp b/be/test/format_v2/lance/lance_session_cache_profile_test.cpp new file mode 100644 index 00000000000000..425779d487c5d6 --- /dev/null +++ b/be/test/format_v2/lance/lance_session_cache_profile_test.cpp @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "format_v2/lance/lance_session_cache_profile.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "util/defer_op.h" + +namespace doris::format::lance { + +TEST(LanceSessionCacheProfileTest, EarlyLimitSnapshotsWhileAnotherWorkerIsInNativeRead) { + std::unique_ptr session( + lance_session_new(64 * 1024 * 1024, 64 * 1024 * 1024), lance_session_close); + ASSERT_NE(nullptr, session); + auto snapshot = [&](LanceSessionManager::CacheStatistics* stats) { + LanceSessionCacheStats native {}; + if (lance_session_get_cache_stats(session.get(), &native) != 0) { + return Status::InternalError("native snapshot unavailable"); + } + *stats = {native.index_cache_misses, native.metadata_cache_misses}; + return Status::OK(); + }; + RuntimeProfile profile("early_limit"); + auto* index = profile.add_counter("index", TUnit::UNIT); + auto* metadata = profile.add_counter("metadata", TUnit::UNIT); + LanceSessionCacheProfile slow(&profile, index, metadata, snapshot); + LanceSessionCacheProfile fast(&profile, index, metadata, snapshot); + LanceSessionManager::CacheStatistics before_slow; + ASSERT_TRUE(snapshot(&before_slow).ok()); + slow.start(); + const char* uri = "./be/test/format_v2/table/lance/data/all_types.lance"; + std::unique_ptr slow_dataset( + lance_dataset_open_with_session(uri, nullptr, 0, session.get()), lance_dataset_close); + ASSERT_NE(nullptr, slow_dataset); + LanceSessionManager::CacheStatistics before_fast; + ASSERT_TRUE(snapshot(&before_fast).ok()); + fast.start(); + std::unique_ptr fast_dataset( + lance_dataset_open_with_session(uri, nullptr, 0, session.get()), lance_dataset_close); + ASSERT_NE(nullptr, fast_dataset); + const char* columns[] = {"row_id", nullptr}; + std::unique_ptr slow_scanner( + lance_scanner_new(slow_dataset.get(), columns, nullptr), lance_scanner_close); + ASSERT_NE(nullptr, slow_scanner); + struct BlockedRead { + std::promise entered; + std::shared_future released; + std::atomic held = false; + }; + std::promise release; + BlockedRead blocked; + blocked.released = release.get_future().share(); + auto entered = blocked.entered.get_future(); + ASSERT_EQ(0, lance_scanner_set_statistics_callback( + slow_scanner.get(), + [](void* ctx, const LanceScanStatistics*) { + auto* blocked_read = static_cast(ctx); + if (!blocked_read->held.exchange(true)) { + // This callback runs inside native execution while the + // scanner worker is still waiting for its read to finish. + blocked_read->entered.set_value(); + blocked_read->released.wait(); + } + }, + &blocked)); + std::thread worker([&] { + LanceBatch* batch = nullptr; + int32_t result; + while ((result = lance_scanner_next(slow_scanner.get(), &batch)) == 0) { + lance_batch_free(batch); + } + EXPECT_EQ(1, result); + slow_scanner.reset(); + slow.stop(); + }); + Defer join {[&] { + release.set_value(); + worker.join(); + }}; + ASSERT_EQ(std::future_status::ready, entered.wait_for(std::chrono::seconds(10))); + std::unique_ptr fast_scanner( + lance_scanner_new(fast_dataset.get(), columns, nullptr), lance_scanner_close); + ASSERT_NE(nullptr, fast_scanner); + ASSERT_EQ(0, lance_scanner_set_limit(fast_scanner.get(), 1)); + LanceBatch* batch = nullptr; + ASSERT_EQ(0, lance_scanner_next(fast_scanner.get(), &batch)); + lance_batch_free(batch); + fast_scanner.reset(); + fast.stop(); + LanceSessionManager::CacheStatistics current; + ASSERT_TRUE(snapshot(¤t).ok()); + ASSERT_GT(current.metadata_cache_misses, before_slow.metadata_cache_misses); + TRuntimeProfileTree saved; + profile.to_thrift(&saved); + auto restored = RuntimeProfile::from_thrift(saved); + EXPECT_EQ("true", saved.nodes[0].info_strings.at("LanceSessionCacheStatsAvailable")); + EXPECT_GE(restored->get_counter("metadata")->value(), + (current.metadata_cache_misses - before_slow.metadata_cache_misses) + + (current.metadata_cache_misses - before_fast.metadata_cache_misses)); +} + +TEST(LanceSessionCacheProfileTest, SnapshotOpenWindowsAndDoNotRecountOnClose) { + RuntimeProfile profile("session"); + auto* index = profile.add_counter("index", TUnit::UNIT); + auto* metadata = profile.add_counter("metadata", TUnit::UNIT); + LanceSessionManager::CacheStatistics current {10, 20}; + auto snapshot = [&](auto* stats) { + *stats = current; + return Status::OK(); + }; + LanceSessionCacheProfile first(&profile, index, metadata, snapshot); + LanceSessionCacheProfile second(&profile, index, metadata, snapshot); + first.start(); + current = {12, 23}; + second.start(); + current = {15, 27}; + first.stop(); + + TRuntimeProfileTree saved; + profile.to_thrift(&saved); + auto restored = RuntimeProfile::from_thrift(saved); + EXPECT_EQ(8, restored->get_counter("index")->value()); + EXPECT_EQ(11, restored->get_counter("metadata")->value()); + EXPECT_EQ("true", saved.nodes[0].info_strings.at("LanceSessionCacheStatsAvailable")); + profile.to_thrift(&saved); + second.stop(); + second.stop(); + EXPECT_EQ(8, index->value()); + EXPECT_EQ(11, metadata->value()); + + current = {100, 200}; + profile.to_thrift(&saved); + EXPECT_EQ(8, index->value()); + second.start(); + current = {102, 205}; + second.stop(); + EXPECT_EQ(10, index->value()); + EXPECT_EQ(16, metadata->value()); +} + +TEST(LanceSessionCacheProfileTest, ConcurrentSnapshotAndCloseCountEachDeltaOnce) { + RuntimeProfile profile("session"); + auto* index = profile.add_counter("index", TUnit::UNIT); + auto* metadata = profile.add_counter("metadata", TUnit::UNIT); + std::atomic current = 0; + LanceSessionCacheProfile window(&profile, index, metadata, [&](auto* stats) { + const auto value = current.load(); + *stats = {value, value}; + return Status::OK(); + }); + std::barrier rendezvous(2); + std::thread reporter([&] { + for (int i = 0; i < 100; ++i) { + rendezvous.arrive_and_wait(); + TRuntimeProfileTree saved; + profile.to_thrift(&saved); + rendezvous.arrive_and_wait(); + } + }); + for (int i = 0; i < 100; ++i) { + window.start(); + current.fetch_add(1); + rendezvous.arrive_and_wait(); + window.stop(); + rendezvous.arrive_and_wait(); + } + reporter.join(); + EXPECT_EQ(100, index->value()); + EXPECT_EQ(100, metadata->value()); +} + +TEST(LanceSessionCacheProfileTest, FailedSnapshotsStayUnavailableAndExpiredWindowsAreSafe) { + RuntimeProfile profile("session"); + auto* index = profile.add_counter("index", TUnit::UNIT); + auto* metadata = profile.add_counter("metadata", TUnit::UNIT); + bool fail = false; + { + LanceSessionCacheProfile window(&profile, index, metadata, [&](auto* stats) { + *stats = {1, 1}; + return fail ? Status::InternalError("snapshot unavailable") : Status::OK(); + }); + window.start(); + fail = true; + TRuntimeProfileTree saved; + profile.to_thrift(&saved); + EXPECT_EQ("false", saved.nodes[0].info_strings.at("LanceSessionCacheStatsAvailable")); + fail = false; + profile.to_thrift(&saved); + EXPECT_EQ("false", saved.nodes[0].info_strings.at("LanceSessionCacheStatsAvailable")); + } + TRuntimeProfileTree saved; + profile.to_thrift(&saved); + EXPECT_EQ(0, index->value()); + EXPECT_EQ(0, metadata->value()); +} + +} // namespace doris::format::lance diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index c6f3cc879f2c16..c27190b8fe7751 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -1362,7 +1363,9 @@ TEST(LanceTableReaderSnapshotTest, ProfileReportsObservedSessionCacheMisses) { LanceTableReader reader; ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); ASSERT_TRUE(prepare_range(&reader, make_latest_lance_range(dataset_uri)).ok()); - ASSERT_TRUE(reader.close().ok()); + TRuntimeProfileTree saved; + profile.to_thrift(&saved); + ASSERT_EQ("true", saved.nodes[0].info_strings.at("LanceSessionCacheStatsAvailable")); LanceSessionManager::CacheStatistics after; ASSERT_TRUE(LanceSessionManager::instance().get_cache_statistics(&after).ok()); for (const char* name : {"LanceSessionIndexCacheMisses", "LanceSessionMetadataCacheMisses"}) { diff --git a/be/test/runtime/runtime_profile_test.cpp b/be/test/runtime/runtime_profile_test.cpp index 1b31c1b24fc9c9..fe5fbba2a4d97f 100644 --- a/be/test/runtime/runtime_profile_test.cpp +++ b/be/test/runtime/runtime_profile_test.cpp @@ -34,6 +34,31 @@ namespace doris { class RuntimeProfileTest : public testing::Test {}; +TEST(RuntimeProfileTest, SnapshotCallbacksRunBeforeSerializationWithoutProfileLocks) { + RuntimeProfile profile("snapshot"); + auto* counter = profile.add_counter("observed", TUnit::UNIT); + auto callback = profile.add_snapshot_callback([&] { + EXPECT_EQ(counter, profile.get_counter("observed")); + counter->update(1); + profile.add_info_string("available", "true"); + }); + TRuntimeProfileTree thrift; + profile.to_thrift(&thrift); + ASSERT_EQ(1, thrift.nodes.size()); + EXPECT_EQ("true", thrift.nodes[0].info_strings.at("available")); + auto restored = RuntimeProfile::from_thrift(thrift); + EXPECT_EQ(1, restored->get_counter("observed")->value()); + + PRuntimeProfileTree proto; + profile.to_proto(&proto); + auto proto_restored = RuntimeProfile::from_proto(proto); + EXPECT_EQ(2, proto_restored->get_counter("observed")->value()); + + callback.reset(); + profile.to_thrift(&thrift); + EXPECT_EQ(2, counter->value()); +} + TEST(RuntimeProfileTest, Basic) { RuntimeProfile profile_a("ProfileA"); RuntimeProfile profile_a1("ProfileA1"); diff --git a/docs/lance-cache-and-index-selection.md b/docs/lance-cache-and-index-selection.md index 111d0a0612a62e..32b934e67a564e 100644 --- a/docs/lance-cache-and-index-selection.md +++ b/docs/lance-cache-and-index-selection.md @@ -9,9 +9,13 @@ The `LanceReader / LanceSessionCache` profile group contains: | `LanceSessionIndexCacheMisses` | Increase in the BE-wide Lance Session index-cache miss counter during this reader's dataset-open windows. | | `LanceSessionMetadataCacheMisses` | Increase in the BE-wide Lance Session metadata-cache miss counter during the same windows, including dataset opening. | -The reader takes a baseline before opening its dataset and publishes the difference -when closing it. Reopening a dataset adds another observation window; closing a -reader twice does not count the same window twice. `LanceSessionCacheStatsAvailable` +The reader takes a baseline before opening its dataset. Profile serialization samples +active observation windows before copying counters and availability information, even +if an early LIMIT leaves a worker blocked in a native read. Dataset close publishes +the remaining delta and ends the window. Sampling and close share a synchronized +baseline, so repeated snapshots and close do not count the same delta twice. +Reopening a dataset adds another observation window. Activity after the final Profile +snapshot is outside that saved observation. `LanceSessionCacheStatsAvailable` is `true` only after statistics have been collected successfully for every observed window. When it is `false`, zero counters do not establish that the caches had no misses.