From 85b89c9c5c2749f6e2e0170c6d4cb44fb7bde905 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Sat, 31 Oct 2026 10:23:14 -0700 Subject: [PATCH] feat(global-index): support row filters in full-text search Add FullTextSearchBuilder::WithFilter(), aligned with Java withFilter. Rows that fail the filter are excluded before ranking, so the result is the top k among matching rows. Matching rows reach the existing index as include row ids, so BM25 statistics stay those of the full corpus. The builder splits the filter into partition and data predicates, as Java does: a conjunct over both partition and data fields stays in the data filter and also restricts the partitions it implies. The scan attaches the scalar index files that intersect each split, which bumps the split encoding to version 2, and the read resolves the data filter into matched rows with those files. The scan and the split reject a scalar index file whose row range is negative or reversed, or whose row count overflows int64. GlobalIndexEvaluatorImpl now reports the fields that decided a result, so that a superset can be told from an exact answer. Unlike in Java, a field with a conjunct that no index could evaluate does not count. The rows of a superset are excluded with a warning by default. With the new global-index.filter.refine-from-data option, FilteredRowIdReader verifies them against the planned snapshot instead. It reads the data with the schema that the search resolved, so a column renamed or replaced since then cannot point the filter at another field. Once the planned snapshot has expired, a search of a tag reads the tag instead, which must still refer to that snapshot. StaticFromTagStartingScanner now reads tags from the branch of the scan, as in Java, instead of always from the main branch, so a refined search of a branch tag verifies rows of that branch. The new scalar-index.search-mode option, which falls back on global-index.search-mode, supports only fast so far; a row filter in another mode fails with NotImplemented. --- docs/source/user_guide/global_index.rst | 57 +- include/paimon/defs.h | 13 + include/paimon/table/source/full_text_read.h | 12 +- include/paimon/table/source/full_text_scan.h | 4 +- .../table/source/full_text_search_builder.h | 23 +- src/paimon/CMakeLists.txt | 2 + src/paimon/common/defs.cpp | 2 + src/paimon/core/core_options.cpp | 28 +- src/paimon/core/core_options.h | 4 + src/paimon/core/core_options_test.cpp | 51 +- .../global_index_evaluator_impl.cpp | 117 +++- .../global_index_evaluator_impl.h | 31 +- .../global_index_evaluator_impl_test.cpp | 103 +++- .../global_index/global_index_scan_impl.cpp | 42 ++ .../global_index/global_index_scan_impl.h | 18 + .../source/data_evolution_full_text_read.cpp | 144 ++++- .../source/data_evolution_full_text_read.h | 39 +- .../data_evolution_full_text_read_test.cpp | 271 ++++++++- .../source/data_evolution_full_text_scan.cpp | 75 ++- .../source/data_evolution_full_text_scan.h | 21 +- .../data_evolution_full_text_scan_test.cpp | 59 ++ .../table/source/filtered_row_id_reader.cpp | 248 ++++++++ .../table/source/filtered_row_id_reader.h | 83 +++ .../source/filtered_row_id_reader_test.cpp | 108 ++++ .../table/source/full_text_search_builder.cpp | 135 ++++- .../table/source/full_text_search_split.cpp | 78 ++- .../source/full_text_search_split_test.cpp | 105 +++- .../index_full_text_search_split_impl.h | 28 +- .../static_from_tag_starting_scanner.h | 3 +- test/inte/global_index_test.cpp | 566 +++++++++++++++++- 30 files changed, 2262 insertions(+), 208 deletions(-) create mode 100644 src/paimon/core/table/source/filtered_row_id_reader.cpp create mode 100644 src/paimon/core/table/source/filtered_row_id_reader.h create mode 100644 src/paimon/core/table/source/filtered_row_id_reader_test.cpp diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index 5901e5c71..f3f3c0d4c 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -303,8 +303,7 @@ an index shard covers are searched, as in the default ``fast`` ``full-text-index Java Paimon, so rows written since the index was last built are not found until it is built for them. A search whose ``full-text-index.search-mode``, or else ``global-index.search-mode``, is ``full`` or ``detail``, which also search the rows that no index covers, fails with -``NotImplemented``. Row filters, which Java Paimon evaluates with scalar indexes before ranking, are -not supported yet. +``NotImplemented``. .. code-block:: cpp @@ -338,6 +337,8 @@ candidates only: which of the rows tied at the ``limit``-th score of a shard bec to its index. - ``SetPartitionFilter()`` restricts the search to some partitions, as in ``ScanContextBuilder``. +- ``WithFilter()`` restricts the search to the rows matching a predicate; see + `Row Filters`_ below. - The search reads the snapshot that the time-travel options select, as Java Paimon does: snapshot ``scan.snapshot-id``, the snapshot of tag ``scan.tag-name``, the latest snapshot at or before ``scan.timestamp`` or ``scan.timestamp-millis``, or else the latest snapshot. At most one of these @@ -359,6 +360,45 @@ to its index. - A column with a primary-key full-text index (``pk-full-text.index.columns``) cannot be searched yet. +Row Filters +~~~~~~~~~~~ + +``WithFilter()`` adds a row filter, as Java Paimon's ``withFilter`` does. Only the matching rows are +ranked, so the result is the top ``limit`` rows among them, not a filtered subset of the unfiltered +top ``limit``. The matching rows are handed to the existing index as include row ids, so BM25 +statistics stay those of the whole index shard. Repeated calls are ANDed. Conjuncts over partition +fields alone restrict the partitions instead, ANDed with ``SetPartitionFilter()``. As in Java +Paimon, any other conjunct still filters rows but also restricts the partitions to those it +implies: ``(p = 1 AND a > 0) OR (p = 2 AND a > 0)`` searches only partitions ``p = 1`` and +``p = 2``. + +.. code-block:: cpp + + auto filter = paimon::PredicateBuilder::Equal(/*field_index=*/1, "category", + paimon::FieldType::INT, paimon::Literal(7)); + search_builder.WithFilter(filter); + +The scan keeps the scalar global index files that index a field of the filter, and the read +resolves the filter with them before ranking: + +- If the indexes decide the whole filter, the rows they return are searched. +- If they can only narrow the filter down to candidates, because a conjunct has no index or the + filter has a ``Contains``, ``EndsWith`` or ``Like`` predicate, the candidates are excluded and a + warning is logged. The search then never returns a non-matching row, but may return fewer rows + than ``limit``. With ``global-index.filter.refine-from-data=true``, the read instead verifies the + candidates by reading their filter columns and ``_ROW_ID`` in the planned snapshot, on the caller + and possibly for every candidate. +- Rows that no scalar index covers are excluded. If no index can evaluate the filter at all, no row + is searched and a warning is logged. This is the default ``fast`` ``scalar-index.search-mode``, + which falls back on ``global-index.search-mode``. A row filter in ``full`` or ``detail`` mode, + which decide such rows by reading their filter columns, fails with ``NotImplemented``. + +Only single-field indexes that do not support full-text search evaluate a filter. A filter with a +``PredicateBuilder::FullTextMatch`` predicate fails with ``NotImplemented``. + +Pinning the Data Read +~~~~~~~~~~~~~~~~~~~~~ + The row ids of the result refer to the snapshot the search was planned from. If the table may change before the rows are read, plan with ``NewFullTextScan()`` and pin the data scan to the snapshot of the plan, setting ``scan.mode`` too so that a ``scan.mode`` of the table cannot override @@ -386,12 +426,13 @@ outlive the snapshot. ``ExecuteLocal()`` plans and searches in one process. To plan in one process and search in others, plan with ``NewFullTextScan()``, ship the splits with ``FullTextSearchSplit::Serialize()`` and ``FullTextSearchSplit::Deserialize()``, and search them with the ``NewFullTextRead()`` of a builder -for the same table, column and query. The serialized form is versioned and specific to Paimon C++: -Java Paimon serializes its splits with Java serialization, so splits cannot be exchanged with it. -Each split records the snapshot it was planned from, and the read ranks the rows live in that -snapshot. The read rejects a split whose index files do not index the column of the latest schema, -such as after the column was replaced. A read returns the top rows of the splits it is given, so -keep the ``limit`` highest-scoring rows of all reads if the splits are searched by several of them. +for the same table, column, query and filter. The serialized form is versioned and specific to +Paimon C++: Java Paimon serializes its splits with Java serialization, so splits cannot be exchanged +with it. Each split records the snapshot it was planned from, and the read ranks the rows live in +that snapshot. The read rejects a split whose index files do not index the column of the latest +schema, such as after the column was replaced. A read returns the top rows of the splits it is +given, so keep the ``limit`` highest-scoring rows of all reads if the splits are searched by several +of them. .. code-block:: cpp diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 10cc3759d..88af98c0d 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -623,6 +623,19 @@ struct PAIMON_EXPORT Options { /// index covers. Default value is "global-index.search-mode", or else "fast". Paimon C++ /// supports only "fast" so far, and a full-text search in another mode fails. static const char FULL_TEXT_INDEX_SEARCH_MODE[]; + /// "scalar-index.search-mode" - Search mode for scalar index queries: "fast" evaluates only the + /// rows that an index covers, while "full" and "detail" also evaluate the rows that no index + /// covers. Default value is "global-index.search-mode", or else "fast". Paimon C++ supports + /// only "fast" so far, and a full-text search with a row filter in another mode fails. + static const char SCALAR_INDEX_SEARCH_MODE[]; + /// "global-index.filter.refine-from-data" - Whether a vector, hybrid or full-text search may + /// read the filter columns of candidate rows to verify a row filter that the scalar global + /// index can only answer with a superset, such as contains, ends-with or like on a BTree index + /// or a conjunction with a member no index can evaluate. When false, such candidates are + /// excluded from the search, which never returns a non-matching row but may return fewer than + /// the requested top-k. When true, the read runs on the caller and may cover every candidate + /// row. Default value is "false". Paimon C++ reads it only in full-text search so far. + static const char GLOBAL_INDEX_FILTER_REFINE_FROM_DATA[]; /// "pk-btree.index.columns" - Comma-separated columns indexed by primary-key BTree indexes. /// No default value. static const char PK_BTREE_INDEX_COLUMNS[]; diff --git a/include/paimon/table/source/full_text_read.h b/include/paimon/table/source/full_text_read.h index 2760510e6..1d4a6f718 100644 --- a/include/paimon/table/source/full_text_read.h +++ b/include/paimon/table/source/full_text_read.h @@ -51,10 +51,18 @@ class PAIMON_EXPORT FullTextRead { /// ranks only its search row ranges, and with deletion vectors enabled only the rows that are /// live in the snapshot the splits were planned from; BM25 statistics still come from its whole /// index shard. - /// @param splits Splits planned by a `FullTextScan` of the same column, all from one snapshot. + /// + /// With a row filter (`FullTextSearchBuilder::WithFilter()`), a split ranks only the rows that + /// satisfy it, as resolved before ranking by the scalar indexes that the splits carry. Rows + /// that no scalar index can evaluate are not ranked. Candidates that the indexes cannot decide + /// exactly, as for `Contains`, `EndsWith` and `Like`, are excluded too, so the result may hold + /// fewer than `limit` rows, unless `global-index.filter.refine-from-data` verifies them against + /// the data of the planned snapshot on this thread. + /// @param splits Splits planned by a `FullTextScan` of the same column and filter, all from one + /// snapshot. /// @return The rows found, or an error if a split is invalid, its index files do not index the /// column of the latest schema (for example, after the column was replaced), or an - /// index cannot be read. + /// index or data file cannot be read. virtual Result> Read( const std::vector>& splits) = 0; }; diff --git a/include/paimon/table/source/full_text_scan.h b/include/paimon/table/source/full_text_scan.h index 8e0d17b86..dd4ccf569 100644 --- a/include/paimon/table/source/full_text_scan.h +++ b/include/paimon/table/source/full_text_scan.h @@ -76,7 +76,9 @@ class PAIMON_EXPORT FullTextScan { /// without the backend of a `full-text` or `lucene-fts` index it finds. When the row ranges of /// index shards overlap, each row is ranked by one shard only: an index whose indexed field is /// the column comes before one that carries it as an extra field, and further ties are broken - /// by index type and fields, then by the start and end of the shard. + /// by index type and fields, then by the start and end of the shard. With a row filter, each + /// split also carries the scalar index files that intersect its shard and index a field of the + /// filter. /// @return The plan, or an error if more than one time-travel option is set, the option does /// not agree with `scan.mode`, the selected snapshot or tag does not exist, or reading /// the index manifests fails; `NotImplemented` for an index whose backend is not built. diff --git a/include/paimon/table/source/full_text_search_builder.h b/include/paimon/table/source/full_text_search_builder.h index 2d18388c8..2d07d9829 100644 --- a/include/paimon/table/source/full_text_search_builder.h +++ b/include/paimon/table/source/full_text_search_builder.h @@ -25,6 +25,7 @@ #include #include "paimon/global_index/global_index_result.h" +#include "paimon/predicate/predicate.h" #include "paimon/result.h" #include "paimon/table/source/full_text_read.h" #include "paimon/table/source/full_text_scan.h" @@ -49,7 +50,8 @@ class Executor; /// /// `ExecuteLocal()` plans and searches in this process. To run them in different processes, plan /// with `NewFullTextScan()`, serialize the splits with `FullTextSearchSplit::Serialize()`, and -/// search them with the `NewFullTextRead()` of a builder for the same table, column and query. +/// search them with the `NewFullTextRead()` of a builder for the same table, column, query and +/// filter. /// /// The row ids of the result refer to the snapshot the search was planned from, which /// `FullTextScan::Plan::SnapshotId()` returns. If the table may change in between, pin the data @@ -68,6 +70,17 @@ class PAIMON_EXPORT FullTextSearchBuilder { FullTextSearchBuilder& SetPartitionFilter( const std::vector>& partition_filters); + /// Restricts the search to the rows matching `predicate`, as Java + /// `FullTextSearchBuilder#withFilter` does: only those rows are ranked, so the result is the + /// top `limit` rows among them, and BM25 statistics stay those of the whole index shard. The + /// fields of the predicate are matched by name. Repeated calls are ANDed. Conjuncts over + /// partition fields alone restrict the partitions, ANDed with `SetPartitionFilter()`, and the + /// rest is the row filter that `FullTextRead::Read()` resolves. A conjunct of the row filter + /// also restricts the partitions to those it implies, as `(p = 1 AND a > 0) OR (p = 2 AND + /// a > 0)` does to `p = 1 OR p = 2`. + /// @param predicate The filter; a null predicate adds no filter. + FullTextSearchBuilder& WithFilter(const std::shared_ptr& predicate); + /// Sets the number of highest-scoring rows to return. It must be positive to create a read. FullTextSearchBuilder& SetLimit(int32_t limit); @@ -100,9 +113,11 @@ class PAIMON_EXPORT FullTextSearchBuilder { /// Creates a scan that plans the search on the current settings. The query must be set, and /// its column must exist in the latest schema of the table. /// @return The scan, or an error if the settings are invalid or the table cannot be loaded; - /// `NotImplemented` for a primary-key full-text index (`pk-full-text.index.columns`) or - /// a search mode other than `fast` (`full-text-index.search-mode`, or else - /// `global-index.search-mode`). + /// `NotImplemented` for a primary-key full-text index (`pk-full-text.index.columns`), a + /// search mode other than `fast` (`full-text-index.search-mode`, or else + /// `global-index.search-mode`), a row filter whose scalar search mode is not `fast` + /// (`scalar-index.search-mode`, or else `global-index.search-mode`), or a filter with a + /// full-text match. Result> NewFullTextScan() const; /// Creates a read that searches the splits of a scan on the current settings. In addition to diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index ec0c6ea8c..1dc19bef5 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -484,6 +484,7 @@ set(PAIMON_CORE_SRCS core/table/source/data_evolution_batch_scan.cpp core/table/source/data_evolution_full_text_read.cpp core/table/source/data_evolution_full_text_scan.cpp + core/table/source/filtered_row_id_reader.cpp core/table/source/full_text_search_builder.cpp core/table/source/full_text_search_split.cpp core/table/source/primary_key_sorted_index_scan.cpp @@ -1033,6 +1034,7 @@ if(PAIMON_BUILD_TESTS) core/table/source/data_evolution_batch_scan_test.cpp core/table/source/data_evolution_full_text_read_test.cpp core/table/source/data_evolution_full_text_scan_test.cpp + core/table/source/filtered_row_id_reader_test.cpp core/table/source/full_text_search_split_test.cpp core/table/source/data_split_test.cpp core/table/source/deletion_file_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 8edcec3d3..dc88b6b38 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -164,6 +164,8 @@ const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; const char Options::GLOBAL_INDEX_SEARCH_MODE[] = "global-index.search-mode"; const char Options::FULL_TEXT_INDEX_SEARCH_MODE[] = "full-text-index.search-mode"; +const char Options::SCALAR_INDEX_SEARCH_MODE[] = "scalar-index.search-mode"; +const char Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA[] = "global-index.filter.refine-from-data"; const char Options::PK_BTREE_INDEX_COLUMNS[] = "pk-btree.index.columns"; const char Options::PK_BITMAP_INDEX_COLUMNS[] = "pk-bitmap.index.columns"; const char Options::PK_VECTOR_INDEX_COLUMNS[] = "pk-vector.index.columns"; diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index 6b8666a30..4ebe5cc55 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -178,11 +178,10 @@ class ConfigParser { "sort engine", sort_engine); } - // Parse GlobalIndexSearchMode of full-text search, as Java `CoreOptions#indexSearchMode` does - Status ParseFullTextIndexSearchMode(GlobalIndexSearchMode* search_mode) const { - const char* key = config_map_.count(Options::FULL_TEXT_INDEX_SEARCH_MODE) > 0 - ? Options::FULL_TEXT_INDEX_SEARCH_MODE - : Options::GLOBAL_INDEX_SEARCH_MODE; + // Parse GlobalIndexSearchMode of an index family, as Java `CoreOptions#indexSearchMode` does + Status ParseIndexSearchMode(const char* family_key, GlobalIndexSearchMode* search_mode) const { + const char* key = + config_map_.count(family_key) > 0 ? family_key : Options::GLOBAL_INDEX_SEARCH_MODE; return ParseEnum(key, {{"fast", GlobalIndexSearchMode::FAST}, {"full", GlobalIndexSearchMode::FULL}, @@ -450,6 +449,8 @@ struct CoreOptions::Impl { std::optional lookup_compact_max_interval; std::optional global_index_thread_num; GlobalIndexSearchMode full_text_index_search_mode = GlobalIndexSearchMode::FAST; + GlobalIndexSearchMode scalar_index_search_mode = GlobalIndexSearchMode::FAST; + bool global_index_filter_refine_from_data = false; bool realtime_enabled = false; bool realtime_spill_enabled = false; @@ -927,7 +928,14 @@ struct CoreOptions::Impl { PAIMON_RETURN_NOT_OK( parser.Parse(Options::GLOBAL_INDEX_THREAD_NUM, &global_index_thread_num)); // Parse full-text-index.search-mode, falling back to global-index.search-mode, default fast - PAIMON_RETURN_NOT_OK(parser.ParseFullTextIndexSearchMode(&full_text_index_search_mode)); + PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::FULL_TEXT_INDEX_SEARCH_MODE, + &full_text_index_search_mode)); + // Parse scalar-index.search-mode, falling back to global-index.search-mode, default fast + PAIMON_RETURN_NOT_OK(parser.ParseIndexSearchMode(Options::SCALAR_INDEX_SEARCH_MODE, + &scalar_index_search_mode)); + // Parse global-index.filter.refine-from-data, default false + PAIMON_RETURN_NOT_OK(parser.Parse(Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, + &global_index_filter_refine_from_data)); // Parse global-index.external-path - global index root directory PAIMON_RETURN_NOT_OK( parser.Parse(Options::GLOBAL_INDEX_EXTERNAL_PATH, &global_index_external_path)); @@ -1920,6 +1928,14 @@ GlobalIndexSearchMode CoreOptions::GetFullTextIndexSearchMode() const { return impl_->full_text_index_search_mode; } +GlobalIndexSearchMode CoreOptions::GetScalarIndexSearchMode() const { + return impl_->scalar_index_search_mode; +} + +bool CoreOptions::GlobalIndexFilterRefineFromData() const { + return impl_->global_index_filter_refine_from_data; +} + std::optional CoreOptions::GetGlobalIndexExternalPath() const { return impl_->global_index_external_path; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 136490362..bfc6ee251 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -307,6 +307,10 @@ class PAIMON_EXPORT CoreOptions { /// Returns `full-text-index.search-mode`, or else `global-index.search-mode`, or else /// `GlobalIndexSearchMode::FAST`, as Java `CoreOptions#fullTextIndexSearchMode` does. GlobalIndexSearchMode GetFullTextIndexSearchMode() const; + /// Returns `scalar-index.search-mode`, or else `global-index.search-mode`, or else + /// `GlobalIndexSearchMode::FAST`, as Java `CoreOptions#scalarIndexSearchMode` does. + GlobalIndexSearchMode GetScalarIndexSearchMode() const; + bool GlobalIndexFilterRefineFromData() const; const std::vector& GetBlobFields() const; const std::vector& GetBlobDescriptorFields() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 64c43a346..1344d9101 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -182,6 +182,7 @@ TEST(CoreOptionsTest, TestDefaultValue) { ASSERT_TRUE(core_options.BlobViewResolveEnabled()); ASSERT_TRUE(core_options.LegacyPartitionNameEnabled()); ASSERT_TRUE(core_options.GlobalIndexEnabled()); + ASSERT_FALSE(core_options.GlobalIndexFilterRefineFromData()); ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexExternalPath()); ASSERT_EQ(std::nullopt, core_options.GetGlobalIndexThreadNum()); ASSERT_EQ(std::nullopt, core_options.GetScanTagName()); @@ -312,6 +313,7 @@ TEST(CoreOptionsTest, TestFromMap) { {Options::BLOB_VIEW_RESOLVE_ENABLED, "false"}, {Options::PARTITION_GENERATE_LEGACY_NAME, "false"}, {Options::GLOBAL_INDEX_ENABLED, "false"}, + {Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, "true"}, {Options::GLOBAL_INDEX_THREAD_NUM, "4"}, {Options::GLOBAL_INDEX_EXTERNAL_PATH, "FILE:///tmp/global_index/"}, {Options::SCAN_TAG_NAME, "test-tag"}, @@ -487,6 +489,7 @@ TEST(CoreOptionsTest, TestFromMap) { ASSERT_FALSE(core_options.BlobViewResolveEnabled()); ASSERT_FALSE(core_options.LegacyPartitionNameEnabled()); ASSERT_FALSE(core_options.GlobalIndexEnabled()); + ASSERT_TRUE(core_options.GlobalIndexFilterRefineFromData()); ASSERT_EQ(core_options.GetGlobalIndexThreadNum(), 4); ASSERT_TRUE(core_options.GetGlobalIndexExternalPath()); ASSERT_EQ(core_options.GetGlobalIndexExternalPath().value(), "FILE:///tmp/global_index/"); @@ -1055,26 +1058,36 @@ TEST(CoreOptionsTest, TestParseChangelogProducer) { "invalid changelog producer: invalid"); } -TEST(CoreOptionsTest, TestParseFullTextIndexSearchMode) { - auto search_mode = - [](const std::map& options) -> Result { - PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); - return core_options.GetFullTextIndexSearchMode(); +TEST(CoreOptionsTest, TestParseIndexSearchMode) { + struct IndexFamily { + std::string key; + GlobalIndexSearchMode (CoreOptions::*get_search_mode)() const; }; - ASSERT_OK_AND_ASSIGN(GlobalIndexSearchMode mode, search_mode({})); - ASSERT_EQ(mode, GlobalIndexSearchMode::FAST); - ASSERT_OK_AND_ASSIGN(mode, search_mode({{Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); - ASSERT_EQ(mode, GlobalIndexSearchMode::FULL); - ASSERT_OK_AND_ASSIGN(mode, search_mode({{Options::FULL_TEXT_INDEX_SEARCH_MODE, "DETAIL"}})); - ASSERT_EQ(mode, GlobalIndexSearchMode::DETAIL); - ASSERT_OK_AND_ASSIGN(mode, search_mode({{Options::FULL_TEXT_INDEX_SEARCH_MODE, "fast"}, - {Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); - ASSERT_EQ(mode, GlobalIndexSearchMode::FAST); - ASSERT_NOK_WITH_MSG(search_mode({{Options::FULL_TEXT_INDEX_SEARCH_MODE, "invalid"}, - {Options::GLOBAL_INDEX_SEARCH_MODE, "fast"}}), - "invalid full-text-index.search-mode: invalid"); - ASSERT_NOK_WITH_MSG(search_mode({{Options::GLOBAL_INDEX_SEARCH_MODE, "invalid"}}), - "invalid global-index.search-mode: invalid"); + const std::vector families = { + {Options::FULL_TEXT_INDEX_SEARCH_MODE, &CoreOptions::GetFullTextIndexSearchMode}, + {Options::SCALAR_INDEX_SEARCH_MODE, &CoreOptions::GetScalarIndexSearchMode}}; + for (const IndexFamily& family : families) { + SCOPED_TRACE(family.key); + auto search_mode = [&family](const std::map& options) + -> Result { + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); + return (core_options.*family.get_search_mode)(); + }; + ASSERT_OK_AND_ASSIGN(GlobalIndexSearchMode mode, search_mode({})); + ASSERT_EQ(mode, GlobalIndexSearchMode::FAST); + ASSERT_OK_AND_ASSIGN(mode, search_mode({{Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); + ASSERT_EQ(mode, GlobalIndexSearchMode::FULL); + ASSERT_OK_AND_ASSIGN(mode, search_mode({{family.key, "DETAIL"}})); + ASSERT_EQ(mode, GlobalIndexSearchMode::DETAIL); + ASSERT_OK_AND_ASSIGN( + mode, search_mode({{family.key, "fast"}, {Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); + ASSERT_EQ(mode, GlobalIndexSearchMode::FAST); + ASSERT_NOK_WITH_MSG( + search_mode({{family.key, "invalid"}, {Options::GLOBAL_INDEX_SEARCH_MODE, "fast"}}), + "invalid " + family.key + ": invalid"); + ASSERT_NOK_WITH_MSG(search_mode({{Options::GLOBAL_INDEX_SEARCH_MODE, "invalid"}}), + "invalid global-index.search-mode: invalid"); + } } TEST(CoreOptionsTest, TestParseExternalPathStrategy) { diff --git a/src/paimon/core/global_index/global_index_evaluator_impl.cpp b/src/paimon/core/global_index/global_index_evaluator_impl.cpp index c857a9f0b..32f31f4c3 100644 --- a/src/paimon/core/global_index/global_index_evaluator_impl.cpp +++ b/src/paimon/core/global_index/global_index_evaluator_impl.cpp @@ -79,13 +79,39 @@ bool IsIsNotNull(const std::shared_ptr& predicate) { Result> GlobalIndexEvaluatorImpl::Evaluate( const std::shared_ptr& predicate) { - std::shared_ptr compound_result; - if (predicate) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_predicate, - NormalizePredicate(predicate)); - PAIMON_ASSIGN_OR_RAISE(compound_result, EvaluatePredicate(normalized_predicate)); + PAIMON_ASSIGN_OR_RAISE(std::optional evaluation, + EvaluateWithContributingFields(predicate)); + if (!evaluation) { + return std::shared_ptr(nullptr); + } + return evaluation->result; +} + +Result> +GlobalIndexEvaluatorImpl::EvaluateWithContributingFields( + const std::shared_ptr& predicate) { + if (!predicate) { + return std::optional(); } - return compound_result; + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr normalized_predicate, + NormalizePredicate(predicate)); + return EvaluatePredicate(normalized_predicate); +} + +Result> GlobalIndexEvaluatorImpl::CollectFieldIds( + const TableSchema& table_schema, const std::shared_ptr& predicate) { + std::set field_ids; + if (!predicate) { + return field_ids; + } + std::set field_names; + PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(predicate, &field_names)); + for (const DataField& field : table_schema.Fields()) { + if (field_names.count(field.Name()) > 0) { + field_ids.insert(field.Id()); + } + } + return field_ids; } Result> GlobalIndexEvaluatorImpl::NormalizePredicate( @@ -138,9 +164,7 @@ Result> GlobalIndexEvaluatorImpl::NormalizePredicate( } Result>> GlobalIndexEvaluatorImpl::GetIndexReaders( - const std::string& field_name) { - PAIMON_ASSIGN_OR_RAISE(DataField data_field, table_schema_->GetField(field_name)); - int32_t field_id = data_field.Id(); + int32_t field_id) { // get or create global index readers for current field std::vector> readers; auto iter = index_readers_cache_.find(field_id); @@ -153,23 +177,24 @@ Result>> GlobalIndexEvaluatorImpl return readers; } -Result> GlobalIndexEvaluatorImpl::EvaluatePredicate( - const std::shared_ptr& predicate) { +Result> +GlobalIndexEvaluatorImpl::EvaluatePredicate(const std::shared_ptr& predicate) { if (predicate == nullptr) { - return std::shared_ptr(nullptr); + return std::optional(); } if (auto compound_predicate = std::dynamic_pointer_cast(predicate)) { return EvaluateCompoundPredicate(compound_predicate); } else if (auto leaf_predicate = std::dynamic_pointer_cast(predicate)) { - const std::string& field_name = leaf_predicate->FieldName(); + PAIMON_ASSIGN_OR_RAISE(DataField field, + table_schema_->GetField(leaf_predicate->FieldName())); PAIMON_ASSIGN_OR_RAISE(std::vector> readers, - GetIndexReaders(field_name)); + GetIndexReaders(field.Id())); if (readers.empty()) { // No usable index for this field within the requested range. Treat as "no // pushdown available" so the upstream falls back to a full scan instead of // wrongly producing an empty result. - return std::shared_ptr(nullptr); + return std::optional(); } // calculate compound result as field may has multiple indexes std::shared_ptr compound_result; @@ -190,60 +215,86 @@ Result> GlobalIndexEvaluatorImpl::EvaluatePre if (compound_result) { PAIMON_ASSIGN_OR_RAISE(bool is_empty, compound_result->IsEmpty()); if (is_empty) { - return compound_result; + break; } } } - return compound_result; + if (!compound_result) { + return std::optional(); + } + return std::optional( + Evaluation{compound_result, {field.Id()}, /*declined_field_ids=*/{}}); } return Status::Invalid(fmt::format( "cannot cast predicate {} to CompoundPredicate or LeafPredicate", predicate->ToString())); } -Result> GlobalIndexEvaluatorImpl::EvaluateCompoundPredicate( +Result> +GlobalIndexEvaluatorImpl::EvaluateCompoundPredicate( const std::shared_ptr& compound_predicate) { + std::shared_ptr compound_result; + std::set contributing_field_ids; + std::set declined_field_ids; + auto merge_fields = [&](const Evaluation& evaluation) { + contributing_field_ids.insert(evaluation.contributing_field_ids.begin(), + evaluation.contributing_field_ids.end()); + declined_field_ids.insert(evaluation.declined_field_ids.begin(), + evaluation.declined_field_ids.end()); + }; if (compound_predicate->GetFunction().GetType() == Function::Type::OR) { - std::shared_ptr compound_result; for (const auto& child : compound_predicate->Children()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sub_result, + PAIMON_ASSIGN_OR_RAISE(std::optional sub_evaluation, EvaluatePredicate(child)); - if (!sub_result) { - return std::shared_ptr(nullptr); + if (!sub_evaluation) { + return std::optional(); } if (!compound_result) { - compound_result = sub_result; + compound_result = sub_evaluation->result; } else { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr or_result, - compound_result->Or(sub_result)); + compound_result->Or(sub_evaluation->result)); compound_result = or_result; } + merge_fields(*sub_evaluation); } - return compound_result; } else if (compound_predicate->GetFunction().GetType() == Function::Type::AND) { - std::shared_ptr compound_result; for (const auto& child : compound_predicate->Children()) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr sub_result, + PAIMON_ASSIGN_OR_RAISE(std::optional sub_evaluation, EvaluatePredicate(child)); - if (sub_result) { + if (sub_evaluation) { if (!compound_result) { - compound_result = sub_result; + compound_result = sub_evaluation->result; } else { PAIMON_ASSIGN_OR_RAISE(std::shared_ptr and_result, - compound_result->And(sub_result)); + compound_result->And(sub_evaluation->result)); compound_result = and_result; } + merge_fields(*sub_evaluation); + } else { + PAIMON_ASSIGN_OR_RAISE(std::set child_field_ids, + CollectFieldIds(*table_schema_, child)); + declined_field_ids.insert(child_field_ids.begin(), child_field_ids.end()); } if (compound_result) { PAIMON_ASSIGN_OR_RAISE(bool is_empty, compound_result->IsEmpty()); if (is_empty) { - return compound_result; + break; } } } - return compound_result; + } else { + return Status::Invalid("CompoundPredicate only support And/Or function"); + } + if (!compound_result) { + return std::optional(); + } + for (int32_t declined_field_id : declined_field_ids) { + contributing_field_ids.erase(declined_field_id); } - return Status::Invalid("CompoundPredicate only support And/Or function"); + return std::optional(Evaluation{std::move(compound_result), + std::move(contributing_field_ids), + std::move(declined_field_ids)}); } } // namespace paimon diff --git a/src/paimon/core/global_index/global_index_evaluator_impl.h b/src/paimon/core/global_index/global_index_evaluator_impl.h index 7c1591870..d967efe6c 100644 --- a/src/paimon/core/global_index/global_index_evaluator_impl.h +++ b/src/paimon/core/global_index/global_index_evaluator_impl.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,19 @@ class GlobalIndexEvaluatorImpl : public GlobalIndexEvaluator { using IndexReadersCreator = std::function>>(int32_t)>; + /// The rows that the indexes matched for a predicate, aligned with Java + /// `GlobalIndexEvaluator.Evaluation`. + struct Evaluation { + std::shared_ptr result; + /// The ids of the fields whose indexes decided `result`. Unlike Java, a field of + /// `declined_field_ids` does not count even if another of its predicates was evaluated, as + /// `result` is then only a superset of the rows matching that field. + std::set contributing_field_ids; + /// The ids of the fields of the conjuncts that were dropped, at any depth of the predicate, + /// because no index could evaluate them. + std::set declined_field_ids; + }; + GlobalIndexEvaluatorImpl(const std::shared_ptr& table_schema, IndexReadersCreator create_index_readers) : table_schema_(table_schema), create_index_readers_(std::move(create_index_readers)) {} @@ -46,20 +60,29 @@ class GlobalIndexEvaluatorImpl : public GlobalIndexEvaluator { Result> Evaluate( const std::shared_ptr& predicate) override; + /// Evaluates `predicate` as `Evaluate()` does, and reports the fields that decided the result. + /// @return The evaluation, or nullopt if the indexes cannot evaluate the predicate. + Result> EvaluateWithContributingFields( + const std::shared_ptr& predicate); + + /// Returns the ids of the fields of `table_schema` that `predicate` refers to, as Java + /// `PredicateVisitor#collectFieldIds` does; empty for a null predicate. + static Result> CollectFieldIds(const TableSchema& table_schema, + const std::shared_ptr& predicate); + /// Applies Java-compatible compound flattening and removes redundant IS NOT NULL /// predicates from AND expressions. static Result> NormalizePredicate( const std::shared_ptr& predicate); private: - Result> EvaluatePredicate( + Result> EvaluatePredicate( const std::shared_ptr& predicate); - Result> EvaluateCompoundPredicate( + Result> EvaluateCompoundPredicate( const std::shared_ptr& compound_predicate); - Result>> GetIndexReaders( - const std::string& field_name); + Result>> GetIndexReaders(int32_t field_id); private: std::shared_ptr table_schema_; diff --git a/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp b/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp index 550509368..b33ca72aa 100644 --- a/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp +++ b/src/paimon/core/global_index/global_index_evaluator_impl_test.cpp @@ -166,9 +166,10 @@ class GlobalIndexEvaluatorImplTest : public ::testing::Test { std::vector fields = { DataField(0, arrow::field("a", arrow::int64())), DataField(1, arrow::field("b", arrow::int64())), + DataField(2, arrow::field("c", arrow::int64())), }; table_schema_ = std::make_shared( - /*version=*/1, /*id=*/0, fields, /*highest_field_id=*/1, + /*version=*/1, /*id=*/0, fields, /*highest_field_id=*/2, /*partition_keys=*/std::vector(), /*primary_keys=*/std::vector(), /*options=*/std::map(), /*comment=*/std::nullopt, @@ -181,7 +182,7 @@ class GlobalIndexEvaluatorImplTest : public ::testing::Test { return GlobalIndexEvaluatorImpl( table_schema_, [reader](int32_t field_id) -> Result>> { - if (field_id == 0) { + if (field_id == 0 || field_id == 2) { return std::vector>{reader}; } return std::vector>(); @@ -222,6 +223,104 @@ TEST_F(GlobalIndexEvaluatorImplTest, SupportedAndUnsupportedLeavesCombineSafely) ASSERT_EQ(nullptr, or_result); } +TEST_F(GlobalIndexEvaluatorImplTest, ReportsContributingFields) { + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + std::shared_ptr indexed = Equal("a", 0); + std::shared_ptr unindexed = Equal("b", 1); + + ASSERT_OK_AND_ASSIGN(std::optional evaluation, + evaluator.EvaluateWithContributingFields(nullptr)); + ASSERT_FALSE(evaluation); + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(unindexed)); + ASSERT_FALSE(evaluation); + + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(indexed)); + ASSERT_TRUE(evaluation); + ASSERT_EQ((std::set{1, 3}), CollectPositions(evaluation->result)); + ASSERT_EQ((std::set{0}), evaluation->contributing_field_ids); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr and_predicate, + PredicateBuilder::And({indexed, unindexed})); + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(and_predicate)); + ASSERT_TRUE(evaluation); + ASSERT_EQ((std::set{1, 3}), CollectPositions(evaluation->result)); + ASSERT_EQ((std::set{0}), evaluation->contributing_field_ids); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr or_predicate, + PredicateBuilder::Or({indexed, unindexed})); + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(or_predicate)); + ASSERT_FALSE(evaluation); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr discarded_or, + PredicateBuilder::And({or_predicate, Equal("c", 2)})); + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(discarded_or)); + ASSERT_TRUE(evaluation); + ASSERT_EQ((std::set{1, 3}), CollectPositions(evaluation->result)); + ASSERT_EQ((std::set{2}), evaluation->contributing_field_ids); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr unvisited_or, + PredicateBuilder::Or({unindexed, Equal("missing", 0)})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr discarded_unvisited_or, + PredicateBuilder::And({unvisited_or, indexed})); + ASSERT_OK_AND_ASSIGN(evaluation, + evaluator.EvaluateWithContributingFields(discarded_unvisited_or)); + ASSERT_TRUE(evaluation); + ASSERT_EQ((std::set{1, 3}), CollectPositions(evaluation->result)); + ASSERT_EQ((std::set{0}), evaluation->contributing_field_ids); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr indexed_or, + PredicateBuilder::Or({indexed, PredicateBuilder::IsNull(0, "a", FieldType::BIGINT)})); + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(indexed_or)); + ASSERT_TRUE(evaluation); + ASSERT_EQ((std::set{1, 2, 3}), CollectPositions(evaluation->result)); + ASSERT_EQ((std::set{0}), evaluation->contributing_field_ids); + + std::shared_ptr declined = + PredicateBuilder::NotEqual(0, "a", FieldType::BIGINT, Literal(static_cast(1))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr partly_declined, + PredicateBuilder::And({indexed, declined})); + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(partly_declined)); + ASSERT_TRUE(evaluation); + ASSERT_EQ((std::set{1, 3}), CollectPositions(evaluation->result)); + ASSERT_TRUE(evaluation->contributing_field_ids.empty()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr nested_declined, + PredicateBuilder::Or({indexed, partly_declined})); + ASSERT_OK_AND_ASSIGN(evaluation, evaluator.EvaluateWithContributingFields(nested_declined)); + ASSERT_TRUE(evaluation); + ASSERT_EQ((std::set{1, 3}), CollectPositions(evaluation->result)); + ASSERT_TRUE(evaluation->contributing_field_ids.empty()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, AndWithEmptyResultShortCircuits) { + GlobalIndexEvaluatorImpl evaluator = CreateEvaluator(); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr predicate, + PredicateBuilder::And( + {Equal("a", 0), PredicateBuilder::IsNull(0, "a", FieldType::BIGINT), Equal("c", 2)})); + ASSERT_OK_AND_ASSIGN(std::optional evaluation, + evaluator.EvaluateWithContributingFields(predicate)); + ASSERT_TRUE(evaluation); + ASSERT_TRUE(CollectPositions(evaluation->result).empty()); + ASSERT_EQ((std::set{0}), evaluation->contributing_field_ids); + ASSERT_EQ(1, reader_->EqualCalls()); +} + +TEST_F(GlobalIndexEvaluatorImplTest, CollectFieldIds) { + ASSERT_OK_AND_ASSIGN(std::set field_ids, + GlobalIndexEvaluatorImpl::CollectFieldIds(*table_schema_, nullptr)); + ASSERT_TRUE(field_ids.empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr predicate, + PredicateBuilder::Or({Equal("a", 0), Equal("b", 1)})); + ASSERT_OK_AND_ASSIGN(field_ids, + GlobalIndexEvaluatorImpl::CollectFieldIds(*table_schema_, predicate)); + ASSERT_EQ((std::set{0, 1}), field_ids); + ASSERT_OK_AND_ASSIGN( + field_ids, GlobalIndexEvaluatorImpl::CollectFieldIds(*table_schema_, Equal("missing", 0))); + ASSERT_TRUE(field_ids.empty()); +} + TEST_F(GlobalIndexEvaluatorImplTest, NormalizationFlattensNestedCompounds) { std::shared_ptr first = Equal("a", 0); std::shared_ptr second = PredicateBuilder::IsNull(0, "a", FieldType::BIGINT); diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index 8cc1c77a9..eebd1a9db 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -144,6 +144,27 @@ Result> GlobalIndexScanImpl::Create( std::move(live_rows_supplier), final_executor, pool)); } +std::unique_ptr GlobalIndexScanImpl::CreateForScalarFilters( + const std::shared_ptr& table_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::vector>& index_files, + const std::shared_ptr& executor, const std::shared_ptr& pool) { + IndexMetaMap index_metas; + for (const auto& index_file : index_files) { + const auto& index_meta = index_file->GetGlobalIndexMeta(); + assert(index_meta); + if (index_meta->extra_field_ids && !index_meta->extra_field_ids->empty()) { + continue; + } + Range range(index_meta->row_range_start, index_meta->row_range_end); + index_metas[index_meta->index_field_id][index_file->IndexType()][range].push_back( + index_file); + } + return std::unique_ptr( + new GlobalIndexScanImpl(table_schema, options, path_factory, std::move(index_metas), + LiveRowGlobalIndexReader::LiveRowsSupplier(), executor, pool)); +} + Result> GlobalIndexScanImpl::GetOrCreateIndexEvaluator() { if (evaluator_) { return evaluator_; @@ -291,4 +312,25 @@ Result> GlobalIndexScanImpl::Scan( return evaluator->Evaluate(predicate); } +Result> GlobalIndexScanImpl::ScanWithCoverage( + const std::shared_ptr& predicate) const { + GlobalIndexEvaluatorImpl evaluator( + table_schema_, + [this](int32_t field_id) -> Result>> { + PAIMON_ASSIGN_OR_RAISE(std::vector> readers, + CreateReaders(field_id, /*row_range_index=*/std::nullopt)); + if (readers.size() <= 1) { + return readers; + } + // Each reader already fans out on `executor_`, so the union stays on this thread and + // no task of `executor_` waits for another one. + std::vector> union_reader = { + std::make_shared(std::move(readers), std::vector(), + std::vector>(), + /*executor=*/nullptr)}; + return union_reader; + }); + return evaluator.EvaluateWithContributingFields(predicate); +} + } // namespace paimon diff --git a/src/paimon/core/global_index/global_index_scan_impl.h b/src/paimon/core/global_index/global_index_scan_impl.h index 364228b53..54753045d 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.h +++ b/src/paimon/core/global_index/global_index_scan_impl.h @@ -28,6 +28,7 @@ #include "paimon/common/predicate/predicate_filter.h" #include "paimon/core/core_options.h" #include "paimon/core/global_index/global_index_evaluator.h" +#include "paimon/core/global_index/global_index_evaluator_impl.h" #include "paimon/core/global_index/global_index_file_manager.h" #include "paimon/core/global_index/live_row_global_index_reader.h" #include "paimon/core/index/index_file_meta.h" @@ -46,6 +47,15 @@ class GlobalIndexScanImpl : public GlobalIndexScan { const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool); + /// Creates a scan of `index_files` alone, to pre-filter the rows of a search, as Java + /// `DataEvolutionGlobalIndexScanner#createForScalarFilters` does. Files of multi-field indexes + /// are left out, as the coverage of a filter is decided per column. + static std::unique_ptr CreateForScalarFilters( + const std::shared_ptr& table_schema, const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::vector>& index_files, + const std::shared_ptr& executor, const std::shared_ptr& pool); + /// Returns `executor` if it is not null, or else creates an executor with /// `global-index.thread-num` threads, which defaults to the number of CPUs. static Result> GetOrCreateExecutor( @@ -53,6 +63,14 @@ class GlobalIndexScanImpl : public GlobalIndexScan { Result> Scan(const std::shared_ptr& predicate); + /// Evaluates `predicate` and reports the fields that decided the result, as Java + /// `DataEvolutionGlobalIndexScanner#scanWithCoverage` does. Unlike `Scan()`, the indexes of a + /// field are read as one union: indexes of different types may cover different rows, so + /// intersecting them would drop the rows that only one of them covers. + /// @return The evaluation, or nullopt if the indexes cannot evaluate the predicate. + Result> ScanWithCoverage( + const std::shared_ptr& predicate) const; + Result>> CreateReaders( const std::string& field_name, const std::optional& row_range_index) const override; diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.cpp b/src/paimon/core/table/source/data_evolution_full_text_read.cpp index e73f4c428..1725cbfe5 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_read.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -35,22 +36,31 @@ #include "paimon/common/global_index/scored_global_index_result_utils.h" #include "paimon/common/utils/arrow/status_utils.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/global_index/global_index_evaluator_impl.h" #include "paimon/core/global_index/global_index_live_row_filter.h" #include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/snapshot.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" +#include "paimon/core/table/source/filtered_row_id_reader.h" #include "paimon/core/tag/tag.h" #include "paimon/core/utils/branch_manager.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/core/utils/tag_manager.h" +#include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" #include "paimon/global_index/global_index_io_meta.h" #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/global_indexer_factory.h" +#include "paimon/logging.h" #include "paimon/predicate/full_text_search.h" namespace paimon { namespace { +Logger* FullTextReadLogger() { + static std::unique_ptr logger = Logger::GetLogger("DataEvolutionFullTextRead"); + return logger.get(); +} + /// The widest span of search row ranges that a split expands into a bitmap to intersect with the /// live rows: at most 65536 run containers, cheaper than copying the live rows of a large table for /// every split. A wider span trims the live rows instead. @@ -95,6 +105,20 @@ Result LoadPlannedSnapshot(const CoreOptions& options, const std::stri snapshot_id)); } +Result ToBitmap(const std::shared_ptr& result) { + if (auto bitmap_result = std::dynamic_pointer_cast(result)) { + PAIMON_ASSIGN_OR_RAISE(const RoaringBitmap64* bitmap, bitmap_result->GetBitmap()); + return *bitmap; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iterator, + result->CreateIterator()); + RoaringBitmap64 bitmap; + while (iterator->HasNext()) { + bitmap.Add(iterator->Next()); + } + return bitmap; +} + /// Describes `split` for the errors of its search. std::string Describe(const IndexFullTextSearchSplitImpl& split) { std::vector file_names; @@ -112,15 +136,19 @@ std::string Describe(const IndexFullTextSearchSplitImpl& split) { DataEvolutionFullTextRead::DataEvolutionFullTextRead( const std::string& root_path, const std::shared_ptr& table_schema, - const CoreOptions& options, const std::shared_ptr& index_file_manager, - const std::shared_ptr& partition_filter, const DataField& text_field, - const std::string& query, int32_t limit, const std::shared_ptr& executor, + const CoreOptions& options, const std::shared_ptr& index_path_factory, + const std::shared_ptr& index_file_manager, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, const DataField& text_field, const std::string& query, + int32_t limit, const std::shared_ptr& executor, const std::shared_ptr& pool) : root_path_(root_path), table_schema_(table_schema), options_(options), + index_path_factory_(index_path_factory), index_file_manager_(index_file_manager), partition_filter_(partition_filter), + filter_(filter), text_field_(text_field), query_(query), limit_(limit), @@ -130,36 +158,38 @@ DataEvolutionFullTextRead::DataEvolutionFullTextRead( Result> DataEvolutionFullTextRead::Create( const std::string& root_path, const std::shared_ptr& table_schema, const CoreOptions& options, const std::shared_ptr& path_factory, - const std::shared_ptr& partition_filter, const DataField& text_field, - const std::string& query, int32_t limit, const std::shared_ptr& executor, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, const DataField& text_field, const std::string& query, + int32_t limit, const std::shared_ptr& executor, const std::shared_ptr& pool) { if (limit <= 0) { return Status::Invalid("Limit must be positive, set via SetLimit()"); } PAIMON_ASSIGN_OR_RAISE(std::shared_ptr final_executor, GlobalIndexScanImpl::GetOrCreateExecutor(executor, options)); + std::shared_ptr index_path_factory = + path_factory->CreateGlobalIndexFileFactory(); auto index_file_manager = std::make_shared( - options.GetFileSystem(), path_factory->CreateGlobalIndexFileFactory(), - /*checkpoint_path_factory=*/nullptr); + options.GetFileSystem(), index_path_factory, /*checkpoint_path_factory=*/nullptr); return std::unique_ptr(new DataEvolutionFullTextRead( - root_path, table_schema, options, index_file_manager, partition_filter, text_field, query, - limit, final_executor, pool)); + root_path, table_schema, options, index_path_factory, index_file_manager, partition_filter, + filter, text_field, query, limit, final_executor, pool)); } std::optional DataEvolutionFullTextRead::IncludeRowIds( - const IndexFullTextSearchSplit& split, const RoaringBitmap64* live_rows) { + const IndexFullTextSearchSplit& split, const RoaringBitmap64* eligible_rows) { const std::vector& search_ranges = split.SearchRowRanges(); - if (search_ranges.empty() || (live_rows && live_rows->IsEmpty())) { + if (search_ranges.empty() || (eligible_rows && eligible_rows->IsEmpty())) { return RoaringBitmap64(); } Range physical_range(split.RowRangeStart(), split.RowRangeEnd()); - if (!live_rows && search_ranges.size() == 1 && search_ranges[0] == physical_range) { + if (!eligible_rows && search_ranges.size() == 1 && search_ranges[0] == physical_range) { return std::nullopt; } Range searched_span(search_ranges.front().from, search_ranges.back().to); RoaringBitmap64 include_row_ids; - if (live_rows && searched_span.Count() > kMaxExpandedSearchRows) { - include_row_ids = GlobalIndexUtils::RowIdsWithin(*live_rows, searched_span); + if (eligible_rows && searched_span.Count() > kMaxExpandedSearchRows) { + include_row_ids = GlobalIndexUtils::RowIdsWithin(*eligible_rows, searched_span); for (size_t i = 1; i < search_ranges.size(); ++i) { include_row_ids.RemoveRange(search_ranges[i - 1].to + 1, search_ranges[i].from); } @@ -167,11 +197,11 @@ std::optional DataEvolutionFullTextRead::IncludeRowIds( for (const auto& range : search_ranges) { include_row_ids |= GlobalIndexUtils::RowIdsOf(range); } - if (live_rows) { - include_row_ids &= *live_rows; + if (eligible_rows) { + include_row_ids &= *eligible_rows; } } - if (live_rows && include_row_ids.Cardinality() == physical_range.Count()) { + if (eligible_rows && include_row_ids.Cardinality() == physical_range.Count()) { return std::nullopt; } return include_row_ids; @@ -190,6 +220,80 @@ Result> DataEvolutionFullTextRead::LiveRows( partition_filter_, options_, executor_, pool_); } +Result> DataEvolutionFullTextRead::MatchedRows( + const std::vector>& index_splits, + int64_t snapshot_id, const std::optional& live_rows) const { + if (!filter_) { + return std::optional(); + } + // Keyed by file name: a scalar index file may intersect more than one split. + std::map> files_by_name; + for (const auto& split : index_splits) { + for (const auto& file : split->ScalarIndexFiles()) { + files_by_name.emplace(file->FileName(), file); + } + } + std::vector> scalar_index_files; + scalar_index_files.reserve(files_by_name.size()); + for (const auto& [file_name, file] : files_by_name) { + scalar_index_files.push_back(file); + } + std::unique_ptr index_scan = GlobalIndexScanImpl::CreateForScalarFilters( + table_schema_, options_, index_path_factory_, scalar_index_files, executor_, pool_); + PAIMON_ASSIGN_OR_RAISE(std::optional evaluation, + index_scan->ScanWithCoverage(filter_)); + if (!evaluation) { + PAIMON_LOG_WARN(FullTextReadLogger(), + "Full-text search on table %s in snapshot %ld has a row filter %s that no " + "scalar global index can evaluate; indexed rows are excluded from the " + "result because %s is fast. Build a scalar index on the filtered columns.", + root_path_.c_str(), snapshot_id, filter_->ToString().c_str(), + Options::SCALAR_INDEX_SEARCH_MODE); + return std::optional(RoaringBitmap64()); + } + + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 index_rows, ToBitmap(evaluation->result)); + if (live_rows) { + index_rows &= live_rows.value(); + } + PAIMON_ASSIGN_OR_RAISE( + bool exact, + FilteredRowIdReader::IsExact(*table_schema_, filter_, evaluation->contributing_field_ids)); + if (exact) { + return std::optional(std::move(index_rows)); + } + + // Only the candidates that a split ranks are worth verifying. + RoaringBitmap64 candidates; + for (const auto& split : index_splits) { + std::optional split_rows = IncludeRowIds(*split, &index_rows); + if (split_rows) { + candidates |= split_rows.value(); + } else { + candidates |= + GlobalIndexUtils::RowIdsOf(Range(split->RowRangeStart(), split->RowRangeEnd())); + } + } + if (candidates.IsEmpty()) { + return std::optional(std::move(candidates)); + } + if (!options_.GlobalIndexFilterRefineFromData()) { + PAIMON_LOG_WARN(FullTextReadLogger(), + "The scalar global index can only answer the row filter %s on table %s " + "in snapshot %ld with candidates, and %s is false, so those %ld rows " + "are excluded from the search; the result may hold fewer rows than " + "requested. Set the option to true to verify the candidates against the " + "data, or build an index that answers the predicate exactly.", + filter_->ToString().c_str(), root_path_.c_str(), snapshot_id, + Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, candidates.Cardinality()); + return std::optional(RoaringBitmap64()); + } + FilteredRowIdReader row_id_reader(root_path_, table_schema_, options_, partition_filter_, + filter_, snapshot_id, executor_, pool_); + PAIMON_ASSIGN_OR_RAISE(RoaringBitmap64 matched_rows, row_id_reader.MatchingRowIds(candidates)); + return std::optional(std::move(matched_rows)); +} + Result> DataEvolutionFullTextRead::SearchSplit( const IndexFullTextSearchSplitImpl& split, const std::optional& include_row_ids) const { @@ -288,12 +392,16 @@ Result> DataEvolutionFullTextRead::Read // `executor_` never waits for it from one of its own threads. They are pinned to the snapshot // the splits were planned from (apache/paimon#9031). PAIMON_ASSIGN_OR_RAISE(std::optional live_rows, LiveRows(snapshot_id.value())); + // Likewise for the row filter, which reads indexes and possibly data. + PAIMON_ASSIGN_OR_RAISE(std::optional matched_rows, + MatchedRows(index_splits, snapshot_id.value(), live_rows)); + const std::optional& eligible_rows = matched_rows ? matched_rows : live_rows; using SearchResult = Result>; std::vector> searches; searches.reserve(index_splits.size()); for (const auto& index_split : index_splits) { auto include_row_ids = std::make_shared>( - IncludeRowIds(*index_split, live_rows ? &live_rows.value() : nullptr)); + IncludeRowIds(*index_split, eligible_rows ? &eligible_rows.value() : nullptr)); if (include_row_ids->has_value() && include_row_ids->value().IsEmpty()) { continue; } diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.h b/src/paimon/core/table/source/data_evolution_full_text_read.h index b0c27d74c..62e1a99b4 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read.h +++ b/src/paimon/core/table/source/data_evolution_full_text_read.h @@ -29,11 +29,13 @@ #include "paimon/common/types/data_field.h" #include "paimon/core/core_options.h" #include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/index/index_path_factory.h" #include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/executor.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate.h" #include "paimon/table/source/full_text_read.h" #include "paimon/utils/roaring_bitmap64.h" @@ -45,18 +47,20 @@ class DataEvolutionFullTextRead : public FullTextRead { /// @param table_schema Schema of the table. /// @param options Options of the table, merged with those of the search. /// @param path_factory Path factory of the table. - /// @param partition_filter Predicate over the partition fields used to read live rows, or null - /// for all partitions. + /// @param partition_filter Predicate over the partition fields used to read live rows and + /// candidate rows, or null for all partitions. + /// @param filter Row filter beyond the partition filter, or null for all rows. /// @param text_field The searched column. /// @param query The JSON query. /// @param limit The number of rows to return; must be positive. /// @param executor Executor to search splits on, or null to create one with /// `global-index.thread-num` threads. - /// @param pool Memory pool for reading indexes and manifests. + /// @param pool Memory pool for reading indexes, manifests and candidate rows. static Result> Create( const std::string& root_path, const std::shared_ptr& table_schema, const CoreOptions& options, const std::shared_ptr& path_factory, - const std::shared_ptr& partition_filter, const DataField& text_field, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, const DataField& text_field, const std::string& query, int32_t limit, const std::shared_ptr& executor, const std::shared_ptr& pool); @@ -66,26 +70,39 @@ class DataEvolutionFullTextRead : public FullTextRead { const std::vector>& splits) override; /// Returns the row ids that `split` ranks, as Java `DataEvolutionFullTextRead#includeRowIds` - /// does: its search row ranges, intersected with `live_rows` if not null. Returns nullopt if - /// they are every row of the physical range of the split, so the index needs no filter. A - /// search span wider than 2^32 rows trims `live_rows` to the ranges instead of expanding them, - /// so that a wide shard costs no more than the live rows. + /// does: its search row ranges, intersected with `eligible_rows` if not null, which are the + /// rows that are live and pass the row filter. Returns nullopt if they are every row of the + /// physical range of the split, so the index needs no filter. A search span wider than 2^32 + /// rows trims `eligible_rows` to the ranges instead of expanding them, so that a wide shard + /// costs no more than the eligible rows. static std::optional IncludeRowIds(const IndexFullTextSearchSplit& split, - const RoaringBitmap64* live_rows); + const RoaringBitmap64* eligible_rows); private: DataEvolutionFullTextRead(const std::string& root_path, const std::shared_ptr& table_schema, const CoreOptions& options, + const std::shared_ptr& index_path_factory, const std::shared_ptr& index_file_manager, const std::shared_ptr& partition_filter, - const DataField& text_field, const std::string& query, int32_t limit, + const std::shared_ptr& filter, const DataField& text_field, + const std::string& query, int32_t limit, const std::shared_ptr& executor, const std::shared_ptr& pool); /// Returns the rows live in snapshot `snapshot_id`, or nullopt if every row counts as live. Result> LiveRows(int64_t snapshot_id) const; + /// Resolves the row filter into the rows of `index_splits` that satisfy it, as Java + /// `DataEvolutionFullTextRead#matchedRows` does in the `fast` `scalar-index.search-mode`, the + /// only one supported so far: rows that no scalar index can evaluate are excluded. Candidates + /// that the indexes cannot decide exactly are verified against the data with + /// `global-index.filter.refine-from-data`, and excluded without it. + /// @return The matched rows, all of them live, or nullopt if there is no row filter. + Result> MatchedRows( + const std::vector>& index_splits, + int64_t snapshot_id, const std::optional& live_rows) const; + /// Searches the index shard of `split` among `include_row_ids`, or all of its rows if not set, /// and returns at most `limit_` rows with global row ids. Result> SearchSplit( @@ -95,8 +112,10 @@ class DataEvolutionFullTextRead : public FullTextRead { std::string root_path_; std::shared_ptr table_schema_; CoreOptions options_; + std::shared_ptr index_path_factory_; std::shared_ptr index_file_manager_; std::shared_ptr partition_filter_; + std::shared_ptr filter_; DataField text_field_; std::string query_; int32_t limit_; diff --git a/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp index 193ae1ab2..8e84aa1e5 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp @@ -49,6 +49,9 @@ #include "paimon/global_index/bitmap_scored_global_index_result.h" #include "paimon/global_index/global_indexer.h" #include "paimon/global_index/global_indexer_factory.h" +#include "paimon/predicate/full_text_match.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" #include "paimon/table/source/full_text_search_builder.h" #include "paimon/testing/mock/mock_global_index_reader.h" #include "paimon/testing/utils/testharness.h" @@ -57,6 +60,8 @@ namespace paimon::test { namespace { constexpr char kFakeIndexType[] = "fake-full-text"; +constexpr char kFakeScalarIndexType[] = "fake-scalar"; +constexpr char kOtherFakeScalarIndexType[] = "other-fake-scalar"; constexpr char kQuery[] = R"({"match":{"query":"paimon"}})"; struct RecordedSearch { @@ -79,6 +84,7 @@ class FakeIndexState { std::lock_guard lock(mutex_); scores_.clear(); searches_.clear(); + scalar_rows_.clear(); } void SetScores(const std::string& file_name, std::vector scores) { @@ -95,6 +101,20 @@ class FakeIndexState { return iter->second; } + void SetScalarRows(const std::string& file_name, const std::vector& rows) { + std::lock_guard lock(mutex_); + scalar_rows_[file_name] = rows; + } + + std::optional> ScalarRows(const std::string& file_name) const { + std::lock_guard lock(mutex_); + auto iter = scalar_rows_.find(file_name); + if (iter == scalar_rows_.end()) { + return std::nullopt; + } + return iter->second; + } + void Record(RecordedSearch search) { std::lock_guard lock(mutex_); searches_[search.file_name] = std::move(search); @@ -109,6 +129,7 @@ class FakeIndexState { mutable std::mutex mutex_; std::map> scores_; std::map searches_; + std::map> scalar_rows_; }; class FakeFullTextReader : public MockGlobalIndexReader { @@ -212,6 +233,60 @@ class FakeFullTextIndexerFactory : public GlobalIndexerFactory { }; REGISTER_PAIMON_FACTORY(FakeFullTextIndexerFactory); +class FakeScalarIndexer : public GlobalIndexer { + public: + Result>> GetExtraFieldNames() const override { + return std::optional>(); + } + + Result> CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const override { + return Status::NotImplemented("fake scalar index cannot be written"); + } + + Result> CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, + const std::shared_ptr& pool) const override { + if (files.size() != 1) { + return Status::Invalid("fake scalar index expects one file per shard"); + } + auto reader = std::make_shared(/*supports_search=*/false); + std::optional> rows = + FakeIndexState::Instance().ScalarRows(PathUtil::GetName(files[0].file_path)); + if (rows) { + reader->SetDefaultResult(rows.value()); + } else { + reader->SetReturnNullptr(); + } + return std::shared_ptr(std::move(reader)); + } +}; + +class FakeScalarIndexerFactory : public GlobalIndexerFactory { + public: + const char* Identifier() const override { + return "fake-scalar-global"; + } + + Result> Create( + const std::map& options) const override { + return std::make_unique(); + } +}; +REGISTER_PAIMON_FACTORY(FakeScalarIndexerFactory); + +class OtherFakeScalarIndexerFactory : public FakeScalarIndexerFactory { + public: + const char* Identifier() const override { + return "other-fake-scalar-global"; + } +}; +REGISTER_PAIMON_FACTORY(OtherFakeScalarIndexerFactory); + using ScoredRows = std::vector>; ScoredRows ToRows(const ScoredGlobalIndexResult& result) { @@ -251,7 +326,8 @@ class DataEvolutionFullTextReadTest : public ::testing::Test { const std::string& file_name, const Range& range, const std::vector& scores, const std::vector& search_row_ranges, int32_t index_field_id = 1, const std::optional>& extra_field_ids = std::nullopt, - int64_t snapshot_id = 1, const std::string& index_type = kFakeIndexType) const { + int64_t snapshot_id = 1, const std::string& index_type = kFakeIndexType, + const std::vector>& scalar_index_files = {}) const { FakeIndexState::Instance().SetScores(file_name, scores); GlobalIndexMeta meta(range.from, range.to, index_field_id, extra_field_ids, /*index_meta=*/nullptr); @@ -259,10 +335,48 @@ class DataEvolutionFullTextReadTest : public ::testing::Test { index_type, file_name, /*file_size=*/128, /*row_count=*/range.Count(), /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); return IndexFullTextSearchSplitImpl::Create(snapshot_id, "content", range.from, range.to, - {file}, search_row_ranges) + {file}, search_row_ranges, scalar_index_files) .value(); } + std::shared_ptr NewFilteredSplit( + const std::string& file_name, const Range& range, const std::vector& scores, + const std::vector>& scalar_index_files) const { + return NewSplit(file_name, range, scores, {range}, /*index_field_id=*/1, + /*extra_field_ids=*/std::nullopt, /*snapshot_id=*/1, kFakeIndexType, + scalar_index_files); + } + + static std::shared_ptr NewScalarFile( + const std::string& file_name, const Range& range, int32_t index_field_id, + const std::string& index_type = kFakeScalarIndexType, + const std::optional>& extra_field_ids = std::nullopt) { + GlobalIndexMeta meta(range.from, range.to, index_field_id, extra_field_ids, + /*index_meta=*/nullptr); + return std::make_shared( + index_type, file_name, /*file_size=*/128, /*row_count=*/range.Count(), + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); + } + + static std::shared_ptr IdEquals(int32_t id) { + return PredicateBuilder::Equal(/*field_index=*/0, "id", FieldType::INT, Literal(id)); + } + + static std::shared_ptr TitleEquals(const std::string& title) { + return PredicateBuilder::Equal(/*field_index=*/2, "title", FieldType::STRING, + Literal(FieldType::STRING, title.data(), title.size())); + } + + Result ReadWithFilter( + const std::vector>& splits, + const std::shared_ptr& filter, int32_t limit = 10) const { + std::unique_ptr builder = NewBuilder(limit); + builder->WithFilter(filter); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, builder->NewFullTextRead()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, read->Read(splits)); + return ToRows(*result); + } + std::unique_ptr NewBuilder(int32_t limit) const { auto builder = std::make_unique(table_path_); builder->SetQuery("content", kQuery).SetLimit(limit); @@ -312,6 +426,115 @@ TEST_F(DataEvolutionFullTextReadTest, TestRanksOnlySearchRowRanges) { ASSERT_EQ(searches["b.index"].include_row_ids, std::vector({2, 3, 4})); } +TEST_F(DataEvolutionFullTextReadTest, TestFilterRanksOnlyMatchedRows) { + FakeIndexState::Instance().SetScalarRows("id.index", {1, 2, 7}); + auto scalar_file = NewScalarFile("id.index", Range(0, 9), /*index_field_id=*/0); + std::vector> splits = { + NewFilteredSplit("a.index", Range(0, 4), {0.1f, 0.9f, 0.5f, 0.3f, 0.7f}, {scalar_file}), + NewFilteredSplit("b.index", Range(5, 9), {0.8f, 0.2f, 0.6f, 0.4f, 1.0f}, {scalar_file})}; + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ReadWithFilter(splits, IdEquals(1), /*limit=*/2)); + ASSERT_EQ(rows, ScoredRows({{1, 0.9f}, {7, 0.6f}})); + std::map searches = FakeIndexState::Instance().Searches(); + ASSERT_EQ(searches.size(), 2u); + ASSERT_EQ(searches["a.index"].include_row_ids, std::vector({1, 2})); + ASSERT_EQ(searches["b.index"].include_row_ids, std::vector({2})); + ASSERT_EQ(searches["a.index"].limit, 2); + + FakeIndexState::Instance().SetScalarRows("narrow-id.index", {0, 1, 2}); + splits = {NewSplit("narrow.index", Range(0, 4), {0.1f, 0.9f, 0.5f, 0.3f, 0.7f}, {Range(1, 4)}, + /*index_field_id=*/1, /*extra_field_ids=*/std::nullopt, /*snapshot_id=*/1, + kFakeIndexType, + {NewScalarFile("narrow-id.index", Range(0, 4), /*index_field_id=*/0)})}; + ASSERT_OK_AND_ASSIGN(rows, ReadWithFilter(splits, IdEquals(1))); + ASSERT_EQ(rows, ScoredRows({{1, 0.9f}, {2, 0.5f}})); + ASSERT_EQ(FakeIndexState::Instance().Searches()["narrow.index"].include_row_ids, + std::vector({1, 2})); + + FakeIndexState::Instance().SetScalarRows("all-id.index", {0, 1}); + splits = {NewFilteredSplit("all.index", Range(0, 1), {0.1f, 0.9f}, + {NewScalarFile("all-id.index", Range(0, 1), /*index_field_id=*/0)})}; + ASSERT_OK_AND_ASSIGN(rows, ReadWithFilter(splits, IdEquals(1))); + ASSERT_EQ(rows, ScoredRows({{0, 0.1f}, {1, 0.9f}})); + ASSERT_FALSE(FakeIndexState::Instance().Searches()["all.index"].include_row_ids); + + FakeIndexState::Instance().Reset(); + FakeIndexState::Instance().SetScalarRows("none-id.index", {}); + splits = { + NewFilteredSplit("none.index", Range(0, 1), {0.1f, 0.9f}, + {NewScalarFile("none-id.index", Range(0, 1), /*index_field_id=*/0)})}; + ASSERT_OK_AND_ASSIGN(rows, ReadWithFilter(splits, IdEquals(1))); + ASSERT_TRUE(rows.empty()); + ASSERT_TRUE(FakeIndexState::Instance().Searches().empty()); +} + +TEST_F(DataEvolutionFullTextReadTest, TestFilterExcludesRowsThatIndexesCannotDecide) { + std::vector scores = {0.1f, 0.9f}; + FakeIndexState::Instance().SetScalarRows("multi.index", {0, 1}); + std::vector>> unusable_files = { + {}, + {NewScalarFile("declining.index", Range(0, 1), /*index_field_id=*/0)}, + {NewScalarFile("multi.index", Range(0, 1), /*index_field_id=*/0, kFakeScalarIndexType, + std::vector({2}))}}; + for (const auto& scalar_index_files : unusable_files) { + std::vector> splits = { + NewFilteredSplit("a.index", Range(0, 1), scores, scalar_index_files)}; + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ReadWithFilter(splits, IdEquals(1))); + ASSERT_TRUE(rows.empty()); + } + + FakeIndexState::Instance().SetScalarRows("id.index", {0, 1}); + FakeIndexState::Instance().SetScalarRows("title.index", {0, 1}); + std::vector> id_splits = { + NewFilteredSplit("a.index", Range(0, 1), scores, + {NewScalarFile("id.index", Range(0, 1), /*index_field_id=*/0)})}; + std::vector> title_splits = { + NewFilteredSplit("a.index", Range(0, 1), scores, + {NewScalarFile("title.index", Range(0, 1), /*index_field_id=*/2)})}; + + ASSERT_OK_AND_ASSIGN(std::shared_ptr partly_indexed, + PredicateBuilder::And({IdEquals(1), TitleEquals("x")})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ReadWithFilter(id_splits, partly_indexed)); + ASSERT_TRUE(rows.empty()); + + const Literal literal(FieldType::STRING, "x", 1); + ASSERT_OK_AND_ASSIGN(std::shared_ptr contains, + PredicateBuilder::Contains(2, "title", FieldType::STRING, literal)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ends_with, + PredicateBuilder::EndsWith(2, "title", FieldType::STRING, literal)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr like, + PredicateBuilder::Like(2, "title", FieldType::STRING, literal)); + for (const auto& candidate_only : {contains, ends_with, like}) { + ASSERT_OK_AND_ASSIGN(rows, ReadWithFilter(title_splits, candidate_only)); + ASSERT_TRUE(rows.empty()); + } + ASSERT_TRUE(FakeIndexState::Instance().Searches().empty()); + + ASSERT_OK_AND_ASSIGN(rows, ReadWithFilter(title_splits, TitleEquals("x"))); + ASSERT_EQ(rows, ScoredRows({{0, 0.1f}, {1, 0.9f}})); + + FakeIndexState::Instance().SetScalarRows("partial.index", {0, 1}); + std::vector> partly_covered_splits = { + NewFilteredSplit("b.index", Range(0, 2), {0.1f, 0.9f, 0.8f}, + {NewScalarFile("partial.index", Range(0, 1), /*index_field_id=*/0)})}; + ASSERT_OK_AND_ASSIGN(rows, ReadWithFilter(partly_covered_splits, IdEquals(1))); + ASSERT_EQ(rows, ScoredRows({{0, 0.1f}, {1, 0.9f}})); +} + +TEST_F(DataEvolutionFullTextReadTest, TestFilterUnionsIndexesOfDifferentTypes) { + FakeIndexState::Instance().SetScalarRows("first.index", {1}); + FakeIndexState::Instance().SetScalarRows("second.index", {2}); + auto first = NewScalarFile("first.index", Range(0, 4), /*index_field_id=*/0); + auto second = + NewScalarFile("second.index", Range(5, 9), /*index_field_id=*/0, kOtherFakeScalarIndexType); + std::vector> splits = {NewFilteredSplit( + "a.index", Range(0, 9), {0.1f, 0.9f, 0.5f, 0.3f, 0.7f, 0.8f, 0.2f, 0.6f, 0.4f, 1.0f}, + {first, second})}; + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ReadWithFilter(splits, IdEquals(1))); + ASSERT_EQ(rows, ScoredRows({{1, 0.9f}, {7, 0.6f}})); + ASSERT_EQ(FakeIndexState::Instance().Searches()["a.index"].include_row_ids, + std::vector({1, 7})); +} + TEST_F(DataEvolutionFullTextReadTest, TestIncludeRowIds) { auto row_ids_of = [](const std::optional& bitmap) { std::vector row_ids; @@ -325,7 +548,7 @@ TEST_F(DataEvolutionFullTextReadTest, TestIncludeRowIds) { NewSplit("a.index", Range(10, 19), scores, {Range(10, 12), Range(17, 19)})); ASSERT_TRUE(split); std::optional row_ids = - DataEvolutionFullTextRead::IncludeRowIds(*split, /*live_rows=*/nullptr); + DataEvolutionFullTextRead::IncludeRowIds(*split, /*eligible_rows=*/nullptr); ASSERT_TRUE(row_ids); ASSERT_EQ(row_ids_of(row_ids), std::vector({10, 11, 12, 17, 18, 19})); RoaringBitmap64 live_rows = RoaringBitmap64::From({0, 11, 12, 15, 18, 25}); @@ -340,7 +563,7 @@ TEST_F(DataEvolutionFullTextReadTest, TestIncludeRowIds) { auto whole_split = std::dynamic_pointer_cast( NewSplit("b.index", Range(10, 19), scores, {Range(10, 19)})); ASSERT_TRUE(whole_split); - ASSERT_FALSE(DataEvolutionFullTextRead::IncludeRowIds(*whole_split, /*live_rows=*/nullptr)); + ASSERT_FALSE(DataEvolutionFullTextRead::IncludeRowIds(*whole_split, /*eligible_rows=*/nullptr)); RoaringBitmap64 all_rows; all_rows.AddRange(0, 30); ASSERT_FALSE(DataEvolutionFullTextRead::IncludeRowIds(*whole_split, &all_rows)); @@ -354,7 +577,8 @@ TEST_F(DataEvolutionFullTextReadTest, TestIncludeRowIds) { auto widest_split = std::dynamic_pointer_cast( NewSplit("widest.index", Range(1, max_row_id), /*scores=*/{}, {Range(1, max_row_id)})); ASSERT_TRUE(widest_split); - ASSERT_FALSE(DataEvolutionFullTextRead::IncludeRowIds(*widest_split, /*live_rows=*/nullptr)); + ASSERT_FALSE( + DataEvolutionFullTextRead::IncludeRowIds(*widest_split, /*eligible_rows=*/nullptr)); RoaringBitmap64 sparse_live_rows = RoaringBitmap64::From({0, 5, 20, max_row_id}); row_ids = DataEvolutionFullTextRead::IncludeRowIds(*widest_split, &sparse_live_rows); ASSERT_TRUE(row_ids); @@ -481,6 +705,43 @@ TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { FullTextSearchBuilder missing_table(PathUtil::JoinPath(dir_->Str(), "foo.db/missing")); missing_table.SetQuery("content", kQuery).SetLimit(3); ASSERT_NOK_WITH_MSG(missing_table.NewFullTextScan(), "schema file not found"); + + std::unique_ptr missing_field = NewBuilder(3); + missing_field->WithFilter( + PredicateBuilder::Equal(/*field_index=*/0, "missing", FieldType::INT, Literal(1))); + ASSERT_NOK_WITH_MSG(missing_field->NewFullTextScan(), "field missing does not exist in schema"); + ASSERT_NOK_WITH_MSG(missing_field->NewFullTextRead(), "field missing does not exist in schema"); + + std::unique_ptr row_id_field = NewBuilder(3); + row_id_field->WithFilter(PredicateBuilder::Equal( + /*field_index=*/0, "_ROW_ID", FieldType::BIGINT, Literal(static_cast(1)))); + ASSERT_NOK_WITH_MSG(row_id_field->NewFullTextScan(), "field _ROW_ID does not exist in schema"); + + std::unique_ptr wrong_type = NewBuilder(3); + wrong_type->WithFilter(PredicateBuilder::Equal(/*field_index=*/0, "id", FieldType::STRING, + Literal(FieldType::STRING, "x", 1))); + ASSERT_NOK(wrong_type->NewFullTextScan()); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr full_text_match, + PredicateBuilder::FullTextMatch( + /*field_index=*/2, std::make_shared("title", kQuery))); + std::unique_ptr with_match = NewBuilder(3); + with_match->WithFilter(full_text_match); + ASSERT_NOK_WITH_MSG(with_match->NewFullTextScan(), + "FullTextMatch predicate execution is not implemented yet"); + + std::unique_ptr scalar_full_mode = NewBuilder(3); + scalar_full_mode->AddOption(Options::SCALAR_INDEX_SEARCH_MODE, "full"); + ASSERT_OK(scalar_full_mode->NewFullTextScan()); + scalar_full_mode->WithFilter(IdEquals(1)); + ASSERT_NOK_WITH_MSG(scalar_full_mode->NewFullTextScan(), + "full and detail are not supported yet"); + ASSERT_NOK_WITH_MSG(scalar_full_mode->NewFullTextRead(), + "full and detail are not supported yet"); + + std::unique_ptr null_filter = NewBuilder(3); + null_filter->WithFilter(nullptr); + ASSERT_OK(null_filter->NewFullTextScan()); } TEST_F(DataEvolutionFullTextReadTest, TestBuilderRejectsPrimaryKeyFullTextIndex) { diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp index 85b8540e2..25e4af065 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp @@ -34,6 +34,7 @@ #include "fmt/format.h" #include "fmt/ranges.h" +#include "paimon/core/global_index/global_index_evaluator_impl.h" #include "paimon/core/index/index_file_handler.h" #include "paimon/core/manifest/index_manifest_entry.h" #include "paimon/core/manifest/index_manifest_file.h" @@ -132,18 +133,26 @@ struct CandidateRanksBefore { std::tie(right->identity, right->file_range.from, right->file_range.to); } }; + +bool IndexesAnyField(const GlobalIndexMeta& meta, const std::set& field_ids) { + return std::any_of(field_ids.begin(), field_ids.end(), [&meta](int32_t field_id) { + return DataEvolutionFullTextScan::IndexesField(meta, field_id); + }); +} } // namespace DataEvolutionFullTextScan::DataEvolutionFullTextScan( const std::string& root_path, const std::shared_ptr& table_schema, const CoreOptions& options, const std::shared_ptr& path_factory, - const std::shared_ptr& partition_filter, const DataField& text_field, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, const DataField& text_field, const std::shared_ptr& pool) : root_path_(root_path), table_schema_(table_schema), options_(options), path_factory_(path_factory), partition_filter_(partition_filter), + filter_(filter), text_field_(text_field), pool_(pool) {} @@ -156,6 +165,27 @@ bool DataEvolutionFullTextScan::IndexesField(const GlobalIndexMeta& meta, int32_ meta.extra_field_ids->end(); } +bool DataEvolutionFullTextScan::IsScalarPreFilterFile(const GlobalIndexMeta& meta, + const std::set& filter_field_ids, + bool supports_full_text) { + bool has_extra_fields = meta.extra_field_ids && !meta.extra_field_ids->empty(); + return IndexesAnyField(meta, filter_field_ids) && (!supports_full_text || has_extra_fields); +} + +Result>> DataEvolutionFullTextScan::IntersectingFiles( + const std::vector>& index_files, const Range& row_range) { + std::vector> files; + for (const auto& file : index_files) { + // Checked before a `Range` is built from it, as a reversed range breaks its invariant. + PAIMON_RETURN_NOT_OK(IndexFullTextSearchSplitImpl::CheckScalarIndexFile(file)); + const GlobalIndexMeta& meta = file->GetGlobalIndexMeta().value(); + if (Range::HasIntersection(Range(meta.row_range_start, meta.row_range_end), row_range)) { + files.push_back(file); + } + } + return files; +} + std::string DataEvolutionFullTextScan::IndexIdentity(const IndexFileMeta& file) { const std::optional& meta = file.GetGlobalIndexMeta(); assert(meta); @@ -280,6 +310,8 @@ Result> DataEvolutionFullTextScan::CreatePla table_schema_->GetFields(table_schema_->PartitionKeys())); auto partition_schema = DataField::ConvertDataFieldsToArrowSchema(partition_fields); int32_t field_id = text_field_.Id(); + PAIMON_ASSIGN_OR_RAISE(std::set filter_field_ids, + GlobalIndexEvaluatorImpl::CollectFieldIds(*table_schema_, filter_)); std::function(const IndexManifestEntry&)> filter = [&](const IndexManifestEntry& entry) -> Result { if (partition_filter_) { @@ -290,35 +322,48 @@ Result> DataEvolutionFullTextScan::CreatePla } } const std::optional& meta = entry.index_file->GetGlobalIndexMeta(); - return meta && IndexesField(meta.value(), field_id); + return meta && (IndexesField(meta.value(), field_id) || + IndexesAnyField(meta.value(), filter_field_ids)); }; PAIMON_ASSIGN_OR_RAISE(std::vector entries, index_file_handler.Scan(snapshot.value(), filter)); - std::map serves_full_text_search; + std::map> indexers; std::vector> full_text_index_files; + std::vector> scalar_index_files; for (const auto& entry : entries) { const std::string& index_type = entry.index_file->IndexType(); - auto iter = serves_full_text_search.find(index_type); - if (iter == serves_full_text_search.end()) { + const GlobalIndexMeta& meta = entry.index_file->GetGlobalIndexMeta().value(); + auto iter = indexers.find(index_type); + if (iter == indexers.end()) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, GlobalIndexerFactory::Get(index_type, options_.ToMap())); - PAIMON_ASSIGN_OR_RAISE(bool served, ServesFullTextSearch(index_type, indexer.get(), - root_path_, snapshot->Id())); - iter = serves_full_text_search.emplace(index_type, served).first; + iter = indexers.emplace(index_type, std::move(indexer)).first; + } + const GlobalIndexer* indexer = iter->second.get(); + if (IndexesField(meta, field_id)) { + PAIMON_ASSIGN_OR_RAISE( + bool served, ServesFullTextSearch(index_type, indexer, root_path_, snapshot->Id())); + if (served) { + full_text_index_files.push_back(entry.index_file); + } } - if (iter->second) { - full_text_index_files.push_back(entry.index_file); + bool supports_full_text = indexer != nullptr && indexer->SupportsFullTextSearch(); + if (IsScalarPreFilterFile(meta, filter_field_ids, supports_full_text)) { + scalar_index_files.push_back(entry.index_file); } } std::vector> splits; for (auto& selection : ChooseIndexRanges(full_text_index_files, field_id)) { - PAIMON_ASSIGN_OR_RAISE(std::shared_ptr split, - IndexFullTextSearchSplitImpl::Create( - snapshot->Id(), text_field_.Name(), selection.file_range.from, - selection.file_range.to, std::move(selection.files), - std::move(selection.search_ranges))); + PAIMON_ASSIGN_OR_RAISE(std::vector> split_scalar_files, + IntersectingFiles(scalar_index_files, selection.file_range)); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + snapshot->Id(), text_field_.Name(), selection.file_range.from, + selection.file_range.to, std::move(selection.files), + std::move(selection.search_ranges), std::move(split_scalar_files))); splits.push_back(std::move(split)); } return std::make_shared(snapshot->Id(), std::move(splits)); diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.h b/src/paimon/core/table/source/data_evolution_full_text_scan.h index 95cee1d49..eb68b7c98 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan.h +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.h @@ -21,6 +21,7 @@ #include #include +#include #include #include @@ -33,6 +34,7 @@ #include "paimon/core/utils/file_store_path_factory.h" #include "paimon/global_index/global_indexer.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate.h" #include "paimon/table/source/full_text_scan.h" #include "paimon/utils/range.h" @@ -59,6 +61,7 @@ class DataEvolutionFullTextScan : public FullTextScan { /// @param options Options of the table, merged with those of the search. /// @param path_factory Path factory of the table. /// @param partition_filter Predicate over the partition fields, or null for all partitions. + /// @param filter Row filter beyond the partition filter, or null for all rows. /// @param text_field The searched column. /// @param pool Memory pool for reading index manifests. DataEvolutionFullTextScan(const std::string& root_path, @@ -66,7 +69,8 @@ class DataEvolutionFullTextScan : public FullTextScan { const CoreOptions& options, const std::shared_ptr& path_factory, const std::shared_ptr& partition_filter, - const DataField& text_field, const std::shared_ptr& pool); + const std::shared_ptr& filter, const DataField& text_field, + const std::shared_ptr& pool); Result> CreatePlan() override; @@ -74,6 +78,20 @@ class DataEvolutionFullTextScan : public FullTextScan { /// extra fields. static bool IndexesField(const GlobalIndexMeta& meta, int32_t field_id); + /// Returns whether a file of `meta` can pre-filter the rows of a search with a row filter over + /// `filter_field_ids`. It must index one of those fields, and if its indexer supports full-text + /// search, it must have extra fields too. Unlike in Java, this holds whichever column such an + /// index serves, as `lucene-fts` answers every scalar predicate with all of its rows. + static bool IsScalarPreFilterFile(const GlobalIndexMeta& meta, + const std::set& filter_field_ids, + bool supports_full_text); + + /// Returns the files of `index_files` whose row range intersects `row_range`, or `Invalid` if + /// a split cannot carry one of them; see + /// `IndexFullTextSearchSplitImpl::CheckScalarIndexFile()`. + static Result>> IntersectingFiles( + const std::vector>& index_files, const Range& row_range); + /// Returns the identity of the index that `file` belongs to: its index type, indexed field id /// and extra field ids, as `type|index_field_id|[extra_field_ids]`. Files of one identity /// differ only by shard, while files of different identities are never read together. @@ -108,6 +126,7 @@ class DataEvolutionFullTextScan : public FullTextScan { CoreOptions options_; std::shared_ptr path_factory_; std::shared_ptr partition_filter_; + std::shared_ptr filter_; DataField text_field_; std::shared_ptr pool_; }; diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp index d73727f18..b58e4b903 100644 --- a/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp +++ b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp @@ -23,9 +23,12 @@ #include #include #include +#include #include +#include #include +#include "fmt/format.h" #include "gtest/gtest.h" #include "paimon/core/index/global_index_meta.h" #include "paimon/core/index/index_file_meta.h" @@ -165,6 +168,62 @@ TEST(DataEvolutionFullTextScanTest, TestIdentityOrderBreaksTies) { ASSERT_EQ(Choose(full_text_inside), expected); } +TEST(DataEvolutionFullTextScanTest, TestScalarPreFilterFiles) { + const std::set filter_field_ids = {2, 3}; + auto is_pre_filter_file = [&filter_field_ids]( + int32_t index_field_id, + const std::optional>& extra_field_ids, + bool supports_full_text) { + GlobalIndexMeta meta(0, 9, index_field_id, extra_field_ids, /*index_meta=*/nullptr); + return DataEvolutionFullTextScan::IsScalarPreFilterFile(meta, filter_field_ids, + supports_full_text); + }; + ASSERT_TRUE(is_pre_filter_file(2, std::nullopt, /*supports_full_text=*/false)); + ASSERT_TRUE(is_pre_filter_file(1, std::vector({3}), /*supports_full_text=*/false)); + ASSERT_FALSE(is_pre_filter_file(1, std::nullopt, /*supports_full_text=*/false)); + ASSERT_FALSE(is_pre_filter_file(1, std::vector({4}), /*supports_full_text=*/false)); + ASSERT_FALSE(is_pre_filter_file(1, std::nullopt, /*supports_full_text=*/true)); + + ASSERT_FALSE(is_pre_filter_file(2, std::nullopt, /*supports_full_text=*/true)); + ASSERT_FALSE(is_pre_filter_file(2, std::vector(), /*supports_full_text=*/true)); + ASSERT_TRUE(is_pre_filter_file(1, std::vector({2}), /*supports_full_text=*/true)); + + GlobalIndexMeta meta(0, 9, 1, std::nullopt, /*index_meta=*/nullptr); + ASSERT_FALSE(DataEvolutionFullTextScan::IsScalarPreFilterFile(meta, /*filter_field_ids=*/{}, + /*supports_full_text=*/false)); +} + +TEST(DataEvolutionFullTextScanTest, TestIntersectingFiles) { + auto first = NewIndexFile("first.index", "btree", Range(0, 9), /*index_field_id=*/2); + auto second = NewIndexFile("second.index", "btree", Range(10, 19), /*index_field_id=*/2); + auto third = NewIndexFile("third.index", "btree", Range(20, 29), /*index_field_id=*/2); + std::vector> files = {first, second, third}; + ASSERT_OK_AND_ASSIGN(std::vector> intersecting, + DataEvolutionFullTextScan::IntersectingFiles(files, Range(9, 10))); + ASSERT_EQ(intersecting, std::vector>({first, second})); + ASSERT_OK_AND_ASSIGN(intersecting, + DataEvolutionFullTextScan::IntersectingFiles(files, Range(25, 40))); + ASSERT_EQ(intersecting, std::vector>({third})); + ASSERT_OK_AND_ASSIGN(intersecting, + DataEvolutionFullTextScan::IntersectingFiles(files, Range(30, 40))); + ASSERT_TRUE(intersecting.empty()); + + std::vector> invalid_ranges = { + {-1, 9}, {9, 0}, {0, std::numeric_limits::max()}}; + for (const auto& [start, end] : invalid_ranges) { + GlobalIndexMeta meta(start, end, /*index_field_id=*/2, /*extra_field_ids=*/std::nullopt, + /*index_meta=*/nullptr); + auto invalid = std::make_shared( + "btree", "invalid.index", /*file_size=*/128, /*row_count=*/10, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); + std::vector> with_invalid = {first, invalid}; + ASSERT_NOK_WITH_MSG(DataEvolutionFullTextScan::IntersectingFiles(with_invalid, Range(0, 9)), + fmt::format("scalar index file invalid.index covers an invalid row " + "range [{}, {}]", + start, end)); + } +} + TEST(DataEvolutionFullTextScanTest, TestServesFullTextSearch) { for (const std::string index_type : {"full-text", "lucene-fts"}) { ASSERT_NOK_WITH_MSG( diff --git a/src/paimon/core/table/source/filtered_row_id_reader.cpp b/src/paimon/core/table/source/filtered_row_id_reader.cpp new file mode 100644 index 000000000..68adbc0de --- /dev/null +++ b/src/paimon/core/table/source/filtered_row_id_reader.cpp @@ -0,0 +1,248 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/filtered_row_id_reader.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "fmt/format.h" +#include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/checked_cast.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/global_index/global_index_evaluator_impl.h" +#include "paimon/core/operation/internal_read_context.h" +#include "paimon/core/table/source/append_only_table_read.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/defs.h" +#include "paimon/format/file_format.h" +#include "paimon/global_index/bitmap_global_index_result.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/leaf_predicate.h" +#include "paimon/predicate/predicate_utils.h" +#include "paimon/read_context.h" +#include "paimon/reader/batch_reader.h" +#include "paimon/scan_context.h" +#include "paimon/table/source/plan.h" +#include "paimon/table/source/startup_mode.h" +#include "paimon/table/source/table_read.h" +#include "paimon/table/source/table_scan.h" + +namespace paimon { +namespace { +/// Returns whether `predicate` has a leaf that a global index may answer with candidates only. +bool HasCandidateOnlyLeaf(const std::shared_ptr& predicate) { + if (auto compound = std::dynamic_pointer_cast(predicate)) { + const auto& children = compound->Children(); + return std::any_of(children.begin(), children.end(), HasCandidateOnlyLeaf); + } + if (auto leaf = std::dynamic_pointer_cast(predicate)) { + Function::Type type = leaf->GetFunction().GetType(); + return type == Function::Type::CONTAINS || type == Function::Type::ENDS_WITH || + type == Function::Type::LIKE; + } + return false; +} + +/// Returns `options` with the scan pinned to snapshot `snapshot_id`, as Java does. A tag may +/// outlive its snapshot, so once the snapshot has expired a search of a tag reads the tag instead, +/// which the caller must check still refers to the snapshot. +Result> PinToSnapshot(const std::string& root_path, + const CoreOptions& options, + int64_t snapshot_id) { + std::map pinned = options.ToMap(); + pinned[Options::SCAN_MODE] = StartupMode::FromSnapshot().ToString(); + pinned.erase(Options::SCAN_TIMESTAMP); + pinned.erase(Options::SCAN_TIMESTAMP_MILLIS); + if (pinned.count(Options::SCAN_TAG_NAME) > 0) { + SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, + BranchManager::NormalizeBranch(options.GetBranch())); + PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); + if (!exists) { + pinned.erase(Options::SCAN_SNAPSHOT_ID); + return pinned; + } + pinned.erase(Options::SCAN_TAG_NAME); + } + pinned[Options::SCAN_SNAPSHOT_ID] = std::to_string(snapshot_id); + return pinned; +} + +/// Creates a read of `read_context` with `table_schema` rather than the latest schema, as Java +/// `FilteredRowIdReader` reads with the table that the search loaded, so that a column renamed or +/// replaced since then cannot point the filter at another field. +Result> CreateReadWithSchema( + std::unique_ptr read_context, const std::shared_ptr& table_schema, + const std::map& options, const std::shared_ptr& pool, + const std::shared_ptr& executor) { + // Row ids come from row tracking, which a primary-key table does not support. + if (!table_schema->PrimaryKeys().empty()) { + return Status::NotImplemented("cannot evaluate a row filter on a primary-key table"); + } + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr internal_context, + InternalReadContext::Create(std::move(read_context), table_schema, options)); + const CoreOptions& core_options = internal_context->GetCoreOptions(); + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, + core_options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + core_options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + internal_context->GetPath(), arrow_schema, table_schema->PartitionKeys(), + core_options.GetPartitionDefaultName(), core_options.GetFileFormat()->Identifier(), + core_options.DataFilePrefix(), core_options.LegacyPartitionNameEnabled(), + external_paths, global_index_external_path, core_options.IndexFileInDataFileDir(), + pool)); + return std::unique_ptr( + std::make_unique(path_factory, internal_context, pool, executor)); +} +} // namespace + +FilteredRowIdReader::FilteredRowIdReader( + const std::string& root_path, const std::shared_ptr& table_schema, + const CoreOptions& options, const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, int64_t snapshot_id, + const std::shared_ptr& executor, const std::shared_ptr& pool) + : root_path_(root_path), + table_schema_(table_schema), + options_(options), + partition_filter_(partition_filter), + filter_(filter), + snapshot_id_(snapshot_id), + executor_(executor), + pool_(pool) {} + +Result FilteredRowIdReader::IsExact(const TableSchema& table_schema, + const std::shared_ptr& filter, + const std::set& contributing_field_ids) { + PAIMON_ASSIGN_OR_RAISE(std::set filter_field_ids, + GlobalIndexEvaluatorImpl::CollectFieldIds(table_schema, filter)); + if (!std::includes(contributing_field_ids.begin(), contributing_field_ids.end(), + filter_field_ids.begin(), filter_field_ids.end())) { + return false; + } + return !HasCandidateOnlyLeaf(filter); +} + +Result FilteredRowIdReader::MatchingRowIds(const RoaringBitmap64& rows) const { + RoaringBitmap64 matching; + if (rows.IsEmpty()) { + return matching; + } + std::map pinned_options; + PAIMON_ASSIGN_OR_RAISE(pinned_options, PinToSnapshot(root_path_, options_, snapshot_id_)); + + // Unlike the read, the scan may load the latest schema, as it selects files by partition and + // row id alone. + ScanContextBuilder scan_builder(root_path_); + scan_builder.SetOptions(pinned_options) + .SetPredicate(partition_filter_) + .SetGlobalIndexResult(std::make_shared( + [rows]() -> Result { return rows; })) + .WithFileSystem(options_.GetFileSystem()) + .WithMemoryPool(pool_) + .WithExecutor(executor_); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_scan, + TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, table_scan->CreatePlan()); + // An empty plan of another snapshot would drop every candidate, so it is checked too. + if (plan->SnapshotId() != snapshot_id_) { + return Status::Invalid(fmt::format( + "cannot evaluate a row filter on snapshot {}, as the data scan resolved another one", + snapshot_id_)); + } + if (plan->Splits().empty()) { + return matching; + } + + // The read schema decides the field indexes that a predicate of the read must refer to. + std::set filter_fields; + PAIMON_RETURN_NOT_OK(PredicateUtils::GetAllNames(filter_, &filter_fields)); + std::vector read_fields; + for (const DataField& field : table_schema_->Fields()) { + if (filter_fields.count(field.Name()) > 0) { + read_fields.push_back(field.Name()); + } + } + read_fields.push_back(SpecialFields::RowId().Name()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr read_filter, + PredicateUtils::CreatePickedFieldFilter( + filter_, ObjectUtils::CreateIdentifierToIndexMap(read_fields))); + + ReadContextBuilder read_builder(root_path_); + read_builder.SetOptions(pinned_options) + .SetReadFieldNames(read_fields) + .SetPredicate(read_filter) + .EnablePredicateFilter(true) + .WithFileSystem(options_.GetFileSystem()) + .WithMemoryPool(pool_) + .WithExecutor(executor_); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr table_read, + CreateReadWithSchema(std::move(read_context), table_schema_, + pinned_options, pool_, executor_)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + table_read->CreateReader(plan->Splits())); + ScopeGuard guard([&reader]() { reader->Close(); }); + while (true) { + PAIMON_ASSIGN_OR_RAISE(BatchReader::ReadBatch batch, reader->NextBatch()); + if (BatchReader::IsEofBatch(batch)) { + break; + } + auto& [c_array, c_schema] = batch; + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr array, + arrow::ImportArray(c_array.get(), c_schema.get())); + if (array->type_id() != arrow::Type::STRUCT) { + return Status::Invalid("cannot evaluate a row filter on a batch that is not a struct"); + } + std::shared_ptr row_id_array = + checked_pointer_cast(array)->GetFieldByName( + SpecialFields::RowId().Name()); + if (!row_id_array || row_id_array->type_id() != arrow::Type::INT64) { + return Status::Invalid(fmt::format("cannot evaluate a row filter without an int64 {}", + SpecialFields::RowId().Name())); + } + auto row_ids = checked_pointer_cast(row_id_array); + for (int64_t i = 0; i < row_ids->length(); ++i) { + int64_t row_id = row_ids->Value(i); + if (rows.Contains(row_id)) { + matching.Add(row_id); + } + } + } + return matching; +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/filtered_row_id_reader.h b/src/paimon/core/table/source/filtered_row_id_reader.h new file mode 100644 index 000000000..e70d5d26e --- /dev/null +++ b/src/paimon/core/table/source/filtered_row_id_reader.h @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/core/core_options.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/executor.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/predicate/predicate.h" +#include "paimon/result.h" +#include "paimon/utils/roaring_bitmap64.h" + +namespace paimon { +/// Evaluates a row filter on the data of the given row ids and returns those that satisfy it, +/// aligned with Java `FilteredRowIdReader`. Only the filter columns and `_ROW_ID` are read, so this +/// is the exact counterpart of a scalar global index lookup: a search can hand the result to its +/// index as include row ids without touching the index corpus. +class FilteredRowIdReader { + public: + /// @param root_path Root directory of the table. + /// @param table_schema Schema that the search resolved, which the data is read with. + /// @param options Options of the table, merged with those of the search. + /// @param partition_filter Predicate over the partition fields, or null for all partitions. + /// @param filter The row filter; must not be null. + /// @param snapshot_id The snapshot to read. Once it has expired, a `scan.tag-name` in + /// `options` is read instead, as a tag may outlive its snapshot, and + /// must still refer to it. + /// @param executor Executor to read data on. + /// @param pool Memory pool for reading data. + FilteredRowIdReader(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& partition_filter, + const std::shared_ptr& filter, int64_t snapshot_id, + const std::shared_ptr& executor, + const std::shared_ptr& pool); + + /// Returns whether an evaluation of `filter` that `contributing_field_ids` decided is an exact + /// match set. The global index contract only promises candidates: an evaluation drops a + /// conjunct no index can evaluate, and BTree answers `Contains`, `EndsWith` and `Like` with + /// every non-null row. + static Result IsExact(const TableSchema& table_schema, + const std::shared_ptr& filter, + const std::set& contributing_field_ids); + + /// Returns the subset of `rows` whose data satisfies the filter. + Result MatchingRowIds(const RoaringBitmap64& rows) const; + + private: + std::string root_path_; + std::shared_ptr table_schema_; + CoreOptions options_; + std::shared_ptr partition_filter_; + std::shared_ptr filter_; + int64_t snapshot_id_; + std::shared_ptr executor_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/filtered_row_id_reader_test.cpp b/src/paimon/core/table/source/filtered_row_id_reader_test.cpp new file mode 100644 index 000000000..0d4fa50b4 --- /dev/null +++ b/src/paimon/core/table/source/filtered_row_id_reader_test.cpp @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/filtered_row_id_reader.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "gtest/gtest.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/predicate/literal.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/testing/utils/testharness.h" + +namespace paimon::test { +class FilteredRowIdReaderTest : public ::testing::Test { + protected: + void SetUp() override { + std::vector fields = { + DataField(0, arrow::field("a", arrow::int32())), + DataField(1, arrow::field("b", arrow::utf8())), + }; + table_schema_ = std::make_shared( + /*version=*/1, /*id=*/0, fields, /*highest_field_id=*/1, + /*partition_keys=*/std::vector(), + /*primary_keys=*/std::vector(), + /*options=*/std::map(), /*comment=*/std::nullopt, + /*time_millis=*/0); + } + + Result IsExact(const std::shared_ptr& filter, + const std::set& contributing_field_ids) const { + return FilteredRowIdReader::IsExact(*table_schema_, filter, contributing_field_ids); + } + + static std::shared_ptr AEquals(int32_t value) { + return PredicateBuilder::Equal(0, "a", FieldType::INT, Literal(value)); + } + + static Literal BLiteral() { + return Literal(FieldType::STRING, "x", 1); + } + + std::shared_ptr table_schema_; +}; + +TEST_F(FilteredRowIdReaderTest, TestExactWhenEveryFilterFieldContributed) { + ASSERT_OK_AND_ASSIGN(bool exact, IsExact(AEquals(1), {0})); + ASSERT_TRUE(exact); + ASSERT_OK_AND_ASSIGN(exact, IsExact(AEquals(1), {})); + ASSERT_FALSE(exact); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr two_fields, + PredicateBuilder::And( + {AEquals(1), PredicateBuilder::Equal(1, "b", FieldType::STRING, BLiteral())})); + ASSERT_OK_AND_ASSIGN(exact, IsExact(two_fields, {0})); + ASSERT_FALSE(exact); + ASSERT_OK_AND_ASSIGN(exact, IsExact(two_fields, {0, 1})); + ASSERT_TRUE(exact); +} + +TEST_F(FilteredRowIdReaderTest, TestCandidateOnlyLeavesAreNotExact) { + ASSERT_OK_AND_ASSIGN(std::shared_ptr contains, + PredicateBuilder::Contains(1, "b", FieldType::STRING, BLiteral())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ends_with, + PredicateBuilder::EndsWith(1, "b", FieldType::STRING, BLiteral())); + ASSERT_OK_AND_ASSIGN(std::shared_ptr like, + PredicateBuilder::Like(1, "b", FieldType::STRING, BLiteral())); + for (const auto& candidate_only : {contains, ends_with, like}) { + ASSERT_OK_AND_ASSIGN(bool exact, IsExact(candidate_only, {1})); + ASSERT_FALSE(exact); + ASSERT_OK_AND_ASSIGN(std::shared_ptr nested, + PredicateBuilder::Or({AEquals(1), candidate_only})); + ASSERT_OK_AND_ASSIGN(exact, IsExact(nested, {0, 1})); + ASSERT_FALSE(exact); + } + + ASSERT_OK_AND_ASSIGN(std::shared_ptr starts_with, + PredicateBuilder::StartsWith(1, "b", FieldType::STRING, BLiteral())); + ASSERT_OK_AND_ASSIGN(bool exact, IsExact(starts_with, {1})); + ASSERT_TRUE(exact); +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/source/full_text_search_builder.cpp b/src/paimon/core/table/source/full_text_search_builder.cpp index 6b85f97df..c7d26a091 100644 --- a/src/paimon/core/table/source/full_text_search_builder.cpp +++ b/src/paimon/core/table/source/full_text_search_builder.cpp @@ -24,7 +24,9 @@ #include "fmt/format.h" #include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/predicate/predicate_validator.h" #include "paimon/common/types/data_field.h" +#include "paimon/common/utils/object_utils.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/string_utils.h" #include "paimon/core/core_options.h" @@ -40,6 +42,10 @@ #include "paimon/executor.h" #include "paimon/format/file_format.h" #include "paimon/memory/memory_pool.h" +#include "paimon/predicate/compound_predicate.h" +#include "paimon/predicate/function.h" +#include "paimon/predicate/predicate_builder.h" +#include "paimon/predicate/predicate_utils.h" #include "paimon/status.h" namespace paimon { @@ -53,6 +59,7 @@ class FullTextSearchBuilder::Impl { CoreOptions options; std::shared_ptr path_factory; std::shared_ptr partition_filter; + std::shared_ptr filter; DataField text_field; }; @@ -68,6 +75,7 @@ class FullTextSearchBuilder::Impl { std::string table_path_; std::vector> partition_filters_; + std::vector> filters_; std::optional limit_; std::optional field_name_; std::string query_; @@ -103,6 +111,89 @@ Status CheckNoPrimaryKeyFullTextIndex(const TableSchema& table_schema, const Cor } return Status::OK(); } + +/// Returns a predicate over the partition fields that every row satisfying `predicate` satisfies, +/// projected as Java `PredicateProjectionConverter` does: an AND keeps the children that can be +/// projected, and an OR needs all of them. So `(p = 1 AND a > 0) OR (p = 2 AND a > 0)` implies +/// `p = 1 OR p = 2`, while `p = 1 OR a > 0` implies nothing and yields null. +Result> ImpliedPartitionPredicate( + const std::shared_ptr& predicate, + const std::map& partition_key_to_idx) { + auto compound = std::dynamic_pointer_cast(predicate); + if (!compound) { + return PredicateUtils::CreatePickedFieldFilter(predicate, partition_key_to_idx); + } + bool is_and = compound->GetFunction().GetType() == Function::Type::AND; + std::vector> implied_children; + for (const auto& child : compound->Children()) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr implied, + ImpliedPartitionPredicate(child, partition_key_to_idx)); + if (implied) { + implied_children.push_back(std::move(implied)); + } else if (!is_and) { + return std::shared_ptr(); + } + } + if (implied_children.empty()) { + return std::shared_ptr(); + } + return is_and ? PredicateBuilder::And(implied_children) + : PredicateBuilder::Or(implied_children); +} + +/// The parts of the filters of a search, split as Java +/// `PartitionPredicate#splitPartitionPredicatesAndDataPredicates` does. +struct SearchFilters { + /// The conjuncts over partition fields alone and the partition predicates that the other + /// conjuncts imply, with field indexes into the partition keys; null if there are none. + std::shared_ptr partition_predicate; + /// The other conjuncts; null if there are none. + std::shared_ptr data_filter; +}; + +/// Validates `filters` and splits their conjunction. +Result ResolveFilters(const std::vector>& filters, + const arrow::Schema& schema, + const std::vector& partition_keys) { + if (filters.empty()) { + return SearchFilters(); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr filter, PredicateBuilder::And(filters)); + if (PredicateUtils::ContainsFullTextMatch(filter)) { + return Status::NotImplemented("FullTextMatch predicate execution is not implemented yet"); + } + PAIMON_RETURN_NOT_OK(PredicateValidator::ValidatePredicateWithSchema( + schema, filter, /*validate_field_idx=*/false)); + + std::map partition_key_to_idx = + ObjectUtils::CreateIdentifierToIndexMap(partition_keys); + std::vector> partition_conjuncts; + std::vector> data_conjuncts; + for (const auto& conjunct : PredicateUtils::SplitAnd(filter)) { + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr partition_conjunct, + PredicateUtils::CreatePickedFieldFilter(conjunct, partition_key_to_idx)); + if (partition_conjunct) { + partition_conjuncts.push_back(std::move(partition_conjunct)); + continue; + } + data_conjuncts.push_back(conjunct); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr implied_partitions, + ImpliedPartitionPredicate(conjunct, partition_key_to_idx)); + if (implied_partitions) { + partition_conjuncts.push_back(std::move(implied_partitions)); + } + } + SearchFilters search_filters; + if (!partition_conjuncts.empty()) { + PAIMON_ASSIGN_OR_RAISE(search_filters.partition_predicate, + PredicateBuilder::And(partition_conjuncts)); + } + if (!data_conjuncts.empty()) { + PAIMON_ASSIGN_OR_RAISE(search_filters.data_filter, PredicateBuilder::And(data_conjuncts)); + } + return search_filters; +} } // namespace Result FullTextSearchBuilder::Impl::Resolve() const { @@ -147,6 +238,16 @@ Result FullTextSearchBuilder::Impl: CheckNoPrimaryKeyFullTextIndex(*table_schema, options, text_field.value())); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(SearchFilters search_filters, + ResolveFilters(filters_, *arrow_schema, table_schema->PartitionKeys())); + if (search_filters.data_filter && + options.GetScalarIndexSearchMode() != GlobalIndexSearchMode::FAST) { + return Status::NotImplemented(fmt::format( + "a full-text search with a row filter only supports the fast {} (or {}), which " + "evaluates only the rows that a scalar index covers; full and detail are not " + "supported yet", + Options::SCALAR_INDEX_SEARCH_MODE, Options::GLOBAL_INDEX_SEARCH_MODE)); + } PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, options.CreateGlobalIndexExternalPath()); @@ -164,8 +265,24 @@ Result FullTextSearchBuilder::Impl: table_schema->PartitionKeys(), options.GetPartitionDefaultName(), arrow_schema, partition_filters_)); } - return ResolvedSearch{root_path, table_schema, std::move(options), - path_factory, partition_filter, std::move(text_field).value()}; + if (search_filters.partition_predicate) { + std::shared_ptr partition_predicate = search_filters.partition_predicate; + if (partition_filter) { + PAIMON_ASSIGN_OR_RAISE(partition_predicate, + PredicateBuilder::And({partition_filter, partition_predicate})); + } + partition_filter = std::dynamic_pointer_cast(partition_predicate); + if (!partition_filter) { + return Status::Invalid("invalid partition predicate, cannot cast to predicate filter"); + } + } + return ResolvedSearch{root_path, + table_schema, + std::move(options), + path_factory, + partition_filter, + search_filters.data_filter, + std::move(text_field).value()}; } Status FullTextSearchBuilder::Impl::CheckLimit() const { @@ -179,7 +296,7 @@ std::unique_ptr FullTextSearchBuilder::Impl::CreateScan( const ResolvedSearch& search) const { return std::make_unique( search.root_path, search.table_schema, search.options, search.path_factory, - search.partition_filter, search.text_field, memory_pool_); + search.partition_filter, search.filter, search.text_field, memory_pool_); } Result> FullTextSearchBuilder::Impl::CreateRead( @@ -187,8 +304,8 @@ Result> FullTextSearchBuilder::Impl::CreateRead( PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, DataEvolutionFullTextRead::Create( search.root_path, search.table_schema, search.options, - search.path_factory, search.partition_filter, search.text_field, - query_, limit_.value(), executor_, memory_pool_)); + search.path_factory, search.partition_filter, search.filter, + search.text_field, query_, limit_.value(), executor_, memory_pool_)); return std::unique_ptr(std::move(read)); } @@ -205,6 +322,14 @@ FullTextSearchBuilder& FullTextSearchBuilder::SetPartitionFilter( return *this; } +FullTextSearchBuilder& FullTextSearchBuilder::WithFilter( + const std::shared_ptr& predicate) { + if (predicate) { + impl_->filters_.push_back(predicate); + } + return *this; +} + FullTextSearchBuilder& FullTextSearchBuilder::SetLimit(int32_t limit) { impl_->limit_ = limit; return *this; diff --git a/src/paimon/core/table/source/full_text_search_split.cpp b/src/paimon/core/table/source/full_text_search_split.cpp index 308afd5d9..7f52cde4d 100644 --- a/src/paimon/core/table/source/full_text_search_split.cpp +++ b/src/paimon/core/table/source/full_text_search_split.cpp @@ -28,13 +28,14 @@ #include "arrow/type.h" #include "fmt/format.h" -#include "fmt/ranges.h" #include "paimon/common/data/binary_array_writer.h" #include "paimon/common/data/binary_row.h" #include "paimon/common/data/binary_section.h" #include "paimon/common/data/serializer/binary_row_serializer.h" #include "paimon/common/io/memory_segment_output_stream.h" #include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/common/utils/object_utils.h" +#include "paimon/common/utils/string_utils.h" #include "paimon/core/index/index_file_meta_serializer.h" #include "paimon/core/table/source/data_evolution_full_text_scan.h" #include "paimon/core/table/source/index_full_text_search_split_impl.h" @@ -50,6 +51,12 @@ constexpr int64_t kSearchRowRangeBytes = 2 * sizeof(int64_t); /// A serialized index file is a binary row that starts with its int32 length. constexpr int64_t kMinIndexFileBytes = sizeof(int32_t); +/// Returns whether [`start`, `end`] is a valid row range: non-negative, ordered, and with a row +/// count that fits in int64, as reads take its `Range::Count()`. +bool IsValidRowRange(int64_t start, int64_t end) { + return start >= 0 && start <= end && end - start != std::numeric_limits::max(); +} + /// Checks that `count` entries of at least `min_entry_bytes` bytes each fit in the rest of `in`, /// so a corrupt count fails before anything is allocated for it. Status CheckEntryCount(const DataInputStream& in, int32_t count, int64_t min_entry_bytes, @@ -206,10 +213,25 @@ Result>> ReadIndexFiles( } } // namespace +Status IndexFullTextSearchSplitImpl::CheckScalarIndexFile( + const std::shared_ptr& file) { + if (!file || !file->GetGlobalIndexMeta()) { + return Status::Invalid("full-text search split requires global index files"); + } + const GlobalIndexMeta& meta = file->GetGlobalIndexMeta().value(); + if (!IsValidRowRange(meta.row_range_start, meta.row_range_end)) { + return Status::Invalid( + fmt::format("scalar index file {} covers an invalid row range [{}, {}]", + file->FileName(), meta.row_range_start, meta.row_range_end)); + } + return Status::OK(); +} + Result> IndexFullTextSearchSplitImpl::Create( int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, - std::vector search_row_ranges) { + std::vector search_row_ranges, + std::vector> scalar_index_files) { if (column_name.empty()) { return Status::Invalid("full-text search split requires a column name"); } @@ -219,9 +241,7 @@ Result> IndexFullTextSearchSplitIm "full-text search split requires a column name of at most {} bytes, but got {} bytes", std::numeric_limits::max(), column_name.size())); } - // The row count of the range must fit in int64, as reads take its `Range::Count()`. - if (row_range_start < 0 || row_range_start > row_range_end || - row_range_end - row_range_start == std::numeric_limits::max()) { + if (!IsValidRowRange(row_range_start, row_range_end)) { return Status::Invalid( fmt::format("invalid physical index range [{}, {}]", row_range_start, row_range_end)); } @@ -272,45 +292,33 @@ Result> IndexFullTextSearchSplitIm previous.ToString(), range.ToString())); } } + for (const auto& file : scalar_index_files) { + PAIMON_RETURN_NOT_OK(CheckScalarIndexFile(file)); + } return std::shared_ptr(new IndexFullTextSearchSplitImpl( snapshot_id, column_name, row_range_start, row_range_end, std::move(full_text_index_files), - std::move(search_row_ranges))); + std::move(search_row_ranges), std::move(scalar_index_files))); } bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl& other) const { if (this == &other) { return true; } - if (snapshot_id_ != other.snapshot_id_ || column_name_ != other.column_name_ || - row_range_start_ != other.row_range_start_ || row_range_end_ != other.row_range_end_ || - search_row_ranges_ != other.search_row_ranges_ || - full_text_index_files_.size() != other.full_text_index_files_.size()) { - return false; - } - for (size_t i = 0; i < full_text_index_files_.size(); ++i) { - if (!(*full_text_index_files_[i] == *other.full_text_index_files_[i])) { - return false; - } - } - return true; + return snapshot_id_ == other.snapshot_id_ && column_name_ == other.column_name_ && + row_range_start_ == other.row_range_start_ && row_range_end_ == other.row_range_end_ && + search_row_ranges_ == other.search_row_ranges_ && + ObjectUtils::Equal(full_text_index_files_, other.full_text_index_files_) && + ObjectUtils::Equal(scalar_index_files_, other.scalar_index_files_); } std::string IndexFullTextSearchSplitImpl::ToString() const { - std::vector ranges; - ranges.reserve(search_row_ranges_.size()); - for (const auto& range : search_row_ranges_) { - ranges.push_back(range.ToString()); - } - std::vector files; - files.reserve(full_text_index_files_.size()); - for (const auto& file : full_text_index_files_) { - files.push_back(file->ToString()); - } return fmt::format( "IndexFullTextSearchSplit{{snapshotId={}, columnName={}, rowRangeStart={}, " - "rowRangeEnd={}, searchRowRanges=[{}], fullTextIndexFiles=[{}]}}", - snapshot_id_, column_name_, row_range_start_, row_range_end_, fmt::join(ranges, ", "), - fmt::join(files, ", ")); + "rowRangeEnd={}, searchRowRanges={}, fullTextIndexFiles={}, scalarIndexFiles={}}}", + snapshot_id_, column_name_, row_range_start_, row_range_end_, + StringUtils::VectorToString(search_row_ranges_), + StringUtils::VectorToString(full_text_index_files_), + StringUtils::VectorToString(scalar_index_files_)); } Result FullTextSearchSplit::Serialize( @@ -334,6 +342,7 @@ Result FullTextSearchSplit::Serialize( } IndexFileMetaSerializer serializer(pool); PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->FullTextIndexFiles(), &out)); + PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->ScalarIndexFiles(), &out)); PAIMON_UNIQUE_PTR bytes = MemorySegmentUtils::CopyToBytes(out.Segments(), 0, out.CurrentSize(), pool.get()); return std::string(bytes->data(), bytes->size()); @@ -382,6 +391,10 @@ Result> FullTextSearchSplit::Deserialize( } PAIMON_ASSIGN_OR_RAISE(std::vector> full_text_index_files, ReadIndexFiles(&in, pool)); + std::vector> scalar_index_files; + if (version >= 2) { + PAIMON_ASSIGN_OR_RAISE(scalar_index_files, ReadIndexFiles(&in, pool)); + } PAIMON_ASSIGN_OR_RAISE(int64_t pos, in.GetPos()); PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, in.Length()); if (pos != stream_length) { @@ -392,7 +405,8 @@ Result> FullTextSearchSplit::Deserialize( PAIMON_ASSIGN_OR_RAISE(std::shared_ptr split, IndexFullTextSearchSplitImpl::Create( snapshot_id, column_name, row_range_start, row_range_end, - std::move(full_text_index_files), std::move(search_row_ranges))); + std::move(full_text_index_files), std::move(search_row_ranges), + std::move(scalar_index_files))); return std::shared_ptr(std::move(split)); } diff --git a/src/paimon/core/table/source/full_text_search_split_test.cpp b/src/paimon/core/table/source/full_text_search_split_test.cpp index 606e4438a..44af2eb03 100644 --- a/src/paimon/core/table/source/full_text_search_split_test.cpp +++ b/src/paimon/core/table/source/full_text_search_split_test.cpp @@ -56,6 +56,15 @@ std::shared_ptr NewIndexFile( /*external_path=*/std::nullopt, meta); } +std::shared_ptr NewScalarIndexFile(int64_t start, int64_t end, + const std::shared_ptr& pool) { + GlobalIndexMeta meta(start, end, /*index_field_id=*/2, /*extra_field_ids=*/std::nullopt, + std::make_shared("{}", pool.get())); + return std::make_shared("btree", "btree-0.index", /*file_size=*/128, + /*row_count=*/10, /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, meta); +} + std::string ToBytes(MemorySegmentOutputStream* out, const std::shared_ptr& pool) { PAIMON_UNIQUE_PTR bytes = MemorySegmentUtils::CopyToBytes(out->Segments(), 0, out->CurrentSize(), pool.get()); @@ -104,7 +113,7 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { /*row_range_end=*/199, files, {Range(100, 119), Range(150, 199)})); ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); - const std::string version_1_index_files = FromHex( + const std::string index_files = FromHex( "00000002" "000000C8" "0030000000000000090000004000000011000000500000008000000000000000" @@ -124,22 +133,19 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { "100000003800000017000000480000000000000000000000" "02000000000000000100000003000000" "7B22746F6B656E697A6572223A2264656661756C74227D00"); - std::string version_1_bytes = "PFTSPLT1" + - FromHex( - "00000001" - "0000000000000007" - "0007") + - "content" + - FromHex( - "0000000000000064" - "00000000000000C7" - "00000002" - "0000000000000064" - "0000000000000077" - "0000000000000096" - "00000000000000C7") + - version_1_index_files; - ASSERT_EQ(bytes, version_1_bytes); + auto bytes_of_version = [&index_files](const std::string& version) { + return "PFTSPLT1" + FromHex(version + "0000000000000007" + "0007") + "content" + + FromHex( + "0000000000000064" + "00000000000000C7" + "00000002" + "0000000000000064" + "0000000000000077" + "0000000000000096" + "00000000000000C7") + + index_files; + }; + ASSERT_EQ(bytes, bytes_of_version("00000002") + FromHex("00000000")); ASSERT_OK_AND_ASSIGN(std::shared_ptr result, FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); auto index_split = std::dynamic_pointer_cast(result); @@ -158,6 +164,14 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { ASSERT_OK_AND_ASSIGN(std::string bytes_again, FullTextSearchSplit::Serialize(result, pool_)); ASSERT_EQ(bytes_again, bytes); + std::string version_1_bytes = bytes_of_version("00000001"); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr version_1_result, + FullTextSearchSplit::Deserialize(version_1_bytes.data(), version_1_bytes.size(), pool_)); + auto version_1_impl = std::dynamic_pointer_cast(version_1_result); + ASSERT_TRUE(version_1_impl); + ASSERT_EQ(*version_1_impl, *split) << version_1_impl->ToString() << " vs " << split->ToString(); + std::string longest_column(std::numeric_limits::max(), 'c'); ASSERT_OK_AND_ASSIGN(split, IndexFullTextSearchSplitImpl::Create( /*snapshot_id=*/7, longest_column, /*row_range_start=*/100, @@ -170,6 +184,35 @@ TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { ASSERT_EQ(index_split->ColumnName(), longest_column); } +TEST_F(FullTextSearchSplitTest, TestSerializeScalarIndexFiles) { + std::vector> files = {NewIndexFile( + "full-text-0.index", "full-text", Range(0, 99), /*index_field_id=*/1, std::nullopt, pool_)}; + std::vector> scalar_files = { + NewIndexFile("btree-0.index", "btree", Range(0, 49), /*index_field_id=*/2, std::nullopt, + pool_), + NewIndexFile("multi-0.index", "full-text", Range(50, 149), /*index_field_id=*/4, + /*extra_field_ids=*/std::vector({2}), pool_)}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 99, files, + {Range(0, 99)}, scalar_files)); + ASSERT_EQ(split->ScalarIndexFiles().size(), 2u); + + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + auto impl = std::dynamic_pointer_cast(result); + ASSERT_TRUE(impl); + ASSERT_EQ(*impl, *split) << impl->ToString() << " vs " << split->ToString(); + ASSERT_EQ(impl->ScalarIndexFiles()[0]->FileName(), "btree-0.index"); + ASSERT_EQ(impl->ScalarIndexFiles()[1]->FileName(), "multi-0.index"); + ASSERT_NE(impl->ToString().find("scalarIndexFiles=["), std::string::npos); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr no_scalar_files, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 99, files, + {Range(0, 99)})); + ASSERT_FALSE(*no_scalar_files == *split); +} + TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { std::vector> files = {NewIndexFile( "full-text-0.index", "full-text", Range(0, 9), /*index_field_id=*/1, std::nullopt, pool_)}; @@ -184,6 +227,12 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { for (size_t size = 0; size < bytes.size(); ++size) { ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), size, pool_)) << size; } + std::string too_many_scalar_files = bytes; + too_many_scalar_files.replace(too_many_scalar_files.size() - sizeof(int32_t), sizeof(int32_t), + FromHex("7FFFFFFF")); + ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Deserialize(too_many_scalar_files.data(), + too_many_scalar_files.size(), pool_), + "2147483647 index files exceed the remaining 0 bytes"); ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Deserialize(nullptr, 0, pool_), "null buffer"); ASSERT_NOK_WITH_MSG( FullTextSearchSplit::Deserialize(bytes.data(), std::numeric_limits::max(), pool_), @@ -203,7 +252,7 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { std::string newer_version = ToBytes(&out, pool_); ASSERT_NOK_WITH_MSG( FullTextSearchSplit::Deserialize(newer_version.data(), newer_version.size(), pool_), - "unsupported IndexFullTextSearchSplit version: 2"); + "unsupported IndexFullTextSearchSplit version: 3"); } ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Serialize(nullptr, pool_), "cannot cast to an index split"); @@ -237,7 +286,7 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptFileLayout) { const size_t index_type_field = 0; const size_t global_index_field = 6; const size_t extra_field_ids_field = 3; - const size_t file_start = bytes.size() - file_row.GetSizeInBytes(); + const size_t file_start = bytes.size() - sizeof(int32_t) - file_row.GetSizeInBytes(); const size_t index_type_slot = slot_of(file_start, file_row.GetFieldCount(), index_type_field); const size_t global_index_slot = slot_of(file_start, file_row.GetFieldCount(), global_index_field); @@ -318,6 +367,17 @@ TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptEntries) { out->WriteValue(0); }; ASSERT_NOK_WITH_MSG(deserialize(empty_file), "binary row of 0 bytes without its"); + auto reversed_scalar_file = [&](MemorySegmentOutputStream* out) { + write_one_range(out); + IndexFileMetaSerializer serializer(pool_); + ASSERT_OK( + serializer.SerializeList({NewIndexFile("full-text-0.index", "full-text", Range(0, 9), + /*index_field_id=*/1, std::nullopt, pool_)}, + out)); + ASSERT_OK(serializer.SerializeList({NewScalarIndexFile(9, 0, pool_)}, out)); + }; + ASSERT_NOK_WITH_MSG(deserialize(reversed_scalar_file), + "scalar index file btree-0.index covers an invalid row range [9, 0]"); } TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { @@ -358,6 +418,9 @@ TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { "full-text", "full-text-3.index", /*file_size=*/128, /*row_count=*/10, /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); ASSERT_NOK_WITH_MSG(create({no_global_meta}, {Range(10, 19)}), "global index files"); + ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, files, + {Range(10, 19)}, {no_global_meta}), + "global index files"); ASSERT_NOK_WITH_MSG( IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "", 10, 19, files, {Range(10, 19)}), "requires a column name"); @@ -371,6 +434,10 @@ TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", start, end, files, {Range(10, 19)}), "invalid physical index range"); + ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create( + /*snapshot_id=*/1, "f0", 10, 19, files, {Range(10, 19)}, + {NewScalarIndexFile(start, end, pool_)}), + "scalar index file btree-0.index covers an invalid row range"); } } diff --git a/src/paimon/core/table/source/index_full_text_search_split_impl.h b/src/paimon/core/table/source/index_full_text_search_split_impl.h index f2cd8cb62..eaaac04aa 100644 --- a/src/paimon/core/table/source/index_full_text_search_split_impl.h +++ b/src/paimon/core/table/source/index_full_text_search_split_impl.h @@ -32,14 +32,14 @@ namespace paimon { /// The implementation of `IndexFullTextSearchSplit`, which also holds the full-text index files of -/// the shard. +/// the shard and the scalar index files that can pre-filter its rows. class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { public: /// "PFTSPLT1" in ASCII. static constexpr int64_t MAGIC = 0x50465453504C5431L; - /// TODO: once row filters (#405) are supported, add the scalar index files that pre-filter rows - /// in a new version, as version 2 of Java `IndexFullTextSearchSplit` does. - static constexpr int32_t VERSION = 1; + /// Version 2 appends the scalar index files that pre-filter rows, as version 2 of Java + /// `IndexFullTextSearchSplit` does. + static constexpr int32_t VERSION = 2; /// Creates a split. /// @param snapshot_id The snapshot the split is planned from. @@ -52,11 +52,18 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { /// global index meta of the shard range. /// @param search_row_ranges The sorted, disjoint and non-adjacent row id ranges to rank, at /// least one and all within the shard range. + /// @param scalar_index_files The global index files that intersect the shard range and can + /// pre-filter its rows, each accepted by `CheckScalarIndexFile()`. /// @return The split, or `Invalid` if an argument breaks these rules. static Result> Create( int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, - std::vector search_row_ranges); + std::vector search_row_ranges, + std::vector> scalar_index_files = {}); + + /// Checks that `file` has a global index meta whose row range is not negative, is ordered, and + /// has a row count that fits in int64, as a split requires of its scalar index files. + static Status CheckScalarIndexFile(const std::shared_ptr& file); int64_t SnapshotId() const override { return snapshot_id_; @@ -82,6 +89,10 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { return full_text_index_files_; } + const std::vector>& ScalarIndexFiles() const { + return scalar_index_files_; + } + bool operator==(const IndexFullTextSearchSplitImpl& other) const; std::string ToString() const; @@ -90,13 +101,15 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { IndexFullTextSearchSplitImpl(int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, int64_t row_range_end, std::vector> full_text_index_files, - std::vector search_row_ranges) + std::vector search_row_ranges, + std::vector> scalar_index_files) : snapshot_id_(snapshot_id), column_name_(column_name), row_range_start_(row_range_start), row_range_end_(row_range_end), full_text_index_files_(std::move(full_text_index_files)), - search_row_ranges_(std::move(search_row_ranges)) {} + search_row_ranges_(std::move(search_row_ranges)), + scalar_index_files_(std::move(scalar_index_files)) {} int64_t snapshot_id_; std::string column_name_; @@ -104,6 +117,7 @@ class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { int64_t row_range_end_; std::vector> full_text_index_files_; std::vector search_row_ranges_; + std::vector> scalar_index_files_; }; } // namespace paimon diff --git a/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h b/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h index a10b11b85..3229a76ef 100644 --- a/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h +++ b/src/paimon/core/table/source/snapshot/static_from_tag_starting_scanner.h @@ -36,7 +36,8 @@ class StaticFromTagStartingScanner : public StartingScanner { Result> Scan( const std::shared_ptr& snapshot_reader) override { - const TagManager tag_manager(snapshot_manager_->Fs(), snapshot_manager_->RootPath()); + const TagManager tag_manager(snapshot_manager_->Fs(), snapshot_manager_->RootPath(), + snapshot_manager_->Branch()); PAIMON_ASSIGN_OR_RAISE(const Tag tag, tag_manager.GetOrThrow(tag_name_)); PAIMON_ASSIGN_OR_RAISE(const Snapshot snapshot, tag.TrimToSnapshot()); PAIMON_ASSIGN_OR_RAISE( diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index 19c606ef0..cfa2052b2 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include "arrow/type.h" #include "fmt/format.h" @@ -30,11 +31,17 @@ #include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/types/data_field.h" #include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" +#include "paimon/core/core_options.h" #include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/global_index/indexed_split_impl.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" #include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/table/source/filtered_row_id_reader.h" +#include "paimon/core/table/source/index_full_text_search_split_impl.h" #include "paimon/core/utils/snapshot_manager.h" #include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" @@ -272,6 +279,15 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter return builder; } + static std::vector RowIdsOf(const ScoredRows& rows) { + std::vector row_ids; + row_ids.reserve(rows.size()); + for (const auto& row : rows) { + row_ids.push_back(row.first); + } + return row_ids; + } + static Result ToScoredRows(const std::shared_ptr& result) { PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iter, result->CreateScoredIterator()); @@ -294,6 +310,19 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter return ToScoredRows(result); } + Result SearchTableWithFilter( + const std::string& table_path, const std::string& query, int32_t limit, + const std::shared_ptr& filter, + const std::vector>& partition_filters = {}, + const std::map& options = {}) const { + std::unique_ptr builder = + NewFullTextSearchBuilder(table_path, query, limit, options); + builder->SetPartitionFilter(partition_filters).WithFilter(filter); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + builder->ExecuteLocal()); + return ToScoredRows(result); + } + Status TagSnapshot(const std::string& table_path, int64_t snapshot_id, const std::string& tag_name) const { std::shared_ptr fs = dir_->GetFileSystem(); @@ -304,11 +333,12 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter /*overwrite=*/false); } - Result PlanShards(const std::string& table_path, - const std::string& query) const { - PAIMON_ASSIGN_OR_RAISE( - std::unique_ptr scan, - NewFullTextSearchBuilder(table_path, query, /*limit=*/1)->NewFullTextScan()); + Result PlanShards(const std::string& table_path, const std::string& query, + const std::shared_ptr& filter = nullptr) const { + std::unique_ptr builder = + NewFullTextSearchBuilder(table_path, query, /*limit=*/1); + builder->WithFilter(filter); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, builder->NewFullTextScan()); PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); PlannedShards shards; for (const auto& split : plan->Splits()) { @@ -428,6 +458,10 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter Range(0, 4))); ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, Range(5, 9))); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f1", "btree", /*options=*/{}, + Range(0, 4))); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f1", "btree", /*options=*/{}, + Range(5, 9))); auto create_scan = [&]() { return GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, @@ -483,6 +517,58 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter CheckFullTextTopK(readers[0], query, RoaringBitmap64::From({0, 1}), {}); CheckTableSearchMatchesReader(table_path, query, readers[0], /*max_limit=*/6); + ScoredRows expected_filtered_rows; + for (const auto& row : expected_live_rows) { + if (row.first >= 5) { + expected_filtered_rows.push_back(row); + } + } + std::shared_ptr f1_filter = + PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(5)); + ASSERT_OK_AND_ASSIGN(ScoredRows filtered_rows, + SearchTableWithFilter(table_path, query, /*limit=*/100, f1_filter)); + ASSERT_EQ(filtered_rows, expected_filtered_rows); + + std::unique_ptr filtered_builder = + NewFullTextSearchBuilder(table_path, query, /*limit=*/100); + filtered_builder->WithFilter(f1_filter); + ASSERT_OK_AND_ASSIGN(std::unique_ptr filtered_scan, + filtered_builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr filtered_plan, + filtered_scan->CreatePlan()); + ASSERT_EQ(filtered_plan->Splits().size(), 2u); + for (const auto& filtered_split : filtered_plan->Splits()) { + auto index_split = + std::dynamic_pointer_cast(filtered_split); + ASSERT_TRUE(index_split); + ASSERT_EQ(index_split->ScalarIndexFiles().size(), 1u); + const std::optional& scalar_meta = + index_split->ScalarIndexFiles()[0]->GetGlobalIndexMeta(); + ASSERT_EQ(scalar_meta->row_range_start, index_split->RowRangeStart()); + ASSERT_EQ(scalar_meta->row_range_end, index_split->RowRangeEnd()); + } + + ScoredRows expected_refined_rows; + for (const auto& row : expected_live_rows) { + if (row.first <= 7) { + expected_refined_rows.push_back(row); + } + } + ASSERT_OK_AND_ASSIGN( + std::shared_ptr starts_with, + PredicateBuilder::StartsWith(0, "f0", FieldType::STRING, + Literal(FieldType::STRING, "apple apple apple", 17))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr partly_indexed, + PredicateBuilder::And({PredicateBuilder::GreaterOrEqual( + 1, "f1", FieldType::INT, Literal(0)), + starts_with})); + ASSERT_OK_AND_ASSIGN( + ScoredRows refined_rows, + SearchTableWithFilter(table_path, query, /*limit=*/100, partly_indexed, + /*partition_filters=*/{}, + {{Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, "true"}})); + ASSERT_EQ(refined_rows, expected_refined_rows); + ASSERT_OK_AND_ASSIGN( std::unique_ptr full_text_read, NewFullTextSearchBuilder(table_path, query, /*limit=*/100)->NewFullTextRead()); @@ -700,25 +786,319 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_EQ(rows_with_bitmap, top_rows); } - void CheckTableSearchWithPartitionFilter(const std::string& table_path, - const std::string& query) const { - auto row_ids_of = [](const ScoredRows& rows) { - std::vector row_ids; - for (const auto& row : rows) { - row_ids.push_back(row.first); + void CheckTableSearchWithFilter(const std::string& index_type, + const std::map& index_options) { + arrow::FieldVector fields = { + arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::utf8()), arrow::field("f3", arrow::int32()), + arrow::field("f4", arrow::utf8())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +["apple", 0, "a-red", 0, "pear"], +["apple apple", 0, "b-red", 1, "pear"], +["apple apple apple", 1, "c-red", 2, "pear"], +["apple apple apple apple", 1, "d-blue", 3, "pear"], +["apple apple apple apple apple", 2, "e-blue", 4, "pear"], +["apple apple apple apple apple apple", 2, "f-blue", 5, "pear"] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, + Range(0, 5))); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", "btree", /*options=*/{}, + Range(0, 5))); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f1", "btree", /*options=*/{}, + Range(0, 5))); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f2", "btree", /*options=*/{}, + Range(0, 5))); + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f4", index_type, index_options, + Range(0, 5))); + + std::string query = R"({"match":{"query":"apple"}})"; + ASSERT_OK_AND_ASSIGN(ScoredRows all_rows, SearchTable(table_path, query, /*limit=*/100)); + ASSERT_EQ(all_rows.size(), 6u); + auto rows_of = [&all_rows](const std::set& row_ids) { + ScoredRows rows; + for (const auto& row : all_rows) { + if (row_ids.count(row.first) > 0) { + rows.push_back(row); + } } - return row_ids; + return rows; + }; + const std::map refine = { + {Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, "true"}}; + auto check_filter = [&](const std::shared_ptr& filter, + const std::set& expected_row_ids, + const std::map& search_options = {}, + int32_t limit = 100) { + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTableWithFilter(table_path, query, limit, filter, + /*partition_filters=*/{}, search_options)); + ASSERT_EQ(rows, rows_of(expected_row_ids)); + }; + + std::shared_ptr f1_is_1 = + PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(1)); + check_filter(f1_is_1, {2, 3}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr f1_in_range, + PredicateBuilder::And( + {PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(1)), + PredicateBuilder::LessThan(1, "f1", FieldType::INT, Literal(2))})); + check_filter(f1_in_range, {2, 3}); + check_filter(PredicateBuilder::In(1, "f1", FieldType::INT, {Literal(0), Literal(2)}), + {0, 1, 4, 5}); + check_filter(PredicateBuilder::NotEqual(1, "f1", FieldType::INT, Literal(1)), {0, 1, 4, 5}); + check_filter(PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(100)), {}); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr f1_or_f2, + PredicateBuilder::Or( + {f1_is_1, PredicateBuilder::Equal(2, "f2", FieldType::STRING, + Literal(FieldType::STRING, "a-red", 5))})); + check_filter(f1_or_f2, {0, 2, 3}); + std::shared_ptr f1_matches_all = + PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(0)); + check_filter(f1_matches_all, {0, 1, 2, 3, 4, 5}); + check_filter(PredicateBuilder::Equal(0, "f0", FieldType::STRING, + Literal(FieldType::STRING, "apple apple", 11)), + {1}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr starts_with, + PredicateBuilder::StartsWith(2, "f2", FieldType::STRING, + Literal(FieldType::STRING, "a-", 2))); + check_filter(starts_with, {0}); + + { + std::unique_ptr builder = + NewFullTextSearchBuilder(table_path, query, /*limit=*/100); + builder + ->WithFilter(PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(1))) + .WithFilter(PredicateBuilder::LessThan(1, "f1", FieldType::INT, Literal(2))); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + builder->ExecuteLocal()); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ToScoredRows(result)); + ASSERT_EQ(rows, rows_of({2, 3})); + } + + for (int32_t f1 = 0; f1 <= 2; ++f1) { + SCOPED_TRACE(f1); + ScoredRows matching_rows = rows_of({2 * f1, 2 * f1 + 1}); + ASSERT_OK_AND_ASSIGN(ScoredRows top_row, + SearchTableWithFilter(table_path, query, /*limit=*/1, + PredicateBuilder::Equal( + 1, "f1", FieldType::INT, Literal(f1)))); + ASSERT_EQ(top_row.size(), 1u); + ASSERT_TRUE(top_row[0] == matching_rows[0] || top_row[0] == matching_rows[1]); + ASSERT_EQ(top_row[0].second, + std::max(matching_rows[0].second, matching_rows[1].second)); + } + + std::shared_ptr f3_is_3 = + PredicateBuilder::Equal(3, "f3", FieldType::INT, Literal(3)); + check_filter(f3_is_3, {}); + check_filter(f3_is_3, {}, refine); + + std::shared_ptr f4_is_pear = PredicateBuilder::Equal( + 4, "f4", FieldType::STRING, Literal(FieldType::STRING, "pear", 4)); + check_filter(f4_is_pear, {}); + check_filter(f4_is_pear, {}, refine); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr partly_indexed, + PredicateBuilder::And({f1_matches_all, f3_is_3})); + check_filter(partly_indexed, {}); + check_filter(partly_indexed, {3}, refine); + check_filter(partly_indexed, {3}, refine, /*limit=*/1); + ASSERT_OK_AND_ASSIGN(std::shared_ptr undecidable_or, + PredicateBuilder::Or({f1_is_1, f3_is_3})); + check_filter(undecidable_or, {}); + check_filter(undecidable_or, {}, refine); + + const Literal blue(FieldType::STRING, "blue", 4); + ASSERT_OK_AND_ASSIGN(std::shared_ptr contains, + PredicateBuilder::Contains(2, "f2", FieldType::STRING, blue)); + check_filter(contains, {}); + check_filter(contains, {3, 4, 5}, refine); + ASSERT_OK_AND_ASSIGN(std::shared_ptr contains_nothing, + PredicateBuilder::Contains(2, "f2", FieldType::STRING, + Literal(FieldType::STRING, "omega", 5))); + check_filter(contains_nothing, {}, refine); + ASSERT_OK_AND_ASSIGN(std::shared_ptr ends_with, + PredicateBuilder::EndsWith(2, "f2", FieldType::STRING, blue)); + check_filter(ends_with, {}); + check_filter(ends_with, {3, 4, 5}, refine); + ASSERT_OK_AND_ASSIGN(std::shared_ptr like, + PredicateBuilder::Like(2, "f2", FieldType::STRING, + Literal(FieldType::STRING, "%-red", 5))); + check_filter(like, {}); + check_filter(like, {0, 1, 2}, refine); + ASSERT_OK_AND_ASSIGN(std::shared_ptr like_first_row, + PredicateBuilder::Like(2, "f2", FieldType::STRING, + Literal(FieldType::STRING, "%a-red", 6))); + check_filter(like_first_row, {0}, refine, /*limit=*/1); + + std::unique_ptr builder = + NewFullTextSearchBuilder(table_path, query, /*limit=*/100); + builder->WithFilter(f1_is_1); + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, builder->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_EQ(plan->Splits().size(), 1u); + auto index_split = + std::dynamic_pointer_cast(plan->Splits()[0]); + ASSERT_TRUE(index_split); + ASSERT_EQ(index_split->ScalarIndexFiles().size(), 1u); + ASSERT_EQ(index_split->ScalarIndexFiles()[0]->GetGlobalIndexMeta()->index_field_id, 1); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr unfiltered_scan, + NewFullTextSearchBuilder(table_path, query, /*limit=*/100)->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr unfiltered_plan, + unfiltered_scan->CreatePlan()); + ASSERT_EQ(unfiltered_plan->Splits().size(), 1u); + index_split = + std::dynamic_pointer_cast(unfiltered_plan->Splits()[0]); + ASSERT_TRUE(index_split); + ASSERT_TRUE(index_split->ScalarIndexFiles().empty()); + + std::vector> splits; + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr deserialized, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + splits.push_back(std::move(deserialized)); + } + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, ToScoredRows(result)); + ASSERT_EQ(rows, rows_of({2, 3})); + + int64_t snapshot_id = plan->SnapshotId().value(); + ASSERT_OK(TagSnapshot(table_path, snapshot_id, "filtered")); + const std::map time_travel_options = { + {Options::SCAN_SNAPSHOT_ID, std::to_string(snapshot_id)}, + {Options::SCAN_TAG_NAME, "filtered"}, + {Options::SCAN_TIMESTAMP_MILLIS, std::to_string(std::numeric_limits::max())}}; + for (const auto& time_travel_option : time_travel_options) { + SCOPED_TRACE(time_travel_option.first); + std::map search_options = refine; + search_options.insert(time_travel_option); + check_filter(partly_indexed, {3}, search_options); + } + } + + void CheckTableSearchWithPartitionAndDataFilter( + const std::string& index_type, const std::map& index_options) { + arrow::FieldVector fields = { + arrow::field("f0", arrow::utf8()), arrow::field("f1", arrow::int32()), + arrow::field("f2", arrow::int32()), arrow::field("f3", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{"f1"}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto write_partition = [&](const std::string& json, + const std::map& partition, + const Range& range) { + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), json) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, WriteArray(table_path, partition, + schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + ASSERT_OK(WriteIndex(table_path, {partition}, "f0", index_type, index_options, range)); + ASSERT_OK(WriteIndex(table_path, {partition}, "f2", "btree", /*options=*/{}, range)); + }; + write_partition(R"([["apple apple", 10, 1, 1], ["apple", 10, 2, 2]])", {{"f1", "10"}}, + Range(0, 1)); + write_partition(R"([["apple apple", 20, 1, 3], ["apple", 20, 2, 4]])", {{"f1", "20"}}, + Range(2, 3)); + + std::string query = R"({"match":{"query":"apple"}})"; + ASSERT_OK_AND_ASSIGN(ScoredRows all_rows, SearchTable(table_path, query, /*limit=*/100)); + ASSERT_EQ(all_rows.size(), 4u); + auto check_filter = [&](const std::shared_ptr& filter, + const std::vector>& partitions, + const std::vector& expected_row_ids, + const std::map& search_options = {}) { + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTableWithFilter(table_path, query, /*limit=*/100, filter, + partitions, search_options)); + ASSERT_EQ(RowIdsOf(rows), expected_row_ids); }; + + std::shared_ptr partition_10 = + PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(10)); + std::shared_ptr partition_20 = + PredicateBuilder::Equal(1, "f1", FieldType::INT, Literal(20)); + std::shared_ptr f2_is_1 = + PredicateBuilder::Equal(2, "f2", FieldType::INT, Literal(1)); + + check_filter(partition_20, /*partitions=*/{}, {2, 3}); + check_filter(partition_20, /*partitions=*/{}, {2, 3}, + {{Options::SCALAR_INDEX_SEARCH_MODE, "full"}}); + check_filter(f2_is_1, /*partitions=*/{}, {0, 2}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr both, + PredicateBuilder::And({partition_10, f2_is_1})); + check_filter(both, /*partitions=*/{}, {0}); + check_filter(f2_is_1, {{{"f1", "20"}}}, {2}); + check_filter(partition_20, {{{"f1", "10"}}}, {}); + + ASSERT_OK_AND_ASSIGN( + std::shared_ptr partly_indexed, + PredicateBuilder::And({partition_20, f2_is_1, + PredicateBuilder::Equal(3, "f3", FieldType::INT, Literal(3))})); + check_filter(partly_indexed, /*partitions=*/{}, {}); + check_filter(partly_indexed, /*partitions=*/{}, {2}, + {{Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, "true"}}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr mixed, + PredicateBuilder::Or({partition_10, f2_is_1})); + check_filter(mixed, /*partitions=*/{}, {}); + + ASSERT_OK_AND_ASSIGN(std::shared_ptr partition_20_and_f2_is_1, + PredicateBuilder::And({partition_20, f2_is_1})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr partition_20_and_f2_is_5, + PredicateBuilder::And( + {partition_20, PredicateBuilder::Equal(2, "f2", FieldType::INT, Literal(5))})); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr implies_partition_20, + PredicateBuilder::Or({partition_20_and_f2_is_1, partition_20_and_f2_is_5})); + ASSERT_OK_AND_ASSIGN(PlannedShards shards, + PlanShards(table_path, query, implies_partition_20)); + ASSERT_EQ(shards, PlannedShards({{Range(2, 3), {Range(2, 3)}}})); + check_filter(implies_partition_20, /*partitions=*/{}, {}); + check_filter(implies_partition_20, /*partitions=*/{}, {2}, + {{Options::GLOBAL_INDEX_FILTER_REFINE_FROM_DATA, "true"}}); + ASSERT_OK_AND_ASSIGN(std::shared_ptr implies_nothing, + PredicateBuilder::Or({partition_20_and_f2_is_1, f2_is_1})); + ASSERT_OK_AND_ASSIGN(shards, PlanShards(table_path, query, implies_nothing)); + ASSERT_EQ(shards, + PlannedShards({{Range(0, 1), {Range(0, 1)}}, {Range(2, 3), {Range(2, 3)}}})); + } + + void CheckTableSearchWithPartitionFilter(const std::string& table_path, + const std::string& query) const { ASSERT_OK_AND_ASSIGN(ScoredRows rows, SearchTable(table_path, query, /*limit=*/10, {{{"f1", "10"}}})); - ASSERT_EQ(row_ids_of(rows), std::vector({0, 1})); + ASSERT_EQ(RowIdsOf(rows), std::vector({0, 1})); ASSERT_OK_AND_ASSIGN(rows, SearchTable(table_path, query, /*limit=*/10, {{{"f1", "20"}}})); - ASSERT_EQ(row_ids_of(rows), std::vector({2})); + ASSERT_EQ(RowIdsOf(rows), std::vector({2})); ASSERT_OK_AND_ASSIGN( rows, SearchTable(table_path, query, /*limit=*/10, {{{"f1", "10"}}, {{"f1", "20"}}})); - ASSERT_EQ(row_ids_of(rows), std::vector({0, 1, 2})); + ASSERT_EQ(RowIdsOf(rows), std::vector({0, 1, 2})); ASSERT_OK_AND_ASSIGN(rows, SearchTable(table_path, query, /*limit=*/10)); - ASSERT_EQ(row_ids_of(rows), std::vector({0, 1, 2})); + ASSERT_EQ(RowIdsOf(rows), std::vector({0, 1, 2})); } Result> ScanGlobalIndexAndData( @@ -3400,6 +3780,22 @@ TEST_P(GlobalIndexTest, TestLuceneTableFullTextSearch) { {{"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); } + +TEST_P(GlobalIndexTest, TestLuceneTableFullTextSearchWithFilter) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckTableSearchWithFilter("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} + +TEST_P(GlobalIndexTest, TestLuceneTableFullTextSearchWithPartitionAndDataFilter) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckTableSearchWithPartitionAndDataFilter( + "lucene-fts", {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} #endif #ifdef PAIMON_ENABLE_FULL_TEXT @@ -3556,8 +3952,146 @@ TEST_P(GlobalIndexTest, TestFullTextSearchSkipsRowsDeletedByDeletionVectors) { TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearch) { CheckTableFullTextSearch("full-text", {{"full-text.tokenizer", "default"}}); } + +TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithFilter) { + CheckTableSearchWithFilter("full-text", {{"full-text.tokenizer", "default"}}); +} + +TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearchWithPartitionAndDataFilter) { + CheckTableSearchWithPartitionAndDataFilter("full-text", {{"full-text.tokenizer", "default"}}); +} #endif +TEST_P(GlobalIndexTest, TestFilteredRowIdReader) { + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}, + {Options::DELETION_VECTORS_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + + auto src_array = arrow::ipc::internal::json::ArrayFromJSON(arrow::struct_(fields), R"([ +["a", 0], +["b", 1], +["c", 2], +["d", 3] + ])") + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto commit_msgs, + WriteArray(table_path, schema->field_names(), src_array)); + ASSERT_OK(Commit(table_path, commit_msgs)); + + SchemaManager schema_manager(dir_->GetFileSystem(), table_path); + ASSERT_OK_AND_ASSIGN(std::optional> table_schema, + schema_manager.Latest()); + ASSERT_TRUE(table_schema); + std::shared_ptr filter = + PredicateBuilder::GreaterOrEqual(1, "f1", FieldType::INT, Literal(1)); + std::shared_ptr executor = CreateDefaultExecutor(); + auto matching_row_ids = [&](int64_t snapshot_id, const RoaringBitmap64& rows, + const std::map& search_options = {}) + -> Result { + std::map merged_options = table_schema.value()->Options(); + merged_options.insert(search_options.begin(), search_options.end()); + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(merged_options, fs_)); + FilteredRowIdReader reader(table_path, table_schema.value(), core_options, + /*partition_filter=*/nullptr, filter, snapshot_id, executor, + pool_); + return reader.MatchingRowIds(rows); + }; + + SnapshotManager snapshot_manager(dir_->GetFileSystem(), table_path); + ASSERT_OK_AND_ASSIGN(std::optional first_snapshot_id, + snapshot_manager.LatestSnapshotId()); + ASSERT_TRUE(first_snapshot_id); + ASSERT_OK_AND_ASSIGN(RoaringBitmap64 rows, matching_row_ids(first_snapshot_id.value(), + RoaringBitmap64::From({0, 1, 3}))); + ASSERT_EQ(rows, RoaringBitmap64::From({1, 3})); + ASSERT_OK_AND_ASSIGN(rows, matching_row_ids(first_snapshot_id.value(), RoaringBitmap64())); + ASSERT_TRUE(rows.IsEmpty()); + ASSERT_OK_AND_ASSIGN(rows, + matching_row_ids(first_snapshot_id.value(), RoaringBitmap64::From({100}))); + ASSERT_TRUE(rows.IsEmpty()); + + ASSERT_OK_AND_ASSIGN(auto split, ScanData(table_path, /*partition_filters=*/{})); + ASSERT_OK_AND_ASSIGN(std::vector anchor_file_names, + DeletionVectorTestHelper::RetrieveAnchorFileNames(split->DataFiles())); + ASSERT_EQ(anchor_file_names.size(), 1u); + ASSERT_OK_AND_ASSIGN(std::shared_ptr dv_commit_msg, + DeletionVectorTestHelper::CreateDeletionVectorCommitMessage( + dir_->GetFileSystem(), table_path, file_format_, commit_msgs[0], + {{anchor_file_names[0], /*deleted_positions=*/{1}}}, pool_)); + ASSERT_OK(Commit(table_path, {dv_commit_msg})); + ASSERT_OK_AND_ASSIGN(std::optional second_snapshot_id, + snapshot_manager.LatestSnapshotId()); + ASSERT_TRUE(second_snapshot_id); + + ASSERT_OK_AND_ASSIGN( + rows, matching_row_ids(second_snapshot_id.value(), RoaringBitmap64::From({1, 3}))); + ASSERT_EQ(rows, RoaringBitmap64::From({3})); + ASSERT_OK_AND_ASSIGN( + rows, matching_row_ids(first_snapshot_id.value(), RoaringBitmap64::From({1, 3}))); + ASSERT_EQ(rows, RoaringBitmap64::From({1, 3})); + + ASSERT_OK(TagSnapshot(table_path, second_snapshot_id.value(), "second")); + ASSERT_OK(TagSnapshot(table_path, first_snapshot_id.value(), "shared")); + std::map tag_options = {{Options::SCAN_TAG_NAME, "second"}}; + std::map shared_tag_options = {{Options::SCAN_TAG_NAME, "shared"}}; + ASSERT_OK_AND_ASSIGN(rows, matching_row_ids(second_snapshot_id.value(), + RoaringBitmap64::From({1, 3}), shared_tag_options)); + ASSERT_EQ(rows, RoaringBitmap64::From({3})); + + std::shared_ptr fs = dir_->GetFileSystem(); + const std::string branch = "dev"; + std::string snapshot_json; + ASSERT_OK( + fs->ReadFile(snapshot_manager.SnapshotPath(second_snapshot_id.value()), &snapshot_json)); + ASSERT_OK(fs->WriteFile(TagManager(fs, table_path, branch).TagPath("shared"), snapshot_json, + /*overwrite=*/false)); + std::string schema_file_name = fmt::format("schema-{}", table_schema.value()->Id()); + std::string schema_json; + ASSERT_OK(fs->ReadFile(PathUtil::JoinPath(schema_manager.SchemaDirectory(), schema_file_name), + &schema_json)); + ASSERT_OK( + fs->WriteFile(PathUtil::JoinPath(SchemaManager(fs, table_path, branch).SchemaDirectory(), + schema_file_name), + schema_json, /*overwrite=*/false)); + std::map branch_tag_options = {{Options::BRANCH, branch}, + {Options::SCAN_TAG_NAME, "shared"}}; + ASSERT_OK_AND_ASSIGN(rows, matching_row_ids(second_snapshot_id.value(), + RoaringBitmap64::From({1, 3}), branch_tag_options)); + ASSERT_EQ(rows, RoaringBitmap64::From({3})); + + ASSERT_OK(fs->Delete(snapshot_manager.SnapshotPath(second_snapshot_id.value()))); + ASSERT_OK_AND_ASSIGN(rows, matching_row_ids(second_snapshot_id.value(), + RoaringBitmap64::From({1, 3}), tag_options)); + ASSERT_EQ(rows, RoaringBitmap64::From({3})); + for (const auto& rows_to_match : + {RoaringBitmap64::From({1, 3}), RoaringBitmap64::From({100})}) { + ASSERT_NOK_WITH_MSG( + matching_row_ids(second_snapshot_id.value(), rows_to_match, shared_tag_options), + "as the data scan resolved another one"); + } + + const std::map& schema_options = table_schema.value()->Options(); + ASSERT_OK(TestHelper::WriteNextSchema( + fs, table_path, {DataField(0, fields[0]), DataField(1, arrow::field("g1", arrow::int32()))}, + /*highest_field_id=*/1, schema_options)); + ASSERT_OK_AND_ASSIGN( + rows, matching_row_ids(first_snapshot_id.value(), RoaringBitmap64::From({0, 1, 3}))); + ASSERT_EQ(rows, RoaringBitmap64::From({1, 3})); + ASSERT_OK(TestHelper::WriteNextSchema( + fs, table_path, {DataField(0, fields[0]), DataField(2, arrow::field("f1", arrow::int32()))}, + /*highest_field_id=*/2, schema_options)); + ASSERT_OK_AND_ASSIGN( + rows, matching_row_ids(first_snapshot_id.value(), RoaringBitmap64::From({0, 1, 3}))); + ASSERT_EQ(rows, RoaringBitmap64::From({1, 3})); +} + TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndex) { // BTreeGlobalIndexWriter requires keys to be written in monotonically increasing order. // Therefore the source data must be pre-sorted by the indexed column (f0, string).