From 83a29ba645573cb14305a061ccfa29ccc77c098f Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Fri, 23 Oct 2026 10:18:50 -0700 Subject: [PATCH] feat(global-index): support table-level full-text search for data-evolution tables Add FullTextSearchBuilder, FullTextScan, FullTextRead and serializable full-text search splits, aligned with Java FullTextSearchBuilder, DataEvolutionFullTextScan and DataEvolutionFullTextRead. - The scan picks its snapshot as Java TimeTravelUtil does: from scan.snapshot-id, scan.tag-name, scan.timestamp or scan.timestamp-millis, or else the latest one. It first checks them against scan.mode with SchemaValidation::ValidateStartupMode, now implemented as Java SchemaValidation does, so that for example scan.snapshot-id is rejected with latest-full. The latest schema of the table is still used, not that of the snapshot. - It applies the partition filter and plans one split per index shard whose indexer supports full-text search. When shards overlap, including multi-field indexes that carry the column as an extra field, each row is ranked by one shard only. A full-text or lucene-fts index whose backend is not built fails the plan instead of being skipped. - The read searches the splits in parallel and keeps the global top-k. With deletion vectors it ranks only the rows live in the snapshot the splits were planned from, loading that snapshot from a tag once it has expired. - Splits carry their snapshot id and use a versioned encoding, so scan and read can run in different processes. The encoding is specific to Paimon C++, as Java serializes its splits with Java serialization. Decoding rejects a null buffer and checks every count, range and binary row layout before reading it, and the read rejects a split whose index files no longer index the column, for example after a column of the same name replaced it. - Add GlobalIndexer::SupportsFullTextSearch(), true for full-text and lucene-fts. - Add the full-text-index.search-mode and global-index.search-mode options, resolved as Java CoreOptions#fullTextIndexSearchMode does. Only the rows that an index covers are searched, so the builder rejects the full and detail modes with NotImplemented. - Share the scored top-k merge, the row id range helpers and executor creation with the existing global index reader and scan. The merge is also compiled into paimon_global_index, whose union reader calls it. Row filters are not supported yet, so splits do not carry the scalar index files that Java uses to pre-filter rows. Columns with a primary-key full-text index are rejected with NotImplemented for now. --- docs/source/api/global_index.rst | 24 + docs/source/user_guide/global_index.rst | 125 ++++ include/paimon/defs.h | 9 + include/paimon/global_index/global_indexer.h | 7 + include/paimon/table/source/full_text_read.h | 62 ++ include/paimon/table/source/full_text_scan.h | 86 +++ .../table/source/full_text_search_builder.h | 123 ++++ .../table/source/full_text_search_split.h | 84 +++ src/paimon/CMakeLists.txt | 8 + src/paimon/common/defs.cpp | 2 + src/paimon/common/global_index/CMakeLists.txt | 1 + .../common/global_index/global_index_utils.h | 24 + .../global_index/global_index_utils_test.cpp | 18 + .../global_indexer_factory_test.cpp | 1 + .../scored_global_index_result_utils.cpp | 76 +++ .../scored_global_index_result_utils.h | 46 ++ .../union_global_index_reader.cpp | 77 +-- src/paimon/core/core_options.cpp | 19 + src/paimon/core/core_options.h | 4 + src/paimon/core/core_options_test.cpp | 23 + .../global_index/global_index_scan_impl.cpp | 37 +- .../global_index/global_index_scan_impl.h | 5 + .../core/options/global_index_search_mode.h | 33 ++ src/paimon/core/schema/schema_validation.cpp | 69 +++ src/paimon/core/schema/schema_validation.h | 10 +- .../core/schema/schema_validation_test.cpp | 50 ++ .../source/data_evolution_full_text_read.cpp | 332 +++++++++++ .../source/data_evolution_full_text_read.h | 107 ++++ .../data_evolution_full_text_read_test.cpp | 542 ++++++++++++++++++ .../source/data_evolution_full_text_scan.cpp | 327 +++++++++++ .../source/data_evolution_full_text_scan.h | 115 ++++ .../data_evolution_full_text_scan_test.cpp | 191 ++++++ .../table/source/full_text_search_builder.cpp | 271 +++++++++ .../table/source/full_text_search_split.cpp | 399 +++++++++++++ .../source/full_text_search_split_test.cpp | 377 ++++++++++++ .../index_full_text_search_split_impl.h | 109 ++++ .../full_text/full_text_global_index.h | 4 + .../full_text/full_text_global_index_test.cpp | 1 + .../global_index/lucene/lucene_global_index.h | 4 + .../lucene/lucene_global_index_test.cpp | 1 + test/inte/global_index_test.cpp | 347 ++++++++++- 41 files changed, 4061 insertions(+), 89 deletions(-) create mode 100644 include/paimon/table/source/full_text_read.h create mode 100644 include/paimon/table/source/full_text_scan.h create mode 100644 include/paimon/table/source/full_text_search_builder.h create mode 100644 include/paimon/table/source/full_text_search_split.h create mode 100644 src/paimon/common/global_index/scored_global_index_result_utils.cpp create mode 100644 src/paimon/common/global_index/scored_global_index_result_utils.h create mode 100644 src/paimon/core/options/global_index_search_mode.h create mode 100644 src/paimon/core/table/source/data_evolution_full_text_read.cpp create mode 100644 src/paimon/core/table/source/data_evolution_full_text_read.h create mode 100644 src/paimon/core/table/source/data_evolution_full_text_read_test.cpp create mode 100644 src/paimon/core/table/source/data_evolution_full_text_scan.cpp create mode 100644 src/paimon/core/table/source/data_evolution_full_text_scan.h create mode 100644 src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp create mode 100644 src/paimon/core/table/source/full_text_search_builder.cpp create mode 100644 src/paimon/core/table/source/full_text_search_split.cpp create mode 100644 src/paimon/core/table/source/full_text_search_split_test.cpp create mode 100644 src/paimon/core/table/source/index_full_text_search_split_impl.h diff --git a/docs/source/api/global_index.rst b/docs/source/api/global_index.rst index 8be63a77b..5420df989 100644 --- a/docs/source/api/global_index.rst +++ b/docs/source/api/global_index.rst @@ -64,6 +64,30 @@ Interface :members: :undoc-members: +.. doxygenclass:: paimon::FullTextSearchBuilder + :members: + :undoc-members: + +.. doxygenclass:: paimon::FullTextScan + :members: + :undoc-members: + +.. doxygenclass:: paimon::FullTextScan::Plan + :members: + :undoc-members: + +.. doxygenclass:: paimon::FullTextRead + :members: + :undoc-members: + +.. doxygenclass:: paimon::FullTextSearchSplit + :members: + :undoc-members: + +.. doxygenclass:: paimon::IndexFullTextSearchSplit + :members: + :undoc-members: + .. doxygenclass:: paimon::GlobalIndexWriteTask :members: :undoc-members: diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index c744d018c..b9168c5e0 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -291,6 +291,131 @@ and readers use that copy. The prefix-stripped options are also stored as a flat JSON object in the metadata of each index file, as Java Paimon does. +Table-Level Full-Text Search +---------------------------- + +``FullTextSearchBuilder`` (``paimon/table/source/full_text_search_builder.h``) searches the global +full-text indexes of one column of a data-evolution table, as Java Paimon's +``FullTextSearchBuilder`` does, so a caller does not create index readers or choose index types and +shard ranges. It searches the index files whose indexer supports full-text search +(``GlobalIndexer::SupportsFullTextSearch()``): ``full-text`` and ``lucene-fts``. Only the rows that +an index shard covers are searched, as in the default ``fast`` ``full-text-index.search-mode`` of +Java Paimon, so rows written since the index was last built are not found until it is built for +them. A search whose ``full-text-index.search-mode``, or else ``global-index.search-mode``, is +``full`` or ``detail``, which also search the rows that no index covers, fails with +``NotImplemented``. Row filters, which Java Paimon evaluates with scalar indexes before ranking, are +not supported yet. + +.. code-block:: cpp + + paimon::FullTextSearchBuilder search_builder(table_path); + search_builder.SetQuery("content", R"({"match":{"query":"paimon lake"}})").SetLimit(10); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + search_builder.ExecuteLocal()); + + // Read the rows found, with their scores. + paimon::ScanContextBuilder scan_builder(table_path); + scan_builder.SetGlobalIndexResult(result); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan_context, + scan_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + paimon::TableScan::Create(std::move(scan_context))); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr data_plan, scan->CreatePlan()); + + paimon::ReadContextBuilder read_builder(table_path); + read_builder.SetReadFieldNames({"id", "content", "_INDEX_SCORE"}); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read_context, + read_builder.Finish()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + paimon::TableRead::Create(std::move(read_context))); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr reader, + read->CreateReader(data_plan->Splits())); + +The result holds the ``limit`` highest-scoring rows among the candidates of all index shards, and +a read of the rows returns their scores in the ``_INDEX_SCORE`` field. Each shard returns at most +``limit`` candidates, chosen by its index, so ties are broken by ascending row id among the +candidates only: which of the rows tied at the ``limit``-th score of a shard become candidates is up +to its index. + +- ``SetPartitionFilter()`` restricts the search to some partitions, as in ``ScanContextBuilder``. +- The search reads the snapshot that the time-travel options select, as Java Paimon does: snapshot + ``scan.snapshot-id``, the snapshot of tag ``scan.tag-name``, the latest snapshot at or before + ``scan.timestamp`` or ``scan.timestamp-millis``, or else the latest snapshot. At most one of these + options may be set, and it must agree with an explicit ``scan.mode``: ``from-snapshot`` needs + ``scan.snapshot-id`` or ``scan.tag-name``, ``from-snapshot-full`` needs ``scan.snapshot-id``, + ``from-timestamp`` needs ``scan.timestamp`` or ``scan.timestamp-millis``, and ``latest-full`` and + ``latest`` take none of them. The search still uses the latest schema of the table, not the + schema of that snapshot. +- A shard whose indexed field is the column or that carries it as an extra field is searched. + When shards overlap, each row is ranked by one of them: an index whose indexed field is the + column comes first, then the order of the index type and fields, then the start and end of the + shard. +- With deletion vectors, only the rows live in that snapshot are ranked, as described for + ``GlobalIndexScan`` above. +- The shards are searched in parallel on the executor given to ``WithExecutor()``, or else on one + with ``global-index.thread-num`` threads, and each returns at most ``limit`` rows. +- If Paimon C++ is built without the backend of a ``full-text`` or ``lucene-fts`` index, the + search fails instead of skipping the index and leaving its rows out of the result. +- A column with a primary-key full-text index (``pk-full-text.index.columns``) cannot be searched + yet. + +The row ids of the result refer to the snapshot the search was planned from. If the table may +change before the rows are read, plan with ``NewFullTextScan()`` and pin the data scan to the +snapshot of the plan, setting ``scan.mode`` too so that a ``scan.mode`` of the table cannot override +it. For a search of ``scan.tag-name``, pin the data scan with the same tag instead, as the tag may +outlive the snapshot. + +.. code-block:: cpp + + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr search_scan, + search_builder.NewFullTextScan()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, + search_scan->CreatePlan()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr search_read, + search_builder.NewFullTextRead()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + search_read->Read(*plan)); + + paimon::ScanContextBuilder scan_builder(table_path); + scan_builder.SetGlobalIndexResult(result); + if (plan->SnapshotId()) { + scan_builder.AddOption(paimon::Options::SCAN_MODE, "from-snapshot") + .AddOption(paimon::Options::SCAN_SNAPSHOT_ID, std::to_string(*plan->SnapshotId())); + } + +``ExecuteLocal()`` plans and searches in one process. To plan in one process and search in others, +plan with ``NewFullTextScan()``, ship the splits with ``FullTextSearchSplit::Serialize()`` and +``FullTextSearchSplit::Deserialize()``, and search them with the ``NewFullTextRead()`` of a builder +for the same table, column and query. The serialized form is versioned and specific to Paimon C++: +Java Paimon serializes its splits with Java serialization, so splits cannot be exchanged with it. +Each split records the snapshot it was planned from, and the read ranks the rows live in that +snapshot. The read rejects a split whose index files do not index the column of the latest schema, +such as after the column was replaced. A read returns the top rows of the splits it is given, so +keep the ``limit`` highest-scoring rows of all reads if the splits are searched by several of them. + +.. code-block:: cpp + + // Planning process. + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr scan, + search_builder.NewFullTextScan()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + for (const auto& split : plan->Splits()) { + PAIMON_ASSIGN_OR_RAISE(std::string bytes, + paimon::FullTextSearchSplit::Serialize(split, pool)); + // Send `bytes` to a searching process. + } + + // Searching process. + std::vector> splits; + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr split, + paimon::FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool)); + splits.push_back(std::move(split)); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + search_builder.NewFullTextRead()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + read->Read(splits)); + Vector Index (Lumina) --------------------- diff --git a/include/paimon/defs.h b/include/paimon/defs.h index 55fd956d8..10cc3759d 100644 --- a/include/paimon/defs.h +++ b/include/paimon/defs.h @@ -614,6 +614,15 @@ struct PAIMON_EXPORT Options { /// "global-index.external-path" - Global index root directory, if not set, the global index /// files will be stored under the index directory. static const char GLOBAL_INDEX_EXTERNAL_PATH[]; + /// "global-index.search-mode" - Fallback search mode for global index queries, used when the + /// search mode of the index family is not set: "fast", "full" or "detail". No default value. + /// Paimon C++ reads it only for full-text search so far. + static const char GLOBAL_INDEX_SEARCH_MODE[]; + /// "full-text-index.search-mode" - Search mode for full-text index queries: "fast" searches + /// only the rows that an index covers, while "full" and "detail" also search the rows that no + /// index covers. Default value is "global-index.search-mode", or else "fast". Paimon C++ + /// supports only "fast" so far, and a full-text search in another mode fails. + static const char FULL_TEXT_INDEX_SEARCH_MODE[]; /// "pk-btree.index.columns" - Comma-separated columns indexed by primary-key BTree indexes. /// No default value. static const char PK_BTREE_INDEX_COLUMNS[]; diff --git a/include/paimon/global_index/global_indexer.h b/include/paimon/global_index/global_indexer.h index d7fddfa49..b7d07b8b6 100644 --- a/include/paimon/global_index/global_indexer.h +++ b/include/paimon/global_index/global_indexer.h @@ -83,6 +83,13 @@ class PAIMON_EXPORT GlobalIndexer { virtual bool SupportsCheckpoint() const { return false; } + + /// Whether readers of this indexer answer `GlobalIndexReader::VisitFullTextSearch()`, as Java + /// `GlobalIndexerFactory#supportsFullTextSearch` says. `FullTextSearchBuilder` searches only + /// the index files whose indexer supports it. + virtual bool SupportsFullTextSearch() const { + return false; + } }; } // namespace paimon diff --git a/include/paimon/table/source/full_text_read.h b/include/paimon/table/source/full_text_read.h new file mode 100644 index 000000000..2760510e6 --- /dev/null +++ b/include/paimon/table/source/full_text_read.h @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "paimon/global_index/global_index_result.h" +#include "paimon/result.h" +#include "paimon/table/source/full_text_scan.h" +#include "paimon/table/source/full_text_search_split.h" +#include "paimon/visibility.h" + +namespace paimon { +/// Searches the splits of a table-level full-text search, aligned with Java +/// `org.apache.paimon.table.source.FullTextRead`. Create one with +/// `FullTextSearchBuilder::NewFullTextRead()`. +class PAIMON_EXPORT FullTextRead { + public: + virtual ~FullTextRead() = default; + + /// Searches the splits of `plan`; see `Read(splits)`. + Result> Read(const FullTextScan::Plan& plan) { + return Read(plan.Splits()); + } + + /// Searches `splits` and returns the `limit` highest-scoring rows among the candidates that + /// they return. The result holds global row ids and iterates them in ascending order; pass it + /// to `ScanContextBuilder::SetGlobalIndexResult()` to read the rows, whose scores the read + /// returns in the `_INDEX_SCORE` field. + /// + /// The splits are searched in parallel, and each one returns at most `limit` candidates, chosen + /// by its index. Ties among the candidates are broken by ascending row id, but which of the + /// rows tied at the `limit`-th score of a split become candidates is up to its index. A split + /// ranks only its search row ranges, and with deletion vectors enabled only the rows that are + /// live in the snapshot the splits were planned from; BM25 statistics still come from its whole + /// index shard. + /// @param splits Splits planned by a `FullTextScan` of the same column, all from one snapshot. + /// @return The rows found, or an error if a split is invalid, its index files do not index the + /// column of the latest schema (for example, after the column was replaced), or an + /// index cannot be read. + virtual Result> Read( + const std::vector>& splits) = 0; +}; + +} // namespace paimon diff --git a/include/paimon/table/source/full_text_scan.h b/include/paimon/table/source/full_text_scan.h new file mode 100644 index 000000000..8e0d17b86 --- /dev/null +++ b/include/paimon/table/source/full_text_scan.h @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/result.h" +#include "paimon/table/source/full_text_search_split.h" +#include "paimon/visibility.h" + +namespace paimon { +/// Plans a table-level full-text search into splits, aligned with Java +/// `org.apache.paimon.table.source.FullTextScan`. Create one with +/// `FullTextSearchBuilder::NewFullTextScan()`. +class PAIMON_EXPORT FullTextScan { + public: + /// The splits of a full-text search and the snapshot they were planned from. + class Plan { + public: + Plan(const std::optional& snapshot_id, + std::vector> splits) + : snapshot_id_(snapshot_id), splits_(std::move(splits)) {} + + /// @return The splits to search. They can be read in this process or serialized and read + /// in others. + const std::vector>& Splits() const { + return splits_; + } + + /// @return The id of the snapshot the plan was built from, or nullopt if the table has no + /// snapshot. + const std::optional& SnapshotId() const { + return snapshot_id_; + } + + private: + std::optional snapshot_id_; + std::vector> splits_; + }; + + virtual ~FullTextScan() = default; + + /// Plans the search on the snapshot that the options select, as Java + /// `TimeTravelUtil#tryTravelOrLatest` does: snapshot `scan.snapshot-id`, the snapshot of tag + /// `scan.tag-name`, the latest snapshot at or before `scan.timestamp` or + /// `scan.timestamp-millis`, or else the latest snapshot. As in Java, at most one of these + /// options may be set, and it must agree with an explicit `scan.mode`: `from-snapshot` needs + /// `scan.snapshot-id` or `scan.tag-name`, `from-snapshot-full` needs `scan.snapshot-id`, + /// `from-timestamp` needs `scan.timestamp` or `scan.timestamp-millis`, and `latest-full` and + /// `latest` take none of them. + /// + /// Each index shard of the searched column becomes one split. The scan considers only the + /// index files that are in the partitions of the partition filter, that index the column as + /// their indexed field or as one of their extra fields, and whose indexer supports full-text + /// search (`GlobalIndexer::SupportsFullTextSearch()`). The plan fails if Paimon C++ is built + /// without the backend of a `full-text` or `lucene-fts` index it finds. When the row ranges of + /// index shards overlap, each row is ranked by one shard only: an index whose indexed field is + /// the column comes before one that carries it as an extra field, and further ties are broken + /// by index type and fields, then by the start and end of the shard. + /// @return The plan, or an error if more than one time-travel option is set, the option does + /// not agree with `scan.mode`, the selected snapshot or tag does not exist, or reading + /// the index manifests fails; `NotImplemented` for an index whose backend is not built. + virtual Result> CreatePlan() = 0; +}; + +} // namespace paimon diff --git a/include/paimon/table/source/full_text_search_builder.h b/include/paimon/table/source/full_text_search_builder.h new file mode 100644 index 000000000..2d18388c8 --- /dev/null +++ b/include/paimon/table/source/full_text_search_builder.h @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/global_index/global_index_result.h" +#include "paimon/result.h" +#include "paimon/table/source/full_text_read.h" +#include "paimon/table/source/full_text_scan.h" +#include "paimon/type_fwd.h" +#include "paimon/visibility.h" + +namespace paimon { +class Executor; + +/// Builds a table-level full-text search of a data-evolution table, aligned with Java +/// `org.apache.paimon.table.source.FullTextSearchBuilder`. It searches the global full-text +/// indexes of one column and returns the highest-scoring rows: +/// +/// @code +/// FullTextSearchBuilder builder(table_path); +/// builder.SetQuery("content", R"({"match":{"query":"paimon lake"}})").SetLimit(10); +/// PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, +/// builder.ExecuteLocal()); +/// ScanContextBuilder scan_builder(table_path); +/// scan_builder.SetGlobalIndexResult(result); +/// @endcode +/// +/// `ExecuteLocal()` plans and searches in this process. To run them in different processes, plan +/// with `NewFullTextScan()`, serialize the splits with `FullTextSearchSplit::Serialize()`, and +/// search them with the `NewFullTextRead()` of a builder for the same table, column and query. +/// +/// The row ids of the result refer to the snapshot the search was planned from, which +/// `FullTextScan::Plan::SnapshotId()` returns. If the table may change in between, pin the data +/// scan to it with `scan.mode` = `from-snapshot` and `scan.snapshot-id`, or with the same +/// `scan.tag-name` for a search of a tag, as the tag may outlive the snapshot. +class PAIMON_EXPORT FullTextSearchBuilder { + public: + /// @param table_path The root path of the table. + explicit FullTextSearchBuilder(const std::string& table_path); + + ~FullTextSearchBuilder(); + + /// Restricts the search to the given partitions. Each map in the vector is one partition + /// spec whose entries are ANDed, and the specs are ORed, as in + /// `ScanContextBuilder::SetPartitionFilter()`. If not set, every partition is searched. + FullTextSearchBuilder& SetPartitionFilter( + const std::vector>& partition_filters); + + /// Sets the number of highest-scoring rows to return. It must be positive to create a read. + FullTextSearchBuilder& SetLimit(int32_t limit); + + /// Sets the column to search and the JSON query of `FullTextSearch::query`, which the index + /// receives unchanged. + FullTextSearchBuilder& SetQuery(const std::string& field_name, const std::string& query); + + /// Adds an option that overrides the one of the same key in the table schema. Use one of + /// `scan.snapshot-id`, `scan.tag-name`, `scan.timestamp` and `scan.timestamp-millis` to search + /// an earlier snapshot; it must agree with `scan.mode`, as `FullTextScan::CreatePlan()` + /// describes. + FullTextSearchBuilder& AddOption(const std::string& key, const std::string& value); + + /// Sets the options that override the options in the table schema, replacing the options + /// added before. + FullTextSearchBuilder& SetOptions(const std::map& options); + + /// Sets the memory pool. If not set, the default pool is used. + FullTextSearchBuilder& WithMemoryPool(const std::shared_ptr& memory_pool); + + /// Sets the executor that searches splits in parallel and reads manifests. If not set, each + /// read creates an executor with `global-index.thread-num` threads, which defaults to the + /// number of CPUs. + FullTextSearchBuilder& WithExecutor(const std::shared_ptr& executor); + + /// Sets the file system to access the table with. If not set, the file system is created from + /// the `file-system` option. + FullTextSearchBuilder& WithFileSystem(const std::shared_ptr& file_system); + + /// Creates a scan that plans the search on the current settings. The query must be set, and + /// its column must exist in the latest schema of the table. + /// @return The scan, or an error if the settings are invalid or the table cannot be loaded; + /// `NotImplemented` for a primary-key full-text index (`pk-full-text.index.columns`) or + /// a search mode other than `fast` (`full-text-index.search-mode`, or else + /// `global-index.search-mode`). + Result> NewFullTextScan() const; + + /// Creates a read that searches the splits of a scan on the current settings. In addition to + /// the requirements of `NewFullTextScan()`, the limit must be positive. + /// @return The read, or an error if the settings are invalid or the table cannot be loaded. + Result> NewFullTextRead() const; + + /// Plans and searches in this process: `NewFullTextRead()->Read(plan)` with the plan of + /// `NewFullTextScan()`. + Result> ExecuteLocal() const; + + private: + class Impl; + + std::unique_ptr impl_; +}; + +} // namespace paimon diff --git a/include/paimon/table/source/full_text_search_split.h b/include/paimon/table/source/full_text_search_split.h new file mode 100644 index 000000000..839c12e6a --- /dev/null +++ b/include/paimon/table/source/full_text_search_split.h @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/memory/memory_pool.h" +#include "paimon/result.h" +#include "paimon/utils/range.h" +#include "paimon/visibility.h" + +namespace paimon { +/// A unit of work of a table-level full-text search, planned by `FullTextScan` and searched by +/// `FullTextRead`, aligned with Java `org.apache.paimon.table.source.FullTextSearchSplit`. +/// +/// Splits can be serialized, so that a search can be planned in one process and run in others. +/// The encoding is specific to Paimon C++: Java Paimon serializes its splits with Java +/// serialization, so there is no cross-language format to match. +class PAIMON_EXPORT FullTextSearchSplit { + public: + virtual ~FullTextSearchSplit() = default; + + /// Serializes `split` to a binary string. + /// @param split The split to serialize; it must come from `FullTextScan` or `Deserialize()`. + /// @param pool Memory pool for temporary allocations. + static Result Serialize(const std::shared_ptr& split, + const std::shared_ptr& pool); + + /// Deserializes a split written by `Serialize()`. + /// @param buffer Pointer to the serialized bytes. + /// @param length Number of serialized bytes. + /// @param pool Memory pool for the objects of the split. + /// @return The split, or an error if `buffer` is null, or the bytes are corrupt or not a split + /// of a supported version. + static Result> Deserialize( + const char* buffer, size_t length, const std::shared_ptr& pool); +}; + +/// A split that searches the full-text index files of one index shard, aligned with Java +/// `org.apache.paimon.table.source.IndexFullTextSearchSplit`. +/// +/// When index shards overlap, the scan gives each row to one shard, so a split may rank only some +/// rows of its shard: `SearchRowRanges()` lie within [`RowRangeStart()`, `RowRangeEnd()`], the +/// physical range of the shard, which maps the shard-local row ids of the index to global ones. +class PAIMON_EXPORT IndexFullTextSearchSplit : public FullTextSearchSplit { + public: + /// @return The id of the snapshot the split was planned from. The read ranks only the rows + /// live in this snapshot. + virtual int64_t SnapshotId() const = 0; + + /// @return The name of the searched table column. + virtual const std::string& ColumnName() const = 0; + + /// @return The first global row id of the index shard. + virtual int64_t RowRangeStart() const = 0; + + /// @return The last global row id of the index shard, inclusive. + virtual int64_t RowRangeEnd() const = 0; + + /// @return The sorted, disjoint global row id ranges that this split ranks. + virtual const std::vector& SearchRowRanges() const = 0; +}; + +} // namespace paimon diff --git a/src/paimon/CMakeLists.txt b/src/paimon/CMakeLists.txt index b4ad6e80f..3c635bdea 100644 --- a/src/paimon/CMakeLists.txt +++ b/src/paimon/CMakeLists.txt @@ -85,6 +85,7 @@ set(PAIMON_COMMON_SRCS common/fs/credential_provider_factory.cpp common/fs/file_system_factory.cpp common/global_index/union_global_index_reader.cpp + common/global_index/scored_global_index_result_utils.cpp common/global_index/offset_global_index_reader.cpp common/global_index/bitmap_scored_global_index_result.cpp common/global_index/bitmap_global_index_result.cpp @@ -478,6 +479,10 @@ set(PAIMON_CORE_SRCS core/table/source/table_read.cpp core/table/source/table_scan.cpp core/table/source/data_evolution_batch_scan.cpp + core/table/source/data_evolution_full_text_read.cpp + core/table/source/data_evolution_full_text_scan.cpp + core/table/source/full_text_search_builder.cpp + core/table/source/full_text_search_split.cpp core/table/source/primary_key_sorted_index_scan.cpp core/table/source/primary_key_sorted_index_result.cpp core/table/source/primary_key_index_batch_scan.cpp @@ -1022,6 +1027,9 @@ if(PAIMON_BUILD_TESTS) core/table/source/append_count_reader_test.cpp core/table/source/pk_count_reader_test.cpp core/table/source/data_evolution_batch_scan_test.cpp + core/table/source/data_evolution_full_text_read_test.cpp + core/table/source/data_evolution_full_text_scan_test.cpp + core/table/source/full_text_search_split_test.cpp core/table/source/data_split_test.cpp core/table/source/deletion_file_test.cpp core/table/source/split_generator_test.cpp diff --git a/src/paimon/common/defs.cpp b/src/paimon/common/defs.cpp index 753dbb43d..8edcec3d3 100644 --- a/src/paimon/common/defs.cpp +++ b/src/paimon/common/defs.cpp @@ -162,6 +162,8 @@ const char Options::BLOB_WRITE_NULL_ON_FETCH_FAILURE[] = "blob-write-null-on-fet const char Options::GLOBAL_INDEX_ENABLED[] = "global-index.enabled"; const char Options::GLOBAL_INDEX_THREAD_NUM[] = "global-index.thread-num"; const char Options::GLOBAL_INDEX_EXTERNAL_PATH[] = "global-index.external-path"; +const char Options::GLOBAL_INDEX_SEARCH_MODE[] = "global-index.search-mode"; +const char Options::FULL_TEXT_INDEX_SEARCH_MODE[] = "full-text-index.search-mode"; const char Options::PK_BTREE_INDEX_COLUMNS[] = "pk-btree.index.columns"; const char Options::PK_BITMAP_INDEX_COLUMNS[] = "pk-bitmap.index.columns"; const char Options::PK_VECTOR_INDEX_COLUMNS[] = "pk-vector.index.columns"; diff --git a/src/paimon/common/global_index/CMakeLists.txt b/src/paimon/common/global_index/CMakeLists.txt index aa1fe82b7..a70ec81ca 100644 --- a/src/paimon/common/global_index/CMakeLists.txt +++ b/src/paimon/common/global_index/CMakeLists.txt @@ -33,6 +33,7 @@ set(PAIMON_GLOBAL_INDEX_SRC rangebitmap/range_bitmap_global_index.cpp rangebitmap/range_bitmap_global_index_factory.cpp offset_global_index_reader.cpp + scored_global_index_result_utils.cpp union_global_index_reader.cpp) add_paimon_lib(paimon_global_index diff --git a/src/paimon/common/global_index/global_index_utils.h b/src/paimon/common/global_index/global_index_utils.h index 0aed10140..a9676538b 100644 --- a/src/paimon/common/global_index/global_index_utils.h +++ b/src/paimon/common/global_index/global_index_utils.h @@ -25,6 +25,8 @@ #include "fmt/format.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/status.h" +#include "paimon/utils/range.h" +#include "paimon/utils/roaring_bitmap64.h" namespace paimon { class GlobalIndexUtils { public: @@ -54,5 +56,27 @@ class GlobalIndexUtils { guard.Release(); return Status::OK(); } + + /// Returns the row ids of `range`. + static RoaringBitmap64 RowIdsOf(const Range& range) { + // `range.to` may be the largest int64 value, so add it apart from the half-open range. + RoaringBitmap64 row_ids; + row_ids.AddRange(range.from, range.to); + row_ids.Add(range.to); + return row_ids; + } + + /// Returns the row ids of `row_ids` within `range`. Trimming a copy costs time in proportion to + /// the containers of `row_ids`, not to the rows or the span of `range`. + static RoaringBitmap64 RowIdsWithin(const RoaringBitmap64& row_ids, const Range& range) { + RoaringBitmap64 result = row_ids; + result.RemoveRange(0, range.from); + if (range.to < RoaringBitmap64::MAX_VALUE) { + // RemoveRange leaves out its end, so the largest int64 value is removed on its own. + result.RemoveRange(range.to + 1, RoaringBitmap64::MAX_VALUE); + result -= RoaringBitmap64::From({RoaringBitmap64::MAX_VALUE}); + } + return result; + } }; } // namespace paimon diff --git a/src/paimon/common/global_index/global_index_utils_test.cpp b/src/paimon/common/global_index/global_index_utils_test.cpp index 7354bcedb..3894e74eb 100644 --- a/src/paimon/common/global_index/global_index_utils_test.cpp +++ b/src/paimon/common/global_index/global_index_utils_test.cpp @@ -19,6 +19,8 @@ #include "paimon/common/global_index/global_index_utils.h" +#include +#include #include #include @@ -26,6 +28,8 @@ #include "arrow/c/bridge.h" #include "gtest/gtest.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" +#include "paimon/utils/roaring_bitmap64.h" namespace paimon::test { @@ -110,4 +114,18 @@ TEST_F(GlobalIndexUtilsTest, TestMismatchedExpectedNextRowId) { "first relative_row_ids 5 mismatch inner expected_next_row_id 100 in CheckRelativeRowIds"); } +TEST_F(GlobalIndexUtilsTest, TestRowIdsOfAndWithin) { + const int64_t max_row_id = std::numeric_limits::max(); + ASSERT_EQ(GlobalIndexUtils::RowIdsOf(Range(3, 5)), RoaringBitmap64::From({3, 4, 5})); + ASSERT_EQ(GlobalIndexUtils::RowIdsOf(Range(max_row_id - 1, max_row_id)), + RoaringBitmap64::From({max_row_id - 1, max_row_id})); + + RoaringBitmap64 row_ids = RoaringBitmap64::From({0, 3, 7, 10, max_row_id}); + ASSERT_EQ(GlobalIndexUtils::RowIdsWithin(row_ids, Range(3, 7)), RoaringBitmap64::From({3, 7})); + ASSERT_EQ(GlobalIndexUtils::RowIdsWithin(row_ids, Range(8, max_row_id)), + RoaringBitmap64::From({10, max_row_id})); + ASSERT_EQ(GlobalIndexUtils::RowIdsWithin(row_ids, Range(4, 6)), RoaringBitmap64()); + ASSERT_EQ(row_ids, RoaringBitmap64::From({0, 3, 7, 10, max_row_id})); +} + } // namespace paimon::test diff --git a/src/paimon/common/global_index/global_indexer_factory_test.cpp b/src/paimon/common/global_index/global_indexer_factory_test.cpp index 0ca10fc33..4e94c753f 100644 --- a/src/paimon/common/global_index/global_indexer_factory_test.cpp +++ b/src/paimon/common/global_index/global_indexer_factory_test.cpp @@ -31,6 +31,7 @@ TEST(GlobalIndexerFactoryTest, TestBitmap) { ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, GlobalIndexerFactory::Get("bitmap", options)); ASSERT_TRUE(dynamic_cast(indexer.get())); + ASSERT_FALSE(indexer->SupportsFullTextSearch()); } TEST(GlobalIndexerFactoryTest, TestNonExist) { diff --git a/src/paimon/common/global_index/scored_global_index_result_utils.cpp b/src/paimon/common/global_index/scored_global_index_result_utils.cpp new file mode 100644 index 000000000..fd8c39991 --- /dev/null +++ b/src/paimon/common/global_index/scored_global_index_result_utils.cpp @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/common/global_index/scored_global_index_result_utils.h" + +#include +#include +#include + +#include "paimon/common/utils/fields_comparator.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/utils/roaring_bitmap64.h" + +namespace paimon { +namespace { +/// Returns whether `left` ranks before `right`: a higher score first, then a lower row id. Scores +/// are ordered as -infinity < -0.0 < +0.0 < +infinity < NaN. +bool RanksBefore(const std::pair& left, const std::pair& right) { + int32_t score_order = FieldsComparator::CompareFloatingPoint(left.second, right.second); + if (score_order != 0) { + return score_order > 0; + } + return left.first < right.first; +} +} // namespace + +Result> ScoredGlobalIndexResultUtils::MergeTopK( + const std::vector>& results, int32_t limit) { + if (limit <= 0) { + return Status::Invalid("merging scored results requires a positive limit"); + } + std::unordered_set merged_row_ids; + std::vector> rows; + for (const auto& result : results) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iter, + result->CreateScoredIterator()); + while (iter->HasNext()) { + std::pair row = iter->NextWithScore(); + if (merged_row_ids.insert(row.first).second) { + rows.push_back(row); + } + } + } + auto top_k = static_cast(limit); + if (rows.size() > top_k) { + std::nth_element(rows.begin(), rows.begin() + top_k, rows.end(), RanksBefore); + rows.resize(top_k); + } + std::sort(rows.begin(), rows.end()); + RoaringBitmap64 bitmap; + std::vector scores; + scores.reserve(rows.size()); + for (const auto& [row_id, score] : rows) { + bitmap.Add(row_id); + scores.push_back(score); + } + return std::make_shared(std::move(bitmap), std::move(scores)); +} + +} // namespace paimon diff --git a/src/paimon/common/global_index/scored_global_index_result_utils.h b/src/paimon/common/global_index/scored_global_index_result_utils.h new file mode 100644 index 000000000..fc4e1aca5 --- /dev/null +++ b/src/paimon/common/global_index/scored_global_index_result_utils.h @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include + +#include "paimon/global_index/global_index_result.h" +#include "paimon/result.h" + +namespace paimon { +/// Helpers that combine `ScoredGlobalIndexResult`s. +class ScoredGlobalIndexResultUtils { + public: + ScoredGlobalIndexResultUtils() = delete; + ~ScoredGlobalIndexResultUtils() = delete; + + /// Merges `results` and keeps the `limit` rows that rank first: a higher score first, then a + /// lower row id, with scores ordered as -infinity < -0.0 < +0.0 < +infinity < NaN. A row id in + /// more than one result keeps its score from the first of those results. This is Java + /// `ScoredGlobalIndexResult#merge` followed by `ScoredGlobalIndexResult#topK`. + /// @param results The results to merge; none may be null. + /// @param limit The number of rows to keep; must be positive. + static Result> MergeTopK( + const std::vector>& results, int32_t limit); +}; + +} // namespace paimon diff --git a/src/paimon/common/global_index/union_global_index_reader.cpp b/src/paimon/common/global_index/union_global_index_reader.cpp index bb11cae39..c6744f234 100644 --- a/src/paimon/common/global_index/union_global_index_reader.cpp +++ b/src/paimon/common/global_index/union_global_index_reader.cpp @@ -19,14 +19,13 @@ #include "paimon/common/global_index/union_global_index_reader.h" -#include #include #include -#include #include #include "paimon/common/executor/future.h" -#include "paimon/common/utils/fields_comparator.h" +#include "paimon/common/global_index/global_index_utils.h" +#include "paimon/common/global_index/scored_global_index_result_utils.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" #include "paimon/predicate/full_text_search.h" #include "paimon/utils/roaring_bitmap64.h" @@ -37,68 +36,6 @@ std::shared_ptr EmptyScoredResult() { return std::make_shared(RoaringBitmap64(), std::vector()); } -RoaringBitmap64 RowIdsOf(const Range& range) { - // `range.to` may be the largest int64 value, so add it apart from the half-open range. - RoaringBitmap64 row_ids; - row_ids.AddRange(range.from, range.to); - row_ids.Add(range.to); - return row_ids; -} - -/// Returns the row ids of `row_ids` within `range`. Trimming a copy costs time in proportion to -/// the containers of `row_ids`, not to the rows or the span of `range`. -RoaringBitmap64 RowIdsWithin(const RoaringBitmap64& row_ids, const Range& range) { - RoaringBitmap64 result = row_ids; - result.RemoveRange(0, range.from); - if (range.to < RoaringBitmap64::MAX_VALUE) { - // RemoveRange leaves out its end, so the largest int64 value is removed on its own. - result.RemoveRange(range.to + 1, RoaringBitmap64::MAX_VALUE); - result -= RoaringBitmap64::From({RoaringBitmap64::MAX_VALUE}); - } - return result; -} - -/// Returns whether `left` ranks before `right`: a higher score first, then a lower row id. Scores -/// are ordered as -infinity < -0.0 < +0.0 < +infinity < NaN. -bool RanksBefore(const std::pair& left, const std::pair& right) { - int32_t score_order = FieldsComparator::CompareFloatingPoint(left.second, right.second); - if (score_order != 0) { - return score_order > 0; - } - return left.first < right.first; -} - -/// Merges scored results and keeps the `limit` rows that rank first. A row id in more than one -/// result keeps its first score. -Result> MergeTopK( - const std::vector>& results, int32_t limit) { - std::unordered_set merged_row_ids; - std::vector> rows; - for (const auto& result : results) { - PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iter, - result->CreateScoredIterator()); - while (iter->HasNext()) { - std::pair row = iter->NextWithScore(); - if (merged_row_ids.insert(row.first).second) { - rows.push_back(row); - } - } - } - auto top_k = static_cast(limit); - if (rows.size() > top_k) { - std::nth_element(rows.begin(), rows.begin() + top_k, rows.end(), RanksBefore); - rows.resize(top_k); - } - std::sort(rows.begin(), rows.end()); - RoaringBitmap64 bitmap; - std::vector scores; - scores.reserve(rows.size()); - for (const auto& [row_id, score] : rows) { - bitmap.Add(row_id); - scores.push_back(score); - } - return std::make_shared(std::move(bitmap), std::move(scores)); -} } // namespace UnionGlobalIndexReader::UnionGlobalIndexReader( @@ -251,7 +188,7 @@ Result> UnionGlobalIndexReader::VisitFu if (scored_results.empty()) { return std::shared_ptr(); } - return MergeTopK(scored_results, full_text_search->limit); + return ScoredGlobalIndexResultUtils::MergeTopK(scored_results, full_text_search->limit); } std::shared_ptr UnionGlobalIndexReader::FullTextSearchForReader( @@ -270,7 +207,7 @@ std::shared_ptr UnionGlobalIndexReader::FullTextSearchForReader( } return std::make_shared(full_text_search->field_name, full_text_search->query, full_text_search->limit, - RowIdsOf(search_range.value())); + GlobalIndexUtils::RowIdsOf(search_range.value())); } const RoaringBitmap64& include_row_ids = full_text_search->include_row_ids.value(); auto first = include_row_ids.EqualOrLarger(search_range->from); @@ -281,9 +218,9 @@ std::shared_ptr UnionGlobalIndexReader::FullTextSearchForReader( if (ranks_all_rows) { return full_text_search; } - return std::make_shared(full_text_search->field_name, full_text_search->query, - full_text_search->limit, - RowIdsWithin(include_row_ids, search_range.value())); + return std::make_shared( + full_text_search->field_name, full_text_search->query, full_text_search->limit, + GlobalIndexUtils::RowIdsWithin(include_row_ids, search_range.value())); } Result> UnionGlobalIndexReader::UnionScored( diff --git a/src/paimon/core/core_options.cpp b/src/paimon/core/core_options.cpp index c5018f6aa..6b8666a30 100644 --- a/src/paimon/core/core_options.cpp +++ b/src/paimon/core/core_options.cpp @@ -178,6 +178,18 @@ class ConfigParser { "sort engine", sort_engine); } + // Parse GlobalIndexSearchMode of full-text search, as Java `CoreOptions#indexSearchMode` does + Status ParseFullTextIndexSearchMode(GlobalIndexSearchMode* search_mode) const { + const char* key = config_map_.count(Options::FULL_TEXT_INDEX_SEARCH_MODE) > 0 + ? Options::FULL_TEXT_INDEX_SEARCH_MODE + : Options::GLOBAL_INDEX_SEARCH_MODE; + return ParseEnum(key, + {{"fast", GlobalIndexSearchMode::FAST}, + {"full", GlobalIndexSearchMode::FULL}, + {"detail", GlobalIndexSearchMode::DETAIL}}, + key, search_mode); + } + // Parse MergeEngine Status ParseMergeEngine(MergeEngine* merge_engine) const { return ParseEnum(Options::MERGE_ENGINE, @@ -437,6 +449,7 @@ struct CoreOptions::Impl { std::optional num_levels; std::optional lookup_compact_max_interval; std::optional global_index_thread_num; + GlobalIndexSearchMode full_text_index_search_mode = GlobalIndexSearchMode::FAST; bool realtime_enabled = false; bool realtime_spill_enabled = false; @@ -913,6 +926,8 @@ struct CoreOptions::Impl { // index, no default value PAIMON_RETURN_NOT_OK( parser.Parse(Options::GLOBAL_INDEX_THREAD_NUM, &global_index_thread_num)); + // Parse full-text-index.search-mode, falling back to global-index.search-mode, default fast + PAIMON_RETURN_NOT_OK(parser.ParseFullTextIndexSearchMode(&full_text_index_search_mode)); // Parse global-index.external-path - global index root directory PAIMON_RETURN_NOT_OK( parser.Parse(Options::GLOBAL_INDEX_EXTERNAL_PATH, &global_index_external_path)); @@ -1901,6 +1916,10 @@ std::optional CoreOptions::GetGlobalIndexThreadNum() const { return impl_->global_index_thread_num; } +GlobalIndexSearchMode CoreOptions::GetFullTextIndexSearchMode() const { + return impl_->full_text_index_search_mode; +} + std::optional CoreOptions::GetGlobalIndexExternalPath() const { return impl_->global_index_external_path; } diff --git a/src/paimon/core/core_options.h b/src/paimon/core/core_options.h index 61562deb8..136490362 100644 --- a/src/paimon/core/core_options.h +++ b/src/paimon/core/core_options.h @@ -31,6 +31,7 @@ #include "paimon/core/options/changelog_producer.h" #include "paimon/core/options/compress_options.h" #include "paimon/core/options/external_path_strategy.h" +#include "paimon/core/options/global_index_search_mode.h" #include "paimon/core/options/lookup_compact_mode.h" #include "paimon/core/options/lookup_strategy.h" #include "paimon/core/options/map_shared_shredding_column_placement_policy.h" @@ -303,6 +304,9 @@ class PAIMON_EXPORT CoreOptions { BucketFunctionType GetBucketFunctionType() const; std::optional GetGlobalIndexThreadNum() const; + /// Returns `full-text-index.search-mode`, or else `global-index.search-mode`, or else + /// `GlobalIndexSearchMode::FAST`, as Java `CoreOptions#fullTextIndexSearchMode` does. + GlobalIndexSearchMode GetFullTextIndexSearchMode() const; const std::vector& GetBlobFields() const; const std::vector& GetBlobDescriptorFields() const; diff --git a/src/paimon/core/core_options_test.cpp b/src/paimon/core/core_options_test.cpp index 31750da0d..64c43a346 100644 --- a/src/paimon/core/core_options_test.cpp +++ b/src/paimon/core/core_options_test.cpp @@ -19,6 +19,7 @@ #include "paimon/core/core_options.h" #include +#include #include #include #include @@ -1054,6 +1055,28 @@ TEST(CoreOptionsTest, TestParseChangelogProducer) { "invalid changelog producer: invalid"); } +TEST(CoreOptionsTest, TestParseFullTextIndexSearchMode) { + auto search_mode = + [](const std::map& options) -> Result { + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); + return core_options.GetFullTextIndexSearchMode(); + }; + ASSERT_OK_AND_ASSIGN(GlobalIndexSearchMode mode, search_mode({})); + ASSERT_EQ(mode, GlobalIndexSearchMode::FAST); + ASSERT_OK_AND_ASSIGN(mode, search_mode({{Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); + ASSERT_EQ(mode, GlobalIndexSearchMode::FULL); + ASSERT_OK_AND_ASSIGN(mode, search_mode({{Options::FULL_TEXT_INDEX_SEARCH_MODE, "DETAIL"}})); + ASSERT_EQ(mode, GlobalIndexSearchMode::DETAIL); + ASSERT_OK_AND_ASSIGN(mode, search_mode({{Options::FULL_TEXT_INDEX_SEARCH_MODE, "fast"}, + {Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); + ASSERT_EQ(mode, GlobalIndexSearchMode::FAST); + ASSERT_NOK_WITH_MSG(search_mode({{Options::FULL_TEXT_INDEX_SEARCH_MODE, "invalid"}, + {Options::GLOBAL_INDEX_SEARCH_MODE, "fast"}}), + "invalid full-text-index.search-mode: invalid"); + ASSERT_NOK_WITH_MSG(search_mode({{Options::GLOBAL_INDEX_SEARCH_MODE, "invalid"}}), + "invalid global-index.search-mode: invalid"); +} + TEST(CoreOptionsTest, TestParseExternalPathStrategy) { { ASSERT_OK_AND_ASSIGN( diff --git a/src/paimon/core/global_index/global_index_scan_impl.cpp b/src/paimon/core/global_index/global_index_scan_impl.cpp index b20437d4a..8cc1c77a9 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.cpp +++ b/src/paimon/core/global_index/global_index_scan_impl.cpp @@ -52,26 +52,33 @@ GlobalIndexScanImpl::GlobalIndexScanImpl( live_rows_supplier_(std::move(live_rows_supplier)), executor_(executor) {} +Result> GlobalIndexScanImpl::GetOrCreateExecutor( + const std::shared_ptr& executor, const CoreOptions& options) { + if (executor) { + return executor; + } + std::optional thread_num = options.GetGlobalIndexThreadNum(); + if (thread_num) { + if (thread_num.value() <= 0) { + return Status::Invalid( + fmt::format("invalid global index thread number {}", thread_num.value())); + } + } else { + uint32_t cpu_count = std::thread::hardware_concurrency(); + thread_num = cpu_count > 0 ? static_cast(cpu_count) : 1; + } + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr created_executor, + CreateDefaultExecutor(static_cast(thread_num.value()))); + return std::shared_ptr(std::move(created_executor)); +} + Result> GlobalIndexScanImpl::Create( const std::string& root_path, const std::shared_ptr& table_schema, const Snapshot& snapshot, const std::shared_ptr& partitions, const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool) { - auto final_executor = executor; - if (!final_executor) { - std::optional thread_num = options.GetGlobalIndexThreadNum(); - if (thread_num) { - if (thread_num.value() <= 0) { - return Status::Invalid( - fmt::format("invalid global index thread number {}", thread_num.value())); - } - } else { - uint32_t cpu_count = std::thread::hardware_concurrency(); - thread_num = cpu_count > 0 ? static_cast(cpu_count) : 1; - } - PAIMON_ASSIGN_OR_RAISE(final_executor, - CreateDefaultExecutor(static_cast(thread_num.value()))); - } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr final_executor, + GetOrCreateExecutor(executor, options)); auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, diff --git a/src/paimon/core/global_index/global_index_scan_impl.h b/src/paimon/core/global_index/global_index_scan_impl.h index bb26ae8ee..364228b53 100644 --- a/src/paimon/core/global_index/global_index_scan_impl.h +++ b/src/paimon/core/global_index/global_index_scan_impl.h @@ -46,6 +46,11 @@ class GlobalIndexScanImpl : public GlobalIndexScan { const CoreOptions& options, const std::shared_ptr& executor, const std::shared_ptr& pool); + /// Returns `executor` if it is not null, or else creates an executor with + /// `global-index.thread-num` threads, which defaults to the number of CPUs. + static Result> GetOrCreateExecutor( + const std::shared_ptr& executor, const CoreOptions& options); + Result> Scan(const std::shared_ptr& predicate); Result>> CreateReaders( diff --git a/src/paimon/core/options/global_index_search_mode.h b/src/paimon/core/options/global_index_search_mode.h new file mode 100644 index 000000000..11aa45405 --- /dev/null +++ b/src/paimon/core/options/global_index_search_mode.h @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +namespace paimon { +/// Specifies which rows a global index search covers, as Java +/// `CoreOptions.GlobalIndexSearchMode` does. +enum class GlobalIndexSearchMode { + // Only search indexed data. + FAST = 1, + // Also scan the raw data of the row ids that the global index does not cover. + FULL = 2, + // Scan data files to find the exact unindexed rows. + DETAIL = 3 +}; +} // namespace paimon diff --git a/src/paimon/core/schema/schema_validation.cpp b/src/paimon/core/schema/schema_validation.cpp index 7717b27fb..6e49182c8 100644 --- a/src/paimon/core/schema/schema_validation.cpp +++ b/src/paimon/core/schema/schema_validation.cpp @@ -70,6 +70,7 @@ #include "paimon/format/file_format.h" #include "paimon/result.h" #include "paimon/table/format/format_table.h" +#include "paimon/table/source/startup_mode.h" namespace paimon { namespace { @@ -196,6 +197,34 @@ Status ValidateBTreeIndexerOptions(const std::map& opt return Status::OK(); } +/// Java `SchemaValidation#checkExactOneOptionExistInMode`. +Status CheckExactOneOptionExistInMode(const std::map& options, + const StartupMode& mode, + const std::vector& keys) { + auto set_count = std::count_if(keys.begin(), keys.end(), [&options](const std::string& key) { + return options.count(key) > 0; + }); + if (set_count != 1) { + return Status::Invalid(fmt::format("must set only one key in [{}] when you use {} for {}", + fmt::join(keys, ","), mode.ToString(), + Options::SCAN_MODE)); + } + return Status::OK(); +} + +/// Java `SchemaValidation#checkOptionsConflict`. +Status CheckOptionsConflict(const std::map& options, + const std::vector& illegal_keys, + const std::vector& legal_keys) { + for (const auto& key : illegal_keys) { + if (options.count(key) > 0) { + return Status::Invalid(fmt::format("[{}] must be null when you set [{}]", key, + fmt::join(legal_keys, ","))); + } + } + return Status::OK(); +} + } // namespace bool SchemaValidation::IsComplexType(const std::shared_ptr& field) { @@ -1339,4 +1368,44 @@ Status SchemaValidation::ValidateVectorFields(const TableSchema& schema, return Status::OK(); } +Status SchemaValidation::ValidateStartupMode(const CoreOptions& options) { + // Only the options of Java `SchemaValidation#validateStartupMode` that Paimon C++ defines. + const std::map& options_map = options.ToMap(); + StartupMode mode = options.GetStartupMode(); + if (mode == StartupMode::FromTimestamp()) { + std::vector timestamp_keys = {Options::SCAN_TIMESTAMP_MILLIS, + Options::SCAN_TIMESTAMP}; + PAIMON_RETURN_NOT_OK(CheckExactOneOptionExistInMode(options_map, mode, timestamp_keys)); + return CheckOptionsConflict( + options_map, {Options::SCAN_SNAPSHOT_ID, Options::SCAN_TAG_NAME}, timestamp_keys); + } + if (mode == StartupMode::FromSnapshot()) { + std::vector snapshot_keys = {Options::SCAN_SNAPSHOT_ID, + Options::SCAN_TAG_NAME}; + PAIMON_RETURN_NOT_OK(CheckExactOneOptionExistInMode(options_map, mode, snapshot_keys)); + return CheckOptionsConflict( + options_map, {Options::SCAN_TIMESTAMP_MILLIS, Options::SCAN_TIMESTAMP}, snapshot_keys); + } + if (mode == StartupMode::FromSnapshotFull()) { + if (options_map.count(Options::SCAN_SNAPSHOT_ID) == 0) { + return Status::Invalid(fmt::format("{} can not be null when you use {} for {}", + Options::SCAN_SNAPSHOT_ID, mode.ToString(), + Options::SCAN_MODE)); + } + return CheckOptionsConflict( + options_map, + {Options::SCAN_TIMESTAMP_MILLIS, Options::SCAN_TIMESTAMP, Options::SCAN_TAG_NAME}, + {Options::SCAN_SNAPSHOT_ID}); + } + const char* const travel_keys[] = {Options::SCAN_TIMESTAMP_MILLIS, Options::SCAN_TIMESTAMP, + Options::SCAN_SNAPSHOT_ID, Options::SCAN_TAG_NAME}; + for (const char* key : travel_keys) { + if (options_map.count(key) > 0) { + return Status::Invalid(fmt::format("{} must be null when you use {} for {}", key, + mode.ToString(), Options::SCAN_MODE)); + } + } + return Status::OK(); +} + } // namespace paimon diff --git a/src/paimon/core/schema/schema_validation.h b/src/paimon/core/schema/schema_validation.h index 813aaebb4..6c8af1be3 100644 --- a/src/paimon/core/schema/schema_validation.h +++ b/src/paimon/core/schema/schema_validation.h @@ -58,6 +58,13 @@ class SchemaValidation { /// Checks the deliberately limited dedicated-vector write path, including runtime overrides. static Status ValidateVectorStore(const TableSchema& schema, const CoreOptions& options); + /// Checks the options that select the snapshot a scan starts from against `scan.mode`, as Java + /// `SchemaValidation#validateStartupMode` does, after operation-specific options have been + /// merged with the table schema options: `from-snapshot`, `from-snapshot-full` and + /// `from-timestamp` need exactly one of their own options and take none of the others, and + /// any other mode takes none of them. + static Status ValidateStartupMode(const CoreOptions& options); + /// Validates a schema a table is about to be created from, whichever catalog is creating it. /// /// It picks the rules by the schema's `type` option: a format table owns no bucket, manifest @@ -121,9 +128,6 @@ class SchemaValidation { static Status ValidateDefaultValues(const TableSchema& schema) { return Status::NotImplemented("validate default values not implemented"); } - static Status ValidateStartupMode(const CoreOptions& options) { - return Status::NotImplemented("validate startup mode not implemented"); - } static Status ValidateFieldsPrefix(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceField(const TableSchema& schema, const CoreOptions& options); static Status ValidateSequenceGroup(const TableSchema& schema, const CoreOptions& options); diff --git a/src/paimon/core/schema/schema_validation_test.cpp b/src/paimon/core/schema/schema_validation_test.cpp index 4f7b41160..eab88a006 100644 --- a/src/paimon/core/schema/schema_validation_test.cpp +++ b/src/paimon/core/schema/schema_validation_test.cpp @@ -29,6 +29,7 @@ #include "paimon/common/data/variant/variant_type_utils.h" #include "paimon/common/table/special_fields.h" #include "paimon/common/utils/checked_cast.h" +#include "paimon/core/core_options.h" #include "paimon/core/schema/table_schema.h" #include "paimon/defs.h" #include "paimon/testing/utils/testharness.h" @@ -215,6 +216,55 @@ TEST(SchemaValidationTest, TestRealtimeDeduplicate) { "requires 'file-index.read.enabled=true'"); } +TEST(SchemaValidationTest, TestStartupMode) { + auto validate = [](const std::map& options) -> Status { + PAIMON_ASSIGN_OR_RAISE(CoreOptions core_options, CoreOptions::FromMap(options)); + return SchemaValidation::ValidateStartupMode(core_options); + }; + ASSERT_OK(validate({})); + ASSERT_OK(validate({{Options::SCAN_MODE, "latest"}})); + ASSERT_OK(validate({{Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_OK(validate({{Options::SCAN_MODE, "from-snapshot"}, {Options::SCAN_TAG_NAME, "tag"}})); + ASSERT_OK( + validate({{Options::SCAN_MODE, "from-snapshot-full"}, {Options::SCAN_SNAPSHOT_ID, "1"}})); + ASSERT_OK(validate({{Options::SCAN_TIMESTAMP, "2023-06-01 00:00:00"}})); + ASSERT_OK( + validate({{Options::SCAN_MODE, "from-timestamp"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}})); + + ASSERT_NOK_WITH_MSG( + validate({{Options::SCAN_MODE, "latest-full"}, {Options::SCAN_SNAPSHOT_ID, "1"}}), + "scan.snapshot-id must be null when you use latest-full for scan.mode"); + ASSERT_NOK_WITH_MSG( + validate({{Options::SCAN_MODE, "latest"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}), + "scan.timestamp-millis must be null when you use latest for scan.mode"); + ASSERT_NOK_WITH_MSG(validate({{Options::SCAN_MODE, "from-snapshot"}}), + "must set only one key in [scan.snapshot-id,scan.tag-name] when you use " + "from-snapshot for scan.mode"); + ASSERT_NOK_WITH_MSG( + validate({{Options::SCAN_SNAPSHOT_ID, "1"}, {Options::SCAN_TAG_NAME, "tag"}}), + "must set only one key in [scan.snapshot-id,scan.tag-name] when you use from-snapshot for " + "scan.mode"); + ASSERT_NOK_WITH_MSG( + validate({{Options::SCAN_TAG_NAME, "tag"}, {Options::SCAN_TIMESTAMP_MILLIS, "1"}}), + "[scan.timestamp-millis] must be null when you set [scan.snapshot-id,scan.tag-name]"); + ASSERT_NOK_WITH_MSG( + validate({{Options::SCAN_MODE, "from-snapshot-full"}, {Options::SCAN_TAG_NAME, "tag"}}), + "scan.snapshot-id can not be null when you use from-snapshot-full for scan.mode"); + ASSERT_NOK_WITH_MSG(validate({{Options::SCAN_MODE, "from-snapshot-full"}, + {Options::SCAN_SNAPSHOT_ID, "1"}, + {Options::SCAN_TAG_NAME, "tag"}}), + "[scan.tag-name] must be null when you set [scan.snapshot-id]"); + ASSERT_NOK_WITH_MSG( + validate({{Options::SCAN_MODE, "from-timestamp"}, {Options::SCAN_SNAPSHOT_ID, "1"}}), + "must set only one key in [scan.timestamp-millis,scan.timestamp] when you use " + "from-timestamp for scan.mode"); + ASSERT_NOK_WITH_MSG(validate({{Options::SCAN_MODE, "from-timestamp"}, + {Options::SCAN_TIMESTAMP_MILLIS, "1"}, + {Options::SCAN_TAG_NAME, "tag"}}), + "[scan.tag-name] must be null when you set " + "[scan.timestamp-millis,scan.timestamp]"); +} + TEST(SchemaValidationTest, TestVectorType) { auto vector_field = arrow::field("embedding", arrow::fixed_size_list(arrow::float32(), 3)); auto schema = arrow::schema({arrow::field("id", arrow::int64()), vector_field}); diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.cpp b/src/paimon/core/table/source/data_evolution_full_text_read.cpp new file mode 100644 index 000000000..e73f4c428 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_full_text_read.cpp @@ -0,0 +1,332 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/data_evolution_full_text_read.h" + +#include +#include +#include +#include +#include + +#include "arrow/c/bridge.h" +#include "arrow/c/helpers.h" +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "paimon/common/executor/future.h" +#include "paimon/common/global_index/global_index_utils.h" +#include "paimon/common/global_index/offset_global_index_reader.h" +#include "paimon/common/global_index/scored_global_index_result_utils.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/scope_guard.h" +#include "paimon/core/global_index/global_index_live_row_filter.h" +#include "paimon/core/global_index/global_index_scan_impl.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/source/data_evolution_full_text_scan.h" +#include "paimon/core/tag/tag.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_index_io_meta.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/predicate/full_text_search.h" + +namespace paimon { +namespace { +/// The widest span of search row ranges that a split expands into a bitmap to intersect with the +/// live rows: at most 65536 run containers, cheaper than copying the live rows of a large table for +/// every split. A wider span trims the live rows instead. +constexpr int64_t kMaxExpandedSearchRows = static_cast(1) << 32; + +/// Loads snapshot `snapshot_id`, or the copy that a tag keeps of it once the snapshot has expired, +/// since a plan of `scan.tag-name` may come from a tag that outlives its snapshot. +Result LoadPlannedSnapshot(const CoreOptions& options, const std::string& root_path, + int64_t snapshot_id) { + std::string branch = BranchManager::NormalizeBranch(options.GetBranch()); + SnapshotManager snapshot_manager(options.GetFileSystem(), root_path, branch); + Result snapshot = snapshot_manager.LoadSnapshot(snapshot_id); + if (snapshot.ok()) { + return snapshot; + } + // The snapshot may expire between planning and loading, or even while it is loaded, so a + // failed load falls back to the tags unless the snapshot is still there. + PAIMON_ASSIGN_OR_RAISE(bool exists, snapshot_manager.SnapshotExists(snapshot_id)); + if (exists) { + return snapshot.status(); + } + TagManager tag_manager(options.GetFileSystem(), root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::vector tag_names, tag_manager.ListTagNames()); + for (const std::string& tag_name : tag_names) { + // A tag deleted after it was listed, even while it is read, is skipped, as it no longer + // keeps the snapshot; any other failure to read a tag is an error. + Result> tag = tag_manager.Get(tag_name); + if (!tag.ok()) { + PAIMON_ASSIGN_OR_RAISE(bool tag_exists, + options.GetFileSystem()->Exists(tag_manager.TagPath(tag_name))); + if (tag_exists) { + return tag.status(); + } + continue; + } + if (tag.value() && tag.value()->Id() == snapshot_id) { + return tag.value()->TrimToSnapshot(); + } + } + return Status::NotExist(fmt::format( + "snapshot {} that the full-text search splits were planned from no longer exists", + snapshot_id)); +} + +/// Describes `split` for the errors of its search. +std::string Describe(const IndexFullTextSearchSplitImpl& split) { + std::vector file_names; + file_names.reserve(split.FullTextIndexFiles().size()); + for (const auto& file : split.FullTextIndexFiles()) { + file_names.push_back(file->FileName()); + } + return fmt::format( + "while searching the split of column {} in snapshot {} over rows [{}, {}] with {} index " + "files [{}]", + split.ColumnName(), split.SnapshotId(), split.RowRangeStart(), split.RowRangeEnd(), + split.FullTextIndexFiles().front()->IndexType(), fmt::join(file_names, ", ")); +} +} // namespace + +DataEvolutionFullTextRead::DataEvolutionFullTextRead( + const std::string& root_path, const std::shared_ptr& table_schema, + const CoreOptions& options, const std::shared_ptr& index_file_manager, + const std::shared_ptr& partition_filter, const DataField& text_field, + const std::string& query, int32_t limit, const std::shared_ptr& executor, + const std::shared_ptr& pool) + : root_path_(root_path), + table_schema_(table_schema), + options_(options), + index_file_manager_(index_file_manager), + partition_filter_(partition_filter), + text_field_(text_field), + query_(query), + limit_(limit), + executor_(executor), + pool_(pool) {} + +Result> DataEvolutionFullTextRead::Create( + const std::string& root_path, const std::shared_ptr& table_schema, + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_filter, const DataField& text_field, + const std::string& query, int32_t limit, const std::shared_ptr& executor, + const std::shared_ptr& pool) { + if (limit <= 0) { + return Status::Invalid("Limit must be positive, set via SetLimit()"); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr final_executor, + GlobalIndexScanImpl::GetOrCreateExecutor(executor, options)); + auto index_file_manager = std::make_shared( + options.GetFileSystem(), path_factory->CreateGlobalIndexFileFactory(), + /*checkpoint_path_factory=*/nullptr); + return std::unique_ptr(new DataEvolutionFullTextRead( + root_path, table_schema, options, index_file_manager, partition_filter, text_field, query, + limit, final_executor, pool)); +} + +std::optional DataEvolutionFullTextRead::IncludeRowIds( + const IndexFullTextSearchSplit& split, const RoaringBitmap64* live_rows) { + const std::vector& search_ranges = split.SearchRowRanges(); + if (search_ranges.empty() || (live_rows && live_rows->IsEmpty())) { + return RoaringBitmap64(); + } + Range physical_range(split.RowRangeStart(), split.RowRangeEnd()); + if (!live_rows && search_ranges.size() == 1 && search_ranges[0] == physical_range) { + return std::nullopt; + } + Range searched_span(search_ranges.front().from, search_ranges.back().to); + RoaringBitmap64 include_row_ids; + if (live_rows && searched_span.Count() > kMaxExpandedSearchRows) { + include_row_ids = GlobalIndexUtils::RowIdsWithin(*live_rows, searched_span); + for (size_t i = 1; i < search_ranges.size(); ++i) { + include_row_ids.RemoveRange(search_ranges[i - 1].to + 1, search_ranges[i].from); + } + } else { + for (const auto& range : search_ranges) { + include_row_ids |= GlobalIndexUtils::RowIdsOf(range); + } + if (live_rows) { + include_row_ids &= *live_rows; + } + } + if (live_rows && include_row_ids.Cardinality() == physical_range.Count()) { + return std::nullopt; + } + return include_row_ids; +} + +Result> DataEvolutionFullTextRead::LiveRows( + int64_t snapshot_id) const { + // Without deletion vectors or row tracking every row is live, so the planned snapshot, which + // may be gone, is not loaded. + if (!options_.DeletionVectorsEnabled() || !options_.RowTrackingEnabled()) { + return std::optional(); + } + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, + LoadPlannedSnapshot(options_, root_path_, snapshot_id)); + return GlobalIndexLiveRowFilter::LiveRows(root_path_, table_schema_, snapshot, + partition_filter_, options_, executor_, pool_); +} + +Result> DataEvolutionFullTextRead::SearchSplit( + const IndexFullTextSearchSplitImpl& split, + const std::optional& include_row_ids) const { + // The splits of a column may come from different indexes, so each split is read with the + // indexer and fields of its own files. + const std::shared_ptr& first_file = split.FullTextIndexFiles().front(); + const std::string& index_type = first_file->IndexType(); + const GlobalIndexMeta& first_meta = first_file->GetGlobalIndexMeta().value(); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, options_.ToMap())); + if (!indexer) { + return Status::NotImplemented( + fmt::format("no global indexer is registered for index type {}", index_type)); + } + std::vector indexed_fields; + if (first_meta.extra_field_ids) { + PAIMON_ASSIGN_OR_RAISE(DataField index_field, + table_schema_->GetField(first_meta.index_field_id)); + indexed_fields.push_back(std::move(index_field)); + for (int32_t extra_field_id : first_meta.extra_field_ids.value()) { + PAIMON_ASSIGN_OR_RAISE(DataField extra_field, table_schema_->GetField(extra_field_id)); + indexed_fields.push_back(std::move(extra_field)); + } + } else { + indexed_fields.push_back(text_field_); + } + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(indexed_fields); + ::ArrowSchema c_arrow_schema; + PAIMON_RETURN_NOT_OK_FROM_ARROW(arrow::ExportSchema(*arrow_schema, &c_arrow_schema)); + ScopeGuard guard([&c_arrow_schema]() { ArrowSchemaRelease(&c_arrow_schema); }); + + std::vector index_io_metas; + index_io_metas.reserve(split.FullTextIndexFiles().size()); + for (const auto& file : split.FullTextIndexFiles()) { + index_io_metas.emplace_back(index_file_manager_->ToPath(file), file->FileSize(), + file->RowCount(), file->GetGlobalIndexMeta()->index_meta); + } + Range physical_range(split.RowRangeStart(), split.RowRangeEnd()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr index_reader, + indexer->CreateReader(&c_arrow_schema, index_file_manager_, index_io_metas, + physical_range.Count(), /*row_ranges=*/std::nullopt, pool_)); + OffsetGlobalIndexReader reader(std::move(index_reader), physical_range.from, physical_range.to); + // Each split returns its own top `limit_` rows. The splits rank disjoint rows, so the top + // `limit_` rows of their union are the global top `limit_` rows, except for which of the rows + // tied at the `limit_`-th score of a split its index returns (apache/paimon#9955). + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + reader.VisitFullTextSearch(std::make_shared( + text_field_.Name(), query_, limit_, include_row_ids))); + if (!result) { + return Status::NotImplemented( + fmt::format("global index type {} does not answer full-text searches", index_type)); + } + return result; +} + +Result> DataEvolutionFullTextRead::Read( + const std::vector>& splits) { + std::vector> index_splits; + index_splits.reserve(splits.size()); + std::optional snapshot_id; + for (const auto& split : splits) { + auto index_split = std::dynamic_pointer_cast(split); + if (!index_split) { + return Status::Invalid("invalid full-text search split, cannot cast to an index split"); + } + if (index_split->ColumnName() != text_field_.Name()) { + return Status::Invalid(fmt::format( + "a split of column {} cannot be read by a full-text search of column {}", + index_split->ColumnName(), text_field_.Name())); + } + // A column of the same name may have replaced the planned one, so match field ids too. + const GlobalIndexMeta& meta = + index_split->FullTextIndexFiles().front()->GetGlobalIndexMeta().value(); + if (!DataEvolutionFullTextScan::IndexesField(meta, text_field_.Id())) { + return Status::Invalid(fmt::format( + "a split of column {} does not index its field id {}; the column may have been " + "replaced after the split was planned", + text_field_.Name(), text_field_.Id())); + } + if (snapshot_id && snapshot_id.value() != index_split->SnapshotId()) { + return Status::Invalid(fmt::format( + "full-text search splits planned from snapshots {} and {} cannot be read together", + snapshot_id.value(), index_split->SnapshotId())); + } + snapshot_id = index_split->SnapshotId(); + index_splits.push_back(std::move(index_split)); + } + if (index_splits.empty()) { + return std::shared_ptr( + std::make_shared(RoaringBitmap64(), + std::vector())); + } + + // Live rows are read on this thread before the searches fan out, so a read of manifests on + // `executor_` never waits for it from one of its own threads. They are pinned to the snapshot + // the splits were planned from (apache/paimon#9031). + PAIMON_ASSIGN_OR_RAISE(std::optional live_rows, LiveRows(snapshot_id.value())); + using SearchResult = Result>; + std::vector> searches; + searches.reserve(index_splits.size()); + for (const auto& index_split : index_splits) { + auto include_row_ids = std::make_shared>( + IncludeRowIds(*index_split, live_rows ? &live_rows.value() : nullptr)); + if (include_row_ids->has_value() && include_row_ids->value().IsEmpty()) { + continue; + } + searches.emplace_back([this, index_split, include_row_ids]() -> SearchResult { + SearchResult result = SearchSplit(*index_split, *include_row_ids); + if (!result.ok()) { + return result.status().WithMessage( + fmt::format("{}; {}", result.status().message(), Describe(*index_split))); + } + return result; + }); + } + std::vector results; + if (searches.size() <= 1) { + for (const auto& search : searches) { + results.push_back(search()); + } + } else { + std::vector> futures; + futures.reserve(searches.size()); + for (const auto& search : searches) { + futures.push_back(Via(executor_.get(), search)); + } + results = CollectAll(futures); + } + std::vector> scored_results; + scored_results.reserve(results.size()); + for (auto& result : results) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr scored_result, + std::move(result)); + scored_results.push_back(std::move(scored_result)); + } + return ScoredGlobalIndexResultUtils::MergeTopK(scored_results, limit_); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/data_evolution_full_text_read.h b/src/paimon/core/table/source/data_evolution_full_text_read.h new file mode 100644 index 000000000..b0c27d74c --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_full_text_read.h @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/core/global_index/global_index_file_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/executor.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/table/source/full_text_read.h" +#include "paimon/utils/roaring_bitmap64.h" + +namespace paimon { +/// `FullTextRead` of a data-evolution table, aligned with Java `DataEvolutionFullTextRead`. +class DataEvolutionFullTextRead : public FullTextRead { + public: + /// @param root_path Root directory of the table. + /// @param table_schema Schema of the table. + /// @param options Options of the table, merged with those of the search. + /// @param path_factory Path factory of the table. + /// @param partition_filter Predicate over the partition fields used to read live rows, or null + /// for all partitions. + /// @param text_field The searched column. + /// @param query The JSON query. + /// @param limit The number of rows to return; must be positive. + /// @param executor Executor to search splits on, or null to create one with + /// `global-index.thread-num` threads. + /// @param pool Memory pool for reading indexes and manifests. + static Result> Create( + const std::string& root_path, const std::shared_ptr& table_schema, + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_filter, const DataField& text_field, + const std::string& query, int32_t limit, const std::shared_ptr& executor, + const std::shared_ptr& pool); + + using FullTextRead::Read; + + Result> Read( + const std::vector>& splits) override; + + /// Returns the row ids that `split` ranks, as Java `DataEvolutionFullTextRead#includeRowIds` + /// does: its search row ranges, intersected with `live_rows` if not null. Returns nullopt if + /// they are every row of the physical range of the split, so the index needs no filter. A + /// search span wider than 2^32 rows trims `live_rows` to the ranges instead of expanding them, + /// so that a wide shard costs no more than the live rows. + static std::optional IncludeRowIds(const IndexFullTextSearchSplit& split, + const RoaringBitmap64* live_rows); + + private: + DataEvolutionFullTextRead(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& index_file_manager, + const std::shared_ptr& partition_filter, + const DataField& text_field, const std::string& query, int32_t limit, + const std::shared_ptr& executor, + const std::shared_ptr& pool); + + /// Returns the rows live in snapshot `snapshot_id`, or nullopt if every row counts as live. + Result> LiveRows(int64_t snapshot_id) const; + + /// Searches the index shard of `split` among `include_row_ids`, or all of its rows if not set, + /// and returns at most `limit_` rows with global row ids. + Result> SearchSplit( + const IndexFullTextSearchSplitImpl& split, + const std::optional& include_row_ids) const; + + std::string root_path_; + std::shared_ptr table_schema_; + CoreOptions options_; + std::shared_ptr index_file_manager_; + std::shared_ptr partition_filter_; + DataField text_field_; + std::string query_; + int32_t limit_; + std::shared_ptr executor_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp new file mode 100644 index 000000000..193ae1ab2 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_full_text_read_test.cpp @@ -0,0 +1,542 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/data_evolution_full_text_read.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arrow/api.h" +#include "arrow/c/bridge.h" +#include "gtest/gtest.h" +#include "paimon/catalog/catalog.h" +#include "paimon/catalog/identifier.h" +#include "paimon/common/utils/arrow/status_utils.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/defs.h" +#include "paimon/factories/factory.h" +#include "paimon/fs/file_system.h" +#include "paimon/global_index/bitmap_scored_global_index_result.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/table/source/full_text_search_builder.h" +#include "paimon/testing/mock/mock_global_index_reader.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" + +namespace paimon::test { +namespace { +constexpr char kFakeIndexType[] = "fake-full-text"; +constexpr char kQuery[] = R"({"match":{"query":"paimon"}})"; + +struct RecordedSearch { + std::string file_name; + std::vector schema_fields; + std::string field_name; + std::string query; + int32_t limit; + std::optional> include_row_ids; +}; + +class FakeIndexState { + public: + static FakeIndexState& Instance() { + static FakeIndexState state; + return state; + } + + void Reset() { + std::lock_guard lock(mutex_); + scores_.clear(); + searches_.clear(); + } + + void SetScores(const std::string& file_name, std::vector scores) { + std::lock_guard lock(mutex_); + scores_[file_name] = std::move(scores); + } + + Result> Scores(const std::string& file_name) const { + std::lock_guard lock(mutex_); + auto iter = scores_.find(file_name); + if (iter == scores_.end()) { + return Status::Invalid("no scores for fake index file " + file_name); + } + return iter->second; + } + + void Record(RecordedSearch search) { + std::lock_guard lock(mutex_); + searches_[search.file_name] = std::move(search); + } + + std::map Searches() const { + std::lock_guard lock(mutex_); + return searches_; + } + + private: + mutable std::mutex mutex_; + std::map> scores_; + std::map searches_; +}; + +class FakeFullTextReader : public MockGlobalIndexReader { + public: + FakeFullTextReader(const std::string& file_name, std::vector schema_fields, + std::vector scores) + : file_name_(file_name), + schema_fields_(std::move(schema_fields)), + scores_(std::move(scores)) {} + + Result> VisitFullTextSearch( + const std::shared_ptr& search) override { + RecordedSearch recorded{file_name_, schema_fields_, search->field_name, + search->query, search->limit, std::nullopt}; + if (search->include_row_ids) { + std::vector row_ids; + for (auto iter = search->include_row_ids->Begin(); + iter != search->include_row_ids->End(); ++iter) { + row_ids.push_back(*iter); + } + recorded.include_row_ids = std::move(row_ids); + } + FakeIndexState::Instance().Record(std::move(recorded)); + + std::vector> rows; + for (int64_t row = 0; row < static_cast(scores_.size()); ++row) { + if (!search->include_row_ids || search->include_row_ids->Contains(row)) { + rows.emplace_back(row, scores_[row]); + } + } + std::sort(rows.begin(), rows.end(), [](const auto& left, const auto& right) { + return left.second != right.second ? left.second > right.second + : left.first < right.first; + }); + rows.resize(std::min(rows.size(), static_cast(search->limit))); + std::sort(rows.begin(), rows.end()); + RoaringBitmap64 bitmap; + std::vector scores; + for (const auto& [row, score] : rows) { + bitmap.Add(row); + scores.push_back(score); + } + return std::make_shared(std::move(bitmap), + std::move(scores)); + } + + private: + std::string file_name_; + std::vector schema_fields_; + std::vector scores_; +}; + +class FakeFullTextIndexer : public GlobalIndexer { + public: + Result>> GetExtraFieldNames() const override { + return std::optional>(); + } + + Result> CreateWriter( + const std::string& field_name, ::ArrowSchema* arrow_schema, + const std::shared_ptr& file_writer, + const std::shared_ptr& pool) const override { + return Status::NotImplemented("fake full-text index cannot be written"); + } + + Result> CreateReader( + ::ArrowSchema* arrow_schema, const std::shared_ptr& file_reader, + const std::vector& files, int64_t total_row_count, + const std::optional>& row_ranges, + const std::shared_ptr& pool) const override { + PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr schema, + arrow::ImportSchema(arrow_schema)); + if (files.size() != 1) { + return Status::Invalid("fake full-text index expects one file per shard"); + } + std::string file_name = PathUtil::GetName(files[0].file_path); + PAIMON_ASSIGN_OR_RAISE(std::vector scores, + FakeIndexState::Instance().Scores(file_name)); + if (static_cast(scores.size()) != total_row_count) { + return Status::Invalid("fake full-text index expects a score for each row"); + } + return std::make_shared(file_name, schema->field_names(), + std::move(scores)); + } + + bool SupportsFullTextSearch() const override { + return true; + } +}; + +class FakeFullTextIndexerFactory : public GlobalIndexerFactory { + public: + const char* Identifier() const override { + return "fake-full-text-global"; + } + + Result> Create( + const std::map& options) const override { + return std::make_unique(); + } +}; +REGISTER_PAIMON_FACTORY(FakeFullTextIndexerFactory); + +using ScoredRows = std::vector>; + +ScoredRows ToRows(const ScoredGlobalIndexResult& result) { + ScoredRows rows; + auto iter = result.CreateScoredIterator().value(); + while (iter->HasNext()) { + rows.push_back(iter->NextWithScore()); + } + return rows; +} +} // namespace + +class DataEvolutionFullTextReadTest : public ::testing::Test { + protected: + void SetUp() override { + FakeIndexState::Instance().Reset(); + dir_ = UniqueTestDirectory::Create("local"); + ASSERT_TRUE(dir_); + auto schema = arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("content", arrow::utf8()), + arrow::field("title", arrow::utf8())}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); + ASSERT_OK(catalog->CreateDatabase("foo", {}, /*ignore_if_exists=*/false)); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "bar"), &c_schema, + /*partition_keys=*/{}, /*primary_keys=*/{}, + {{Options::FILE_FORMAT, "parquet"}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}, + /*ignore_if_exists=*/false)); + table_path_ = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + } + + std::shared_ptr NewSplit( + const std::string& file_name, const Range& range, const std::vector& scores, + const std::vector& search_row_ranges, int32_t index_field_id = 1, + const std::optional>& extra_field_ids = std::nullopt, + int64_t snapshot_id = 1, const std::string& index_type = kFakeIndexType) const { + FakeIndexState::Instance().SetScores(file_name, scores); + GlobalIndexMeta meta(range.from, range.to, index_field_id, extra_field_ids, + /*index_meta=*/nullptr); + auto file = std::make_shared( + index_type, file_name, /*file_size=*/128, /*row_count=*/range.Count(), + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt, meta); + return IndexFullTextSearchSplitImpl::Create(snapshot_id, "content", range.from, range.to, + {file}, search_row_ranges) + .value(); + } + + std::unique_ptr NewBuilder(int32_t limit) const { + auto builder = std::make_unique(table_path_); + builder->SetQuery("content", kQuery).SetLimit(limit); + return builder; + } + + std::unique_ptr dir_; + std::string table_path_; +}; + +TEST_F(DataEvolutionFullTextReadTest, TestMergesShardResultsIntoGlobalTopK) { + std::vector> splits = { + NewSplit("a.index", Range(0, 4), {0.1f, 0.9f, 0.5f, 0.3f, 0.7f}, {Range(0, 4)}), + NewSplit("b.index", Range(5, 9), {0.8f, 0.2f, 0.6f, 0.4f, 1.0f}, {Range(5, 9)})}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); + ASSERT_EQ(ToRows(*result), ScoredRows({{1, 0.9f}, {5, 0.8f}, {9, 1.0f}})); + + std::map searches = FakeIndexState::Instance().Searches(); + ASSERT_EQ(searches.size(), 2u); + for (const auto& [file_name, search] : searches) { + ASSERT_EQ(search.limit, 3) << file_name; + ASSERT_EQ(search.field_name, "content") << file_name; + ASSERT_EQ(search.query, kQuery) << file_name; + ASSERT_EQ(search.schema_fields, std::vector({"content"})) << file_name; + ASSERT_FALSE(search.include_row_ids) << file_name; + } + + FullTextScan::Plan plan(/*snapshot_id=*/1, splits); + ASSERT_OK_AND_ASSIGN(read, NewBuilder(100)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(result, read->Read(plan)); + ASSERT_EQ(ToRows(*result).size(), 10u); +} + +TEST_F(DataEvolutionFullTextReadTest, TestRanksOnlySearchRowRanges) { + std::vector> splits = { + NewSplit("a.index", Range(0, 4), {0.1f, 0.9f, 0.5f, 0.3f, 0.7f}, + {Range(0, 0), Range(3, 4)}), + NewSplit("b.index", Range(10, 14), {0.8f, 0.2f, 0.6f, 0.4f, 1.0f}, {Range(12, 14)})}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); + ASSERT_EQ(ToRows(*result), ScoredRows({{4, 0.7f}, {12, 0.6f}, {14, 1.0f}})); + + std::map searches = FakeIndexState::Instance().Searches(); + ASSERT_EQ(searches.size(), 2u); + ASSERT_EQ(searches["a.index"].include_row_ids, std::vector({0, 3, 4})); + ASSERT_EQ(searches["b.index"].include_row_ids, std::vector({2, 3, 4})); +} + +TEST_F(DataEvolutionFullTextReadTest, TestIncludeRowIds) { + auto row_ids_of = [](const std::optional& bitmap) { + std::vector row_ids; + for (auto iter = bitmap->Begin(); iter != bitmap->End(); ++iter) { + row_ids.push_back(*iter); + } + return row_ids; + }; + std::vector scores(10, 0.1f); + auto split = std::dynamic_pointer_cast( + NewSplit("a.index", Range(10, 19), scores, {Range(10, 12), Range(17, 19)})); + ASSERT_TRUE(split); + std::optional row_ids = + DataEvolutionFullTextRead::IncludeRowIds(*split, /*live_rows=*/nullptr); + ASSERT_TRUE(row_ids); + ASSERT_EQ(row_ids_of(row_ids), std::vector({10, 11, 12, 17, 18, 19})); + RoaringBitmap64 live_rows = RoaringBitmap64::From({0, 11, 12, 15, 18, 25}); + row_ids = DataEvolutionFullTextRead::IncludeRowIds(*split, &live_rows); + ASSERT_TRUE(row_ids); + ASSERT_EQ(row_ids_of(row_ids), std::vector({11, 12, 18})); + RoaringBitmap64 no_live_rows_in_split = RoaringBitmap64::From({0, 15, 16}); + row_ids = DataEvolutionFullTextRead::IncludeRowIds(*split, &no_live_rows_in_split); + ASSERT_TRUE(row_ids); + ASSERT_TRUE(row_ids->IsEmpty()); + + auto whole_split = std::dynamic_pointer_cast( + NewSplit("b.index", Range(10, 19), scores, {Range(10, 19)})); + ASSERT_TRUE(whole_split); + ASSERT_FALSE(DataEvolutionFullTextRead::IncludeRowIds(*whole_split, /*live_rows=*/nullptr)); + RoaringBitmap64 all_rows; + all_rows.AddRange(0, 30); + ASSERT_FALSE(DataEvolutionFullTextRead::IncludeRowIds(*whole_split, &all_rows)); + ASSERT_TRUE(DataEvolutionFullTextRead::IncludeRowIds(*whole_split, &live_rows)); + RoaringBitmap64 empty_live_rows; + row_ids = DataEvolutionFullTextRead::IncludeRowIds(*whole_split, &empty_live_rows); + ASSERT_TRUE(row_ids); + ASSERT_TRUE(row_ids->IsEmpty()); + + const int64_t max_row_id = std::numeric_limits::max(); + auto widest_split = std::dynamic_pointer_cast( + NewSplit("widest.index", Range(1, max_row_id), /*scores=*/{}, {Range(1, max_row_id)})); + ASSERT_TRUE(widest_split); + ASSERT_FALSE(DataEvolutionFullTextRead::IncludeRowIds(*widest_split, /*live_rows=*/nullptr)); + RoaringBitmap64 sparse_live_rows = RoaringBitmap64::From({0, 5, 20, max_row_id}); + row_ids = DataEvolutionFullTextRead::IncludeRowIds(*widest_split, &sparse_live_rows); + ASSERT_TRUE(row_ids); + ASSERT_EQ(row_ids_of(row_ids), std::vector({5, 20, max_row_id})); + auto wide_split = std::dynamic_pointer_cast( + NewSplit("wide.index", Range(1, max_row_id), /*scores=*/{}, + {Range(1, 10), Range(max_row_id - 10, max_row_id)})); + ASSERT_TRUE(wide_split); + row_ids = DataEvolutionFullTextRead::IncludeRowIds(*wide_split, &sparse_live_rows); + ASSERT_TRUE(row_ids); + ASSERT_EQ(row_ids_of(row_ids), std::vector({5, max_row_id})); +} + +TEST_F(DataEvolutionFullTextReadTest, TestReadsEachSplitWithItsOwnIndexFields) { + std::vector> splits = { + NewSplit("dedicated.index", Range(100, 102), {0.1f, 0.5f, 0.2f}, {Range(100, 102)}), + NewSplit("multi.index", Range(200, 202), {0.3f, 0.4f, 0.6f}, {Range(200, 202)}, + /*index_field_id=*/2, std::vector({1}))}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, read->Read(splits)); + ASSERT_EQ(ToRows(*result), ScoredRows({{101, 0.5f}, {201, 0.4f}, {202, 0.6f}})); + std::map searches = FakeIndexState::Instance().Searches(); + ASSERT_EQ(searches.size(), 2u); + ASSERT_EQ(searches["dedicated.index"].schema_fields, std::vector({"content"})); + ASSERT_EQ(searches["multi.index"].schema_fields, + std::vector({"title", "content"})); + for (const auto& [file_name, search] : searches) { + ASSERT_EQ(search.field_name, "content") << file_name; + ASSERT_FALSE(search.include_row_ids) << file_name; + } +} + +TEST_F(DataEvolutionFullTextReadTest, TestRejectsInvalidSplits) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr empty, read->Read({})); + ASSERT_TRUE(ToRows(*empty).empty()); + + auto split = NewSplit("a.index", Range(0, 1), {0.1f, 0.2f}, {Range(0, 1)}); + auto other_snapshot = NewSplit("b.index", Range(2, 3), {0.1f, 0.2f}, {Range(2, 3)}, + /*index_field_id=*/1, std::nullopt, /*snapshot_id=*/2); + ASSERT_NOK_WITH_MSG(read->Read({split, other_snapshot}), + "planned from snapshots 1 and 2 cannot be read together"); + ASSERT_NOK_WITH_MSG(read->Read({nullptr}), "cannot cast to an index split"); + + FullTextSearchBuilder title_builder(table_path_); + title_builder.SetQuery("title", kQuery).SetLimit(3); + ASSERT_OK_AND_ASSIGN(std::unique_ptr title_read, title_builder.NewFullTextRead()); + ASSERT_NOK_WITH_MSG(title_read->Read({split}), + "a split of column content cannot be read by a full-text search of " + "column title"); + + auto unknown_type = NewSplit("c.index", Range(0, 1), {0.1f, 0.2f}, {Range(0, 1)}, + /*index_field_id=*/1, std::nullopt, /*snapshot_id=*/1, + /*index_type=*/"no-such-index"); + ASSERT_NOK_WITH_MSG(read->Read({unknown_type}), + "no global indexer is registered for index type no-such-index; while " + "searching the split of column content in snapshot 1 over rows [0, 1] " + "with no-such-index index files [c.index]"); +} + +TEST_F(DataEvolutionFullTextReadTest, TestRejectsSplitOfReplacedColumn) { + std::vector> splits = { + NewSplit("a.index", Range(0, 1), {0.1f, 0.2f}, {Range(0, 1)}), + NewSplit("multi.index", Range(2, 3), {0.1f, 0.2f}, {Range(2, 3)}, + /*index_field_id=*/2, std::vector({1}))}; + ASSERT_OK_AND_ASSIGN(std::unique_ptr read, NewBuilder(3)->NewFullTextRead()); + ASSERT_OK(read->Read(splits)); + + SchemaManager schema_manager(dir_->GetFileSystem(), table_path_); + ASSERT_OK_AND_ASSIGN(std::optional> latest_schema, + schema_manager.Latest()); + ASSERT_TRUE(latest_schema); + const std::shared_ptr& old_schema = latest_schema.value(); + auto replaced_content = arrow::schema( + {arrow::field("id", arrow::int32()), arrow::field("old_content", arrow::utf8()), + arrow::field("title", arrow::utf8()), arrow::field("content", arrow::utf8())}); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr new_schema, + TableSchema::Create(old_schema->Id() + 1, replaced_content, old_schema->PartitionKeys(), + old_schema->PrimaryKeys(), old_schema->Options())); + ASSERT_OK_AND_ASSIGN(std::string new_schema_json, new_schema->ToJsonString()); + ASSERT_OK(dir_->GetFileSystem()->AtomicStore( + PathUtil::JoinPath(schema_manager.SchemaDirectory(), + "schema-" + std::to_string(new_schema->Id())), + new_schema_json)); + + ASSERT_OK_AND_ASSIGN(read, NewBuilder(3)->NewFullTextRead()); + for (const auto& split : splits) { + ASSERT_NOK_WITH_MSG(read->Read({split}), + "a split of column content does not index its field id 3"); + } +} + +TEST_F(DataEvolutionFullTextReadTest, TestBuilderValidatesSettings) { + FullTextSearchBuilder no_query(table_path_); + no_query.SetLimit(3); + ASSERT_NOK_WITH_MSG(no_query.NewFullTextScan(), "Query must be set via SetQuery()"); + ASSERT_NOK_WITH_MSG(no_query.NewFullTextRead(), "Query must be set via SetQuery()"); + + FullTextSearchBuilder no_limit(table_path_); + no_limit.SetQuery("content", kQuery); + ASSERT_OK(no_limit.NewFullTextScan()); + ASSERT_NOK_WITH_MSG(no_limit.NewFullTextRead(), "Limit must be positive"); + ASSERT_NOK_WITH_MSG(NewBuilder(0)->NewFullTextRead(), "Limit must be positive"); + ASSERT_NOK_WITH_MSG(NewBuilder(0)->ExecuteLocal(), "Limit must be positive"); + + std::unique_ptr no_pool = NewBuilder(3); + no_pool->WithMemoryPool(nullptr); + ASSERT_NOK_WITH_MSG(no_pool->NewFullTextScan(), "memory pool is null pointer"); + + std::unique_ptr full_mode = NewBuilder(3); + full_mode->AddOption(Options::GLOBAL_INDEX_SEARCH_MODE, "full"); + ASSERT_NOK_WITH_MSG(full_mode->NewFullTextScan(), "full and detail are not supported yet"); + ASSERT_NOK_WITH_MSG(full_mode->NewFullTextRead(), "full and detail are not supported yet"); + + std::unique_ptr no_thread = NewBuilder(3); + no_thread->AddOption(Options::GLOBAL_INDEX_THREAD_NUM, "0"); + ASSERT_NOK_WITH_MSG(no_thread->NewFullTextRead(), "invalid global index thread number 0"); + + FullTextSearchBuilder missing_column(table_path_); + missing_column.SetQuery("missing", kQuery).SetLimit(3); + ASSERT_NOK_WITH_MSG(missing_column.NewFullTextScan(), "Text column 'missing' does not exist."); + + FullTextSearchBuilder missing_table(PathUtil::JoinPath(dir_->Str(), "foo.db/missing")); + missing_table.SetQuery("content", kQuery).SetLimit(3); + ASSERT_NOK_WITH_MSG(missing_table.NewFullTextScan(), "schema file not found"); +} + +TEST_F(DataEvolutionFullTextReadTest, TestBuilderRejectsPrimaryKeyFullTextIndex) { + auto schema = arrow::schema({arrow::field("id", arrow::int32(), /*nullable=*/false), + arrow::field("content", arrow::utf8())}); + ::ArrowSchema c_schema; + ASSERT_TRUE(arrow::ExportSchema(*schema, &c_schema).ok()); + ASSERT_OK_AND_ASSIGN(std::unique_ptr catalog, Catalog::Create(dir_->Str(), {})); + ASSERT_OK(catalog->CreateTable(Identifier("foo", "pk"), &c_schema, /*partition_keys=*/{}, + /*primary_keys=*/{"id"}, + {{Options::FILE_FORMAT, "parquet"}, + {Options::FILE_SYSTEM, "local"}, + {Options::BUCKET, "1"}, + {Options::DELETION_VECTORS_ENABLED, "true"}, + {Options::PK_FULL_TEXT_INDEX_COLUMNS, "content"}}, + /*ignore_if_exists=*/false)); + FullTextSearchBuilder builder(PathUtil::JoinPath(dir_->Str(), "foo.db/pk")); + builder.SetQuery("content", kQuery).SetLimit(3); + ASSERT_NOK_WITH_MSG(builder.NewFullTextScan(), + "with a primary-key full-text index is not supported yet"); + ASSERT_NOK_WITH_MSG(builder.NewFullTextRead(), + "with a primary-key full-text index is not supported yet"); +} + +TEST_F(DataEvolutionFullTextReadTest, TestResolvesPlanSnapshot) { + ASSERT_OK_AND_ASSIGN(std::unique_ptr scan, NewBuilder(3)->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + ASSERT_FALSE(plan->SnapshotId()); + ASSERT_TRUE(plan->Splits().empty()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + NewBuilder(3)->ExecuteLocal()); + ASSERT_TRUE(ToRows(*result).empty()); + + std::vector> time_travel_errors = { + {"", Options::SCAN_SNAPSHOT_ID, "snapshot-1"}, + {"", Options::SCAN_TAG_NAME, "Tag '1' doesn't exist."}, + {"", Options::SCAN_TIMESTAMP, + "There is currently no snapshot earlier than or equal to timestamp"}, + {"", Options::SCAN_TIMESTAMP_MILLIS, + "There is currently no snapshot earlier than or equal to timestamp [1]"}, + {"from-snapshot", Options::SCAN_TAG_NAME, "Tag '1' doesn't exist."}, + {"from-snapshot-full", Options::SCAN_SNAPSHOT_ID, "snapshot-1"}, + {"from-timestamp", Options::SCAN_TIMESTAMP_MILLIS, + "There is currently no snapshot earlier than or equal to timestamp [1]"}, + {"latest-full", Options::SCAN_SNAPSHOT_ID, + "scan.snapshot-id must be null when you use latest-full for scan.mode"}}; + for (const auto& [mode, key, error] : time_travel_errors) { + SCOPED_TRACE(mode + " " + key); + std::unique_ptr builder = NewBuilder(3); + if (!mode.empty()) { + builder->AddOption(Options::SCAN_MODE, mode); + } + builder->AddOption(key, key == Options::SCAN_TIMESTAMP ? "2023-06-01 00:00:00" : "1"); + ASSERT_OK_AND_ASSIGN(scan, builder->NewFullTextScan()); + ASSERT_NOK_WITH_MSG(scan->CreatePlan(), error); + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp new file mode 100644 index 000000000..85b8540e2 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.cpp @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/data_evolution_full_text_scan.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "paimon/core/index/index_file_handler.h" +#include "paimon/core/manifest/index_manifest_entry.h" +#include "paimon/core/manifest/index_manifest_file.h" +#include "paimon/core/schema/schema_validation.h" +#include "paimon/core/snapshot.h" +#include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/core/tag/tag.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/index_file_path_factories.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" +#include "paimon/defs.h" +#include "paimon/global_index/full_text/full_text_defs.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/global_index/lucene/lucene_defs.h" + +namespace paimon { +namespace { +/// The files of one index shard that may rank rows of the searched field. +struct IndexRangeCandidate { + Range file_range; + std::string identity; + /// Whether the searched field is the indexed field of the index, not an extra field. + bool primary; + std::vector> files; + /// The rows assigned to this shard, merged as they are assigned in ascending order. + std::vector assigned; +}; + +/// Resolves the snapshot to search as Java `TimeTravelUtil#tryTravelOrLatest` does: the one that +/// the time-travel option selects, or else the latest one. +/// +/// TODO: take the column, the partition keys and the options from the schema of the selected +/// snapshot, as Java time travel does. They still come from the latest schema, so a column dropped +/// and added again with the same name since that snapshot finds no index files. +Result> ResolveSnapshot(const CoreOptions& options, + const SnapshotManager& snapshot_manager, + const TagManager& tag_manager) { + // Java checks the options against `scan.mode` when it loads the table with them, which leaves + // at most one time-travel option set. + PAIMON_RETURN_NOT_OK(SchemaValidation::ValidateStartupMode(options)); + // Java `TimeTravelUtil#SCAN_KEYS`, without `scan.watermark`, which Paimon C++ does not define. + const char* const time_travel_keys[] = {Options::SCAN_SNAPSHOT_ID, Options::SCAN_TAG_NAME, + Options::SCAN_TIMESTAMP, + Options::SCAN_TIMESTAMP_MILLIS}; + const std::map& raw_options = options.ToMap(); + auto key_iter = + std::find_if(std::begin(time_travel_keys), std::end(time_travel_keys), + [&raw_options](const char* key) { return raw_options.count(key) > 0; }); + if (key_iter == std::end(time_travel_keys)) { + return snapshot_manager.LatestSnapshot(); + } + const std::string key = *key_iter; + if (key == Options::SCAN_SNAPSHOT_ID) { + const std::optional& snapshot_id = options.GetScanSnapshotId(); + if (!snapshot_id) { + return Status::Invalid(fmt::format("invalid {}", key)); + } + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, + snapshot_manager.LoadSnapshot(snapshot_id.value())); + return std::optional(std::move(snapshot)); + } + if (key == Options::SCAN_TAG_NAME) { + std::optional tag_name = options.GetScanTagName(); + if (!tag_name) { + return Status::Invalid(fmt::format("invalid {}", key)); + } + PAIMON_ASSIGN_OR_RAISE(Tag tag, tag_manager.GetOrThrow(tag_name.value())); + PAIMON_ASSIGN_OR_RAISE(Snapshot snapshot, tag.TrimToSnapshot()); + return std::optional(std::move(snapshot)); + } + // `CoreOptions` parses both `scan.timestamp` and `scan.timestamp-millis` into milliseconds. + std::optional timestamp_millis = options.GetScanTimestampMillis(); + if (!timestamp_millis) { + return Status::Invalid(fmt::format("invalid {}", key)); + } + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, + snapshot_manager.EarlierOrEqualTimeMillis(timestamp_millis.value())); + if (!snapshot) { + return Status::Invalid( + fmt::format("There is currently no snapshot earlier than or equal to timestamp [{}]", + timestamp_millis.value())); + } + return snapshot; +} + +/// Returns whether `left` is chosen before `right` for a row that both cover, as Java +/// `DataEvolutionFullTextScan#compareCandidates` does. +struct CandidateRanksBefore { + bool operator()(const IndexRangeCandidate* left, const IndexRangeCandidate* right) const { + if (left->primary != right->primary) { + return left->primary; + } + return std::tie(left->identity, left->file_range.from, left->file_range.to) < + std::tie(right->identity, right->file_range.from, right->file_range.to); + } +}; +} // namespace + +DataEvolutionFullTextScan::DataEvolutionFullTextScan( + const std::string& root_path, const std::shared_ptr& table_schema, + const CoreOptions& options, const std::shared_ptr& path_factory, + const std::shared_ptr& partition_filter, const DataField& text_field, + const std::shared_ptr& pool) + : root_path_(root_path), + table_schema_(table_schema), + options_(options), + path_factory_(path_factory), + partition_filter_(partition_filter), + text_field_(text_field), + pool_(pool) {} + +bool DataEvolutionFullTextScan::IndexesField(const GlobalIndexMeta& meta, int32_t field_id) { + if (meta.index_field_id == field_id) { + return true; + } + return meta.extra_field_ids && + std::find(meta.extra_field_ids->begin(), meta.extra_field_ids->end(), field_id) != + meta.extra_field_ids->end(); +} + +std::string DataEvolutionFullTextScan::IndexIdentity(const IndexFileMeta& file) { + const std::optional& meta = file.GetGlobalIndexMeta(); + assert(meta); + std::vector extra_field_ids = meta->extra_field_ids.value_or(std::vector()); + return fmt::format("{}|{}|[{}]", file.IndexType(), meta->index_field_id, + fmt::join(extra_field_ids, ", ")); +} + +std::vector +DataEvolutionFullTextScan::ChooseIndexRanges( + const std::vector>& index_files, int32_t field_id) { + // (physical range, identity) -> candidate + std::map, IndexRangeCandidate> candidates; + for (const auto& file : index_files) { + const std::optional& meta = file->GetGlobalIndexMeta(); + if (!meta || !IndexesField(meta.value(), field_id)) { + continue; + } + Range file_range(meta->row_range_start, meta->row_range_end); + std::string identity = IndexIdentity(*file); + auto iter = candidates + .try_emplace(std::make_pair(file_range, identity), + IndexRangeCandidate{file_range, identity, + meta->index_field_id == field_id, + /*files=*/{}, /*assigned=*/{}}) + .first; + iter->second.files.push_back(file); + } + + // Split the coverage at every boundary: the start of a shard and the row after its end. + std::set boundaries; + std::unordered_map> starting_at; + std::unordered_map> ending_before; + for (auto& entry : candidates) { + IndexRangeCandidate& candidate = entry.second; + boundaries.insert(candidate.file_range.from); + starting_at[candidate.file_range.from].push_back(&candidate); + if (candidate.file_range.to != std::numeric_limits::max()) { + int64_t after_end = candidate.file_range.to + 1; + boundaries.insert(after_end); + ending_before[after_end].push_back(&candidate); + } + } + std::set active; + for (auto boundary = boundaries.begin(); boundary != boundaries.end(); ++boundary) { + int64_t from = *boundary; + for (IndexRangeCandidate* candidate : ending_before[from]) { + active.erase(candidate); + } + for (IndexRangeCandidate* candidate : starting_at[from]) { + active.insert(candidate); + } + if (active.empty()) { + continue; + } + auto next = std::next(boundary); + int64_t to = next != boundaries.end() ? *next - 1 : std::numeric_limits::max(); + IndexRangeCandidate* best = *active.begin(); + // The pieces are visited in ascending order, so a piece can only extend the last one. + if (!best->assigned.empty() && best->assigned.back().to == from - 1) { + best->assigned.back().to = to; + } else { + best->assigned.emplace_back(from, to); + } + } + + std::vector selections; + for (auto& entry : candidates) { + IndexRangeCandidate& candidate = entry.second; + if (candidate.assigned.empty()) { + continue; + } + selections.push_back(IndexRangeSelection{candidate.file_range, candidate.identity, + std::move(candidate.files), + std::move(candidate.assigned)}); + } + std::sort(selections.begin(), selections.end(), + [](const IndexRangeSelection& left, const IndexRangeSelection& right) { + return std::tie(left.file_range.from, left.file_range.to, left.identity) < + std::tie(right.file_range.from, right.file_range.to, right.identity); + }); + return selections; +} + +Result DataEvolutionFullTextScan::ServesFullTextSearch(const std::string& index_type, + const GlobalIndexer* indexer, + const std::string& table_path, + int64_t snapshot_id) { + if (indexer != nullptr) { + return indexer->SupportsFullTextSearch(); + } + if (index_type == full_text::kIdentifier || index_type == lucene::kIdentifier) { + return Status::NotImplemented( + fmt::format("cannot search the {} index of table {} in snapshot {}: Paimon C++ is " + "built without its global indexer", + index_type, table_path, snapshot_id)); + } + // Any other index type without a registered indexer cannot be read, as in GlobalIndexScan. + return false; +} + +Result> DataEvolutionFullTextScan::CreatePlan() { + std::string branch = BranchManager::NormalizeBranch(options_.GetBranch()); + SnapshotManager snapshot_manager(options_.GetFileSystem(), root_path_, branch); + TagManager tag_manager(options_.GetFileSystem(), root_path_, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional snapshot, + ResolveSnapshot(options_, snapshot_manager, tag_manager)); + if (!snapshot) { + return std::make_shared(std::nullopt, + std::vector>()); + } + + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr index_manifest_file, + IndexManifestFile::Create(options_.GetFileSystem(), options_.GetManifestFormat(), + options_.GetManifestCompression(), path_factory_, + options_.GetBucket(), pool_, options_)); + IndexFileHandler index_file_handler(options_.GetFileSystem(), std::move(index_manifest_file), + std::make_shared(path_factory_), + options_.DeletionVectorsBitmap64(), pool_); + PAIMON_ASSIGN_OR_RAISE(std::vector partition_fields, + table_schema_->GetFields(table_schema_->PartitionKeys())); + auto partition_schema = DataField::ConvertDataFieldsToArrowSchema(partition_fields); + int32_t field_id = text_field_.Id(); + std::function(const IndexManifestEntry&)> filter = + [&](const IndexManifestEntry& entry) -> Result { + if (partition_filter_) { + PAIMON_ASSIGN_OR_RAISE(bool accepted, + partition_filter_->Test(partition_schema, entry.partition)); + if (!accepted) { + return false; + } + } + const std::optional& meta = entry.index_file->GetGlobalIndexMeta(); + return meta && IndexesField(meta.value(), field_id); + }; + PAIMON_ASSIGN_OR_RAISE(std::vector entries, + index_file_handler.Scan(snapshot.value(), filter)); + + std::map serves_full_text_search; + std::vector> full_text_index_files; + for (const auto& entry : entries) { + const std::string& index_type = entry.index_file->IndexType(); + auto iter = serves_full_text_search.find(index_type); + if (iter == serves_full_text_search.end()) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr indexer, + GlobalIndexerFactory::Get(index_type, options_.ToMap())); + PAIMON_ASSIGN_OR_RAISE(bool served, ServesFullTextSearch(index_type, indexer.get(), + root_path_, snapshot->Id())); + iter = serves_full_text_search.emplace(index_type, served).first; + } + if (iter->second) { + full_text_index_files.push_back(entry.index_file); + } + } + + std::vector> splits; + for (auto& selection : ChooseIndexRanges(full_text_index_files, field_id)) { + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + snapshot->Id(), text_field_.Name(), selection.file_range.from, + selection.file_range.to, std::move(selection.files), + std::move(selection.search_ranges))); + splits.push_back(std::move(split)); + } + return std::make_shared(snapshot->Id(), std::move(splits)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan.h b/src/paimon/core/table/source/data_evolution_full_text_scan.h new file mode 100644 index 000000000..95cee1d49 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_full_text_scan.h @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include + +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/data_field.h" +#include "paimon/core/core_options.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/table/source/full_text_scan.h" +#include "paimon/utils/range.h" + +namespace paimon { +/// `FullTextScan` of a data-evolution table, aligned with Java `DataEvolutionFullTextScan`: one +/// split per index shard of the searched column, planned from the global index files of one +/// snapshot. +class DataEvolutionFullTextScan : public FullTextScan { + public: + /// The index shard chosen to rank some rows of the searched column. + struct IndexRangeSelection { + /// The physical row range of the shard. + Range file_range; + /// The identity of the index of the shard; see `IndexIdentity()`. + std::string identity; + /// The files of the shard, in index manifest order. + std::vector> files; + /// The sorted, disjoint and non-adjacent row ranges the shard ranks. + std::vector search_ranges; + }; + + /// @param root_path Root directory of the table. + /// @param table_schema Schema of the table. + /// @param options Options of the table, merged with those of the search. + /// @param path_factory Path factory of the table. + /// @param partition_filter Predicate over the partition fields, or null for all partitions. + /// @param text_field The searched column. + /// @param pool Memory pool for reading index manifests. + DataEvolutionFullTextScan(const std::string& root_path, + const std::shared_ptr& table_schema, + const CoreOptions& options, + const std::shared_ptr& path_factory, + const std::shared_ptr& partition_filter, + const DataField& text_field, const std::shared_ptr& pool); + + Result> CreatePlan() override; + + /// Returns whether `meta` indexes the field `field_id`, as its indexed field or as one of its + /// extra fields. + static bool IndexesField(const GlobalIndexMeta& meta, int32_t field_id); + + /// Returns the identity of the index that `file` belongs to: its index type, indexed field id + /// and extra field ids, as `type|index_field_id|[extra_field_ids]`. Files of one identity + /// differ only by shard, while files of different identities are never read together. + static std::string IndexIdentity(const IndexFileMeta& file); + + /// Chooses one index shard for every row that a shard of the field `field_id` covers, as Java + /// `DataEvolutionFullTextScan#chooseIndexRanges` does (apache/paimon#8547). The shards are + /// grouped by physical range and identity, the coverage is split at every shard boundary, and + /// each piece goes to the first of the shards covering it: a shard whose indexed field is + /// `field_id` before one that carries it as an extra field, then by identity, start and end. + /// Adjacent pieces of a shard are merged. + /// @param index_files Global index files; those that do not index `field_id` are ignored. + /// @param field_id The id of the searched field. + /// @return The chosen shards, ordered by start, end and identity. + static std::vector ChooseIndexRanges( + const std::vector>& index_files, int32_t field_id); + + /// Returns whether the index files of `index_type` serve a full-text search. A `full-text` or + /// `lucene-fts` index without a registered indexer fails instead: Paimon C++ is then built + /// without its backend, and skipping it would drop its rows from the result. + /// @param index_type The index type of the files. + /// @param indexer The registered indexer of `index_type`, or null if there is none. + /// @param table_path The table of the files, for the error. + /// @param snapshot_id The snapshot the files are planned from, for the error. + static Result ServesFullTextSearch(const std::string& index_type, + const GlobalIndexer* indexer, + const std::string& table_path, int64_t snapshot_id); + + private: + std::string root_path_; + std::shared_ptr table_schema_; + CoreOptions options_; + std::shared_ptr path_factory_; + std::shared_ptr partition_filter_; + DataField text_field_; + std::shared_ptr pool_; +}; + +} // namespace paimon diff --git a/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp new file mode 100644 index 000000000..d73727f18 --- /dev/null +++ b/src/paimon/core/table/source/data_evolution_full_text_scan_test.cpp @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/core/table/source/data_evolution_full_text_scan.h" + +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/global_index/global_indexer.h" +#include "paimon/global_index/global_indexer_factory.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" + +namespace paimon::test { +namespace { +constexpr int32_t kTextFieldId = 1; + +std::shared_ptr NewIndexFile( + const std::string& file_name, const std::string& index_type, const Range& range, + int32_t index_field_id, + const std::optional>& extra_field_ids = std::nullopt) { + GlobalIndexMeta meta(range.from, range.to, index_field_id, extra_field_ids, + /*index_meta=*/nullptr); + return std::make_shared(index_type, file_name, /*file_size=*/128, + /*row_count=*/range.Count(), /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, meta); +} + +struct Selection { + Range file_range; + std::vector file_names; + std::vector search_ranges; + + bool operator==(const Selection& other) const { + return file_range == other.file_range && file_names == other.file_names && + search_ranges == other.search_ranges; + } +}; + +std::ostream& operator<<(std::ostream& os, const Selection& selection) { + os << "{range=" << selection.file_range.ToString() << ", files=["; + for (const auto& name : selection.file_names) { + os << name << ","; + } + os << "], search=["; + for (const auto& range : selection.search_ranges) { + os << range.ToString() << ","; + } + return os << "]}"; +} + +std::vector Choose(const std::vector>& files) { + std::vector selections; + for (const auto& selection : + DataEvolutionFullTextScan::ChooseIndexRanges(files, kTextFieldId)) { + std::vector file_names; + for (const auto& file : selection.files) { + file_names.push_back(file->FileName()); + } + selections.push_back(Selection{selection.file_range, file_names, selection.search_ranges}); + } + return selections; +} +} // namespace + +TEST(DataEvolutionFullTextScanTest, TestIndexIdentityAndIndexedFields) { + auto dedicated = NewIndexFile("a", "full-text", Range(0, 9), kTextFieldId); + auto multi_field = NewIndexFile("b", "lumina", Range(0, 9), /*index_field_id=*/2, + std::vector({1, 3})); + ASSERT_EQ(DataEvolutionFullTextScan::IndexIdentity(*dedicated), "full-text|1|[]"); + ASSERT_EQ(DataEvolutionFullTextScan::IndexIdentity(*multi_field), "lumina|2|[1, 3]"); + auto empty_extra_fields = + NewIndexFile("c", "full-text", Range(0, 9), kTextFieldId, std::vector()); + ASSERT_EQ(DataEvolutionFullTextScan::IndexIdentity(*empty_extra_fields), "full-text|1|[]"); + + const GlobalIndexMeta& meta = multi_field->GetGlobalIndexMeta().value(); + ASSERT_TRUE(DataEvolutionFullTextScan::IndexesField(meta, 1)); + ASSERT_TRUE(DataEvolutionFullTextScan::IndexesField(meta, 2)); + ASSERT_TRUE(DataEvolutionFullTextScan::IndexesField(meta, 3)); + ASSERT_FALSE(DataEvolutionFullTextScan::IndexesField(meta, 4)); +} + +TEST(DataEvolutionFullTextScanTest, TestDisjointShards) { + const int64_t max_row_id = std::numeric_limits::max(); + std::vector> files = { + NewIndexFile("shard-1", "full-text", Range(100, 199), kTextFieldId), + NewIndexFile("shard-0-a", "full-text", Range(0, 99), kTextFieldId), + NewIndexFile("shard-0-b", "full-text", Range(0, 99), kTextFieldId), + NewIndexFile("shard-3", "full-text", Range(300, 399), kTextFieldId), + NewIndexFile("shard-4", "full-text", Range(400, max_row_id), kTextFieldId), + NewIndexFile("other", "full-text", Range(0, 399), /*index_field_id=*/5)}; + std::vector expected = { + {Range(0, 99), {"shard-0-a", "shard-0-b"}, {Range(0, 99)}}, + {Range(100, 199), {"shard-1"}, {Range(100, 199)}}, + {Range(300, 399), {"shard-3"}, {Range(300, 399)}}, + {Range(400, max_row_id), {"shard-4"}, {Range(400, max_row_id)}}}; + ASSERT_EQ(Choose(files), expected); + ASSERT_TRUE(Choose({}).empty()); +} + +TEST(DataEvolutionFullTextScanTest, TestOverlappingShardsOfOneIndex) { + std::vector> files = { + NewIndexFile("0-2", "full-text", Range(0, 2), kTextFieldId), + NewIndexFile("1-1", "full-text", Range(1, 1), kTextFieldId), + NewIndexFile("1-3", "full-text", Range(1, 3), kTextFieldId)}; + std::vector expected = {{Range(0, 2), {"0-2"}, {Range(0, 2)}}, + {Range(1, 3), {"1-3"}, {Range(3, 3)}}}; + ASSERT_EQ(Choose(files), expected); +} + +TEST(DataEvolutionFullTextScanTest, TestDedicatedIndexBeforeExtraField) { + std::vector> files = { + NewIndexFile("multi", "test-full-text", Range(0, 99), /*index_field_id=*/2, + std::vector({kTextFieldId})), + NewIndexFile("dedicated", "test-full-text", Range(50, 149), kTextFieldId)}; + std::vector expected = {{Range(0, 99), {"multi"}, {Range(0, 49)}}, + {Range(50, 149), {"dedicated"}, {Range(50, 149)}}}; + ASSERT_EQ(Choose(files), expected); + + ASSERT_EQ(Choose({files[0]}), + std::vector({{Range(0, 99), {"multi"}, {Range(0, 99)}}})); +} + +TEST(DataEvolutionFullTextScanTest, TestIdentityOrderBreaksTies) { + std::vector> same_range = { + NewIndexFile("lucene", "lucene-fts", Range(0, 9), kTextFieldId), + NewIndexFile("full-text", "full-text", Range(0, 9), kTextFieldId)}; + ASSERT_EQ(Choose(same_range), + std::vector({{Range(0, 9), {"full-text"}, {Range(0, 9)}}})); + + std::vector> lucene_inside = { + NewIndexFile("full-text", "full-text", Range(0, 99), kTextFieldId), + NewIndexFile("lucene", "lucene-fts", Range(50, 59), kTextFieldId)}; + ASSERT_EQ(Choose(lucene_inside), + std::vector({{Range(0, 99), {"full-text"}, {Range(0, 99)}}})); + + std::vector> full_text_inside = { + NewIndexFile("lucene", "lucene-fts", Range(0, 99), kTextFieldId), + NewIndexFile("full-text", "full-text", Range(40, 59), kTextFieldId)}; + std::vector expected = {{Range(0, 99), {"lucene"}, {Range(0, 39), Range(60, 99)}}, + {Range(40, 59), {"full-text"}, {Range(40, 59)}}}; + ASSERT_EQ(Choose(full_text_inside), expected); +} + +TEST(DataEvolutionFullTextScanTest, TestServesFullTextSearch) { + for (const std::string index_type : {"full-text", "lucene-fts"}) { + ASSERT_NOK_WITH_MSG( + DataEvolutionFullTextScan::ServesFullTextSearch(index_type, /*indexer=*/nullptr, + "/warehouse/foo.db/bar", + /*snapshot_id=*/3), + "cannot search the " + index_type + + " index of table /warehouse/foo.db/bar in snapshot 3: Paimon C++ is built without " + "its global indexer"); + } + ASSERT_OK_AND_ASSIGN(bool served, DataEvolutionFullTextScan::ServesFullTextSearch( + "no-such-index", /*indexer=*/nullptr, + "/warehouse/foo.db/bar", /*snapshot_id=*/3)); + ASSERT_FALSE(served); + ASSERT_OK_AND_ASSIGN(std::unique_ptr bitmap, + GlobalIndexerFactory::Get("bitmap", /*options=*/{})); + ASSERT_TRUE(bitmap); + ASSERT_OK_AND_ASSIGN(served, DataEvolutionFullTextScan::ServesFullTextSearch( + "bitmap", bitmap.get(), "/warehouse/foo.db/bar", + /*snapshot_id=*/3)); + ASSERT_FALSE(served); +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/source/full_text_search_builder.cpp b/src/paimon/core/table/source/full_text_search_builder.cpp new file mode 100644 index 000000000..6b85f97df --- /dev/null +++ b/src/paimon/core/table/source/full_text_search_builder.cpp @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/table/source/full_text_search_builder.h" + +#include +#include + +#include "fmt/format.h" +#include "paimon/common/predicate/predicate_filter.h" +#include "paimon/common/types/data_field.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/common/utils/string_utils.h" +#include "paimon/core/core_options.h" +#include "paimon/core/index/pk/primary_key_index_definitions.h" +#include "paimon/core/operation/file_store_scan.h" +#include "paimon/core/schema/schema_manager.h" +#include "paimon/core/schema/table_schema.h" +#include "paimon/core/table/source/data_evolution_full_text_read.h" +#include "paimon/core/table/source/data_evolution_full_text_scan.h" +#include "paimon/core/utils/branch_manager.h" +#include "paimon/core/utils/file_store_path_factory.h" +#include "paimon/defs.h" +#include "paimon/executor.h" +#include "paimon/format/file_format.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/status.h" + +namespace paimon { + +class FullTextSearchBuilder::Impl { + public: + /// What a scan and a read of the search are created from. + struct ResolvedSearch { + std::string root_path; + std::shared_ptr table_schema; + CoreOptions options; + std::shared_ptr path_factory; + std::shared_ptr partition_filter; + DataField text_field; + }; + + /// Loads the table and checks the settings shared by scans and reads. + Result Resolve() const; + + Status CheckLimit() const; + + std::unique_ptr CreateScan(const ResolvedSearch& search) const; + + /// Creates a read of `search`; the limit must have passed `CheckLimit()`. + Result> CreateRead(const ResolvedSearch& search) const; + + std::string table_path_; + std::vector> partition_filters_; + std::optional limit_; + std::optional field_name_; + std::string query_; + std::map options_; + std::shared_ptr memory_pool_ = GetDefaultPool(); + std::shared_ptr executor_; + std::shared_ptr file_system_; +}; + +namespace { +/// Rejects a column with a primary-key full-text index, which Java searches with +/// `PrimaryKeyFullTextScan` and `PrimaryKeyFullTextRead` instead. +Status CheckNoPrimaryKeyFullTextIndex(const TableSchema& table_schema, const CoreOptions& options, + const DataField& text_field) { + if (options.DataEvolutionEnabled()) { + return Status::OK(); + } + const auto& schema_options = table_schema.Options(); + auto columns = schema_options.find(Options::PK_FULL_TEXT_INDEX_COLUMNS); + if (columns == schema_options.end() || StringUtils::IsNullOrWhitespaceOnly(columns->second)) { + return Status::OK(); + } + PAIMON_ASSIGN_OR_RAISE(PrimaryKeyIndexDefinitions definitions, + PrimaryKeyIndexDefinitions::Create(table_schema)); + for (const auto& definition : definitions.Definitions()) { + if (definition.GetFamily() == PrimaryKeyIndexDefinition::Family::FULL_TEXT && + definition.FieldId() == text_field.Id()) { + return Status::NotImplemented(fmt::format( + "full-text search of column {} with a primary-key full-text index is not " + "supported yet", + text_field.Name())); + } + } + return Status::OK(); +} +} // namespace + +Result FullTextSearchBuilder::Impl::Resolve() const { + if (!field_name_) { + return Status::Invalid("Query must be set via SetQuery()"); + } + if (memory_pool_ == nullptr) { + return Status::Invalid("memory pool is null pointer"); + } + PAIMON_ASSIGN_OR_RAISE(std::string root_path, PathUtil::NormalizePath(table_path_)); + if (root_path.empty()) { + return Status::Invalid("cannot search with empty table path"); + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions tmp_options, CoreOptions::FromMap(options_, file_system_)); + std::string branch = BranchManager::NormalizeBranch(tmp_options.GetBranch()); + SchemaManager schema_manager(tmp_options.GetFileSystem(), root_path, branch); + PAIMON_ASSIGN_OR_RAISE(std::optional> latest_schema, + schema_manager.Latest()); + if (!latest_schema) { + return Status::Invalid( + fmt::format("schema file not found in path {}, branch {}", root_path, branch)); + } + std::shared_ptr table_schema = latest_schema.value(); + std::map merged_options = table_schema->Options(); + for (const auto& [key, value] : options_) { + merged_options[key] = value; + } + PAIMON_ASSIGN_OR_RAISE(CoreOptions options, CoreOptions::FromMap(merged_options, file_system_)); + if (options.GetFullTextIndexSearchMode() != GlobalIndexSearchMode::FAST) { + return Status::NotImplemented(fmt::format( + "full-text search only supports the fast {} (or {}), which searches only the rows " + "that an index covers; full and detail are not supported yet", + Options::FULL_TEXT_INDEX_SEARCH_MODE, Options::GLOBAL_INDEX_SEARCH_MODE)); + } + + Result text_field = table_schema->GetField(field_name_.value()); + if (!text_field.ok()) { + return Status::Invalid( + fmt::format("Text column '{}' does not exist.", field_name_.value())); + } + PAIMON_RETURN_NOT_OK( + CheckNoPrimaryKeyFullTextIndex(*table_schema, options, text_field.value())); + + auto arrow_schema = DataField::ConvertDataFieldsToArrowSchema(table_schema->Fields()); + PAIMON_ASSIGN_OR_RAISE(std::vector external_paths, options.CreateExternalPaths()); + PAIMON_ASSIGN_OR_RAISE(std::optional global_index_external_path, + options.CreateGlobalIndexExternalPath()); + PAIMON_ASSIGN_OR_RAISE( + std::shared_ptr path_factory, + FileStorePathFactory::Create( + root_path, arrow_schema, table_schema->PartitionKeys(), + options.GetPartitionDefaultName(), options.GetFileFormat()->Identifier(), + options.DataFilePrefix(), options.LegacyPartitionNameEnabled(), external_paths, + global_index_external_path, options.IndexFileInDataFileDir(), memory_pool_)); + std::shared_ptr partition_filter; + if (!partition_filters_.empty()) { + PAIMON_ASSIGN_OR_RAISE(partition_filter, + FileStoreScan::CreatePartitionPredicate( + table_schema->PartitionKeys(), options.GetPartitionDefaultName(), + arrow_schema, partition_filters_)); + } + return ResolvedSearch{root_path, table_schema, std::move(options), + path_factory, partition_filter, std::move(text_field).value()}; +} + +Status FullTextSearchBuilder::Impl::CheckLimit() const { + if (!limit_ || limit_.value() <= 0) { + return Status::Invalid("Limit must be positive, set via SetLimit()"); + } + return Status::OK(); +} + +std::unique_ptr FullTextSearchBuilder::Impl::CreateScan( + const ResolvedSearch& search) const { + return std::make_unique( + search.root_path, search.table_schema, search.options, search.path_factory, + search.partition_filter, search.text_field, memory_pool_); +} + +Result> FullTextSearchBuilder::Impl::CreateRead( + const ResolvedSearch& search) const { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, + DataEvolutionFullTextRead::Create( + search.root_path, search.table_schema, search.options, + search.path_factory, search.partition_filter, search.text_field, + query_, limit_.value(), executor_, memory_pool_)); + return std::unique_ptr(std::move(read)); +} + +FullTextSearchBuilder::FullTextSearchBuilder(const std::string& table_path) + : impl_(std::make_unique()) { + impl_->table_path_ = table_path; +} + +FullTextSearchBuilder::~FullTextSearchBuilder() = default; + +FullTextSearchBuilder& FullTextSearchBuilder::SetPartitionFilter( + const std::vector>& partition_filters) { + impl_->partition_filters_ = partition_filters; + return *this; +} + +FullTextSearchBuilder& FullTextSearchBuilder::SetLimit(int32_t limit) { + impl_->limit_ = limit; + return *this; +} + +FullTextSearchBuilder& FullTextSearchBuilder::SetQuery(const std::string& field_name, + const std::string& query) { + impl_->field_name_ = field_name; + impl_->query_ = query; + return *this; +} + +FullTextSearchBuilder& FullTextSearchBuilder::AddOption(const std::string& key, + const std::string& value) { + impl_->options_[key] = value; + return *this; +} + +FullTextSearchBuilder& FullTextSearchBuilder::SetOptions( + const std::map& options) { + impl_->options_ = options; + return *this; +} + +FullTextSearchBuilder& FullTextSearchBuilder::WithMemoryPool( + const std::shared_ptr& memory_pool) { + impl_->memory_pool_ = memory_pool; + return *this; +} + +FullTextSearchBuilder& FullTextSearchBuilder::WithExecutor( + const std::shared_ptr& executor) { + impl_->executor_ = executor; + return *this; +} + +FullTextSearchBuilder& FullTextSearchBuilder::WithFileSystem( + const std::shared_ptr& file_system) { + impl_->file_system_ = file_system; + return *this; +} + +Result> FullTextSearchBuilder::NewFullTextScan() const { + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); + return impl_->CreateScan(search); +} + +Result> FullTextSearchBuilder::NewFullTextRead() const { + PAIMON_RETURN_NOT_OK(impl_->CheckLimit()); + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); + return impl_->CreateRead(search); +} + +Result> FullTextSearchBuilder::ExecuteLocal() const { + PAIMON_RETURN_NOT_OK(impl_->CheckLimit()); + // The scan and the read share one load of the table, so they see the same schema. + PAIMON_ASSIGN_OR_RAISE(Impl::ResolvedSearch search, impl_->Resolve()); + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr read, impl_->CreateRead(search)); + std::unique_ptr scan = impl_->CreateScan(search); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + return read->Read(*plan); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/full_text_search_split.cpp b/src/paimon/core/table/source/full_text_search_split.cpp new file mode 100644 index 000000000..308afd5d9 --- /dev/null +++ b/src/paimon/core/table/source/full_text_search_split.cpp @@ -0,0 +1,399 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/table/source/full_text_search_split.h" + +#include +#include +#include +#include +#include +#include + +#include "arrow/type.h" +#include "fmt/format.h" +#include "fmt/ranges.h" +#include "paimon/common/data/binary_array_writer.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_section.h" +#include "paimon/common/data/serializer/binary_row_serializer.h" +#include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/core/index/index_file_meta_serializer.h" +#include "paimon/core/table/source/data_evolution_full_text_scan.h" +#include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/io/byte_array_input_stream.h" +#include "paimon/io/data_input_stream.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_segment.h" +#include "paimon/status.h" + +namespace paimon { +namespace { +constexpr int64_t kSearchRowRangeBytes = 2 * sizeof(int64_t); +/// A serialized index file is a binary row that starts with its int32 length. +constexpr int64_t kMinIndexFileBytes = sizeof(int32_t); + +/// Checks that `count` entries of at least `min_entry_bytes` bytes each fit in the rest of `in`, +/// so a corrupt count fails before anything is allocated for it. +Status CheckEntryCount(const DataInputStream& in, int32_t count, int64_t min_entry_bytes, + const char* entries) { + if (count < 0) { + return Status::Invalid( + fmt::format("invalid full-text search split, {} {}", count, entries)); + } + PAIMON_ASSIGN_OR_RAISE(int64_t pos, in.GetPos()); + PAIMON_ASSIGN_OR_RAISE(int64_t length, in.Length()); + if (count > (length - pos) / min_entry_bytes) { + return Status::Invalid( + fmt::format("invalid full-text search split, {} {} exceed the remaining {} bytes", + count, entries, length - pos)); + } + return Status::OK(); +} + +Status CheckBinaryRowLayout(const MemorySegment& segment, int64_t offset, int64_t size, + const arrow::StructType& type); + +Status CheckBinaryArrayLayout(const MemorySegment& segment, int64_t offset, int64_t size, + const arrow::DataType& element_type); + +/// Checks the value of `type` in the 8-byte slot at `slot` of the binary row or array that spans +/// [`base`, `base` + `base_size`): an inline string or binary must fit in the slot, and any other +/// variable-length value must lie within the row or array. +Status CheckSlotLayout(const MemorySegment& segment, int64_t base, int64_t base_size, int64_t slot, + const arrow::DataType& type) { + switch (type.id()) { + case arrow::Type::BOOL: + case arrow::Type::INT8: + case arrow::Type::INT16: + case arrow::Type::INT32: + case arrow::Type::DATE32: + case arrow::Type::INT64: + case arrow::Type::FLOAT: + case arrow::Type::DOUBLE: + return Status::OK(); + case arrow::Type::STRING: + case arrow::Type::BINARY: + case arrow::Type::STRUCT: + case arrow::Type::LIST: + break; + default: + return Status::NotImplemented(fmt::format( + "cannot check the layout of {} in a full-text search split", type.ToString())); + } + auto offset_and_size = segment.GetValue(static_cast(slot)); + bool bytes_type = type.id() == arrow::Type::STRING || type.id() == arrow::Type::BINARY; + if (bytes_type && (offset_and_size & BinarySection::HIGHEST_FIRST_BIT) != 0) { + int64_t length = (offset_and_size & BinarySection::HIGHEST_SECOND_TO_EIGHTH_BIT) >> 56; + if (length >= static_cast(sizeof(int64_t))) { + return Status::Invalid(fmt::format( + "invalid full-text search split, inline {} of {} bytes", type.ToString(), length)); + } + return Status::OK(); + } + int64_t value_offset = offset_and_size >> 32; + int64_t value_size = static_cast(offset_and_size); + if (value_offset < 0 || value_size < 0 || value_offset + value_size > base_size) { + return Status::Invalid(fmt::format( + "invalid full-text search split, {} of {} bytes at offset {} exceeds {} bytes", + type.ToString(), value_size, value_offset, base_size)); + } + if (type.id() == arrow::Type::STRUCT) { + return CheckBinaryRowLayout(segment, base + value_offset, value_size, + static_cast(type)); + } + if (type.id() == arrow::Type::LIST) { + return CheckBinaryArrayLayout(segment, base + value_offset, value_size, + *static_cast(type).value_type()); + } + return Status::OK(); +} + +/// Checks that the binary row of `type` at [`offset`, `offset` + `size`) of `segment` has its +/// fixed-length part and that its variable-length values lie within it, so that decoding a +/// corrupt row fails instead of reading out of bounds. +Status CheckBinaryRowLayout(const MemorySegment& segment, int64_t offset, int64_t size, + const arrow::StructType& type) { + int32_t arity = type.num_fields(); + int32_t fixed_part_size = BinaryRow::CalculateFixPartSizeInBytes(arity); + if (size < fixed_part_size) { + return Status::Invalid(fmt::format( + "invalid full-text search split, binary row of {} bytes without its {}-byte " + "fixed-length part", + size, fixed_part_size)); + } + int64_t slots = offset + BinaryRow::CalculateBitSetWidthInBytes(arity); + for (int32_t i = 0; i < arity; ++i) { + const std::shared_ptr& field = type.field(i); + // Readers check a nullable field for null before reading it. + if (field->nullable() && MemorySegmentUtils::BitGet(segment, static_cast(offset), + i + BinaryRow::HEADER_SIZE_IN_BITS)) { + continue; + } + PAIMON_RETURN_NOT_OK(CheckSlotLayout(segment, offset, size, + slots + static_cast(i) * 8, *field->type())); + } + return Status::OK(); +} + +/// Checks the binary array of `element_type` at [`offset`, `offset` + `size`) of `segment` as +/// `CheckBinaryRowLayout()` checks a row. +Status CheckBinaryArrayLayout(const MemorySegment& segment, int64_t offset, int64_t size, + const arrow::DataType& element_type) { + constexpr int64_t kCountBytes = sizeof(int32_t); + if (size < kCountBytes) { + return Status::Invalid( + fmt::format("invalid full-text search split, binary array of {} bytes", size)); + } + auto count = segment.GetValue(static_cast(offset)); + int32_t element_size = BinaryArrayWriter::GetElementSize(element_type.id()); + // `BinaryArray::CalculateHeaderInBytes()`, in int64 so that a corrupt count cannot overflow. + int64_t header_size = kCountBytes + (static_cast(count) + 31) / 32 * 4; + if (count < 0 || header_size + static_cast(count) * element_size > size) { + return Status::Invalid( + fmt::format("invalid full-text search split, binary array of {} elements in {} bytes", + count, size)); + } + // Readers such as `RowArrayDataToDvRanges()` read every element without checking for null. + for (int32_t i = 0; i < count; ++i) { + PAIMON_RETURN_NOT_OK(CheckSlotLayout( + segment, offset, size, offset + header_size + static_cast(i) * element_size, + element_type)); + } + return Status::OK(); +} + +/// Reads the index files that `IndexFileMetaSerializer::SerializeList()` wrote. Unlike +/// `DeserializeList()`, it checks the count, and the length and layout of each file, before +/// decoding anything. +Result>> ReadIndexFiles( + DataInputStream* in, const std::shared_ptr& pool) { + PAIMON_ASSIGN_OR_RAISE(int32_t count, in->ReadValue()); + PAIMON_RETURN_NOT_OK(CheckEntryCount(*in, count, kMinIndexFileBytes, "index files")); + const auto& file_type = static_cast(*IndexFileMeta::DataType()); + BinaryRowSerializer row_serializer(file_type.num_fields(), pool); + IndexFileMetaSerializer serializer(pool); + std::vector> files; + files.reserve(count); + for (int32_t i = 0; i < count; ++i) { + PAIMON_ASSIGN_OR_RAISE(int64_t pos, in->GetPos()); + PAIMON_RETURN_NOT_OK(BinaryRowSerializer::Skip(in)); + PAIMON_RETURN_NOT_OK(in->Seek(pos)); + PAIMON_ASSIGN_OR_RAISE(BinaryRow row, row_serializer.Deserialize(in)); + PAIMON_RETURN_NOT_OK(CheckBinaryRowLayout(row.GetSegment(), row.GetOffset(), + row.GetSizeInBytes(), file_type)); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr file, serializer.FromRow(row)); + files.push_back(std::move(file)); + } + return files; +} +} // namespace + +Result> IndexFullTextSearchSplitImpl::Create( + int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, + int64_t row_range_end, std::vector> full_text_index_files, + std::vector search_row_ranges) { + if (column_name.empty()) { + return Status::Invalid("full-text search split requires a column name"); + } + // `Serialize()` writes the column name with a 16-bit length. + if (column_name.size() > std::numeric_limits::max()) { + return Status::Invalid(fmt::format( + "full-text search split requires a column name of at most {} bytes, but got {} bytes", + std::numeric_limits::max(), column_name.size())); + } + // The row count of the range must fit in int64, as reads take its `Range::Count()`. + if (row_range_start < 0 || row_range_start > row_range_end || + row_range_end - row_range_start == std::numeric_limits::max()) { + return Status::Invalid( + fmt::format("invalid physical index range [{}, {}]", row_range_start, row_range_end)); + } + if (full_text_index_files.empty()) { + return Status::Invalid("full-text search split requires at least one index file"); + } + const std::shared_ptr& first_file = full_text_index_files.front(); + std::string identity; + for (const auto& file : full_text_index_files) { + if (!file || !file->GetGlobalIndexMeta()) { + return Status::Invalid("full-text search split requires global index files"); + } + const GlobalIndexMeta& meta = file->GetGlobalIndexMeta().value(); + if (meta.row_range_start != row_range_start || meta.row_range_end != row_range_end) { + return Status::Invalid( + fmt::format("index file {} covers [{}, {}], not the physical index range [{}, {}]", + file->FileName(), meta.row_range_start, meta.row_range_end, + row_range_start, row_range_end)); + } + std::string file_identity = DataEvolutionFullTextScan::IndexIdentity(*file); + if (identity.empty()) { + identity = std::move(file_identity); + } else if (file_identity != identity) { + return Status::Invalid( + fmt::format("index files {} and {} of one full-text search split belong to " + "different indexes", + first_file->FileName(), file->FileName())); + } + } + if (search_row_ranges.empty()) { + return Status::Invalid("full-text search split requires at least one search row range"); + } + for (size_t i = 0; i < search_row_ranges.size(); ++i) { + const Range& range = search_row_ranges[i]; + if (range.from > range.to || range.from < row_range_start || range.to > row_range_end) { + return Status::Invalid( + fmt::format("search range {} is outside physical index range [{}, {}]", + range.ToString(), row_range_start, row_range_end)); + } + if (i == 0) { + continue; + } + // Adjacent ranges are merged, so a range starts after the row following the previous one. + const Range& previous = search_row_ranges[i - 1]; + if (range.from <= previous.to || range.from - 1 == previous.to) { + return Status::Invalid( + fmt::format("search ranges {} and {} are not sorted, disjoint and non-adjacent", + previous.ToString(), range.ToString())); + } + } + return std::shared_ptr(new IndexFullTextSearchSplitImpl( + snapshot_id, column_name, row_range_start, row_range_end, std::move(full_text_index_files), + std::move(search_row_ranges))); +} + +bool IndexFullTextSearchSplitImpl::operator==(const IndexFullTextSearchSplitImpl& other) const { + if (this == &other) { + return true; + } + if (snapshot_id_ != other.snapshot_id_ || column_name_ != other.column_name_ || + row_range_start_ != other.row_range_start_ || row_range_end_ != other.row_range_end_ || + search_row_ranges_ != other.search_row_ranges_ || + full_text_index_files_.size() != other.full_text_index_files_.size()) { + return false; + } + for (size_t i = 0; i < full_text_index_files_.size(); ++i) { + if (!(*full_text_index_files_[i] == *other.full_text_index_files_[i])) { + return false; + } + } + return true; +} + +std::string IndexFullTextSearchSplitImpl::ToString() const { + std::vector ranges; + ranges.reserve(search_row_ranges_.size()); + for (const auto& range : search_row_ranges_) { + ranges.push_back(range.ToString()); + } + std::vector files; + files.reserve(full_text_index_files_.size()); + for (const auto& file : full_text_index_files_) { + files.push_back(file->ToString()); + } + return fmt::format( + "IndexFullTextSearchSplit{{snapshotId={}, columnName={}, rowRangeStart={}, " + "rowRangeEnd={}, searchRowRanges=[{}], fullTextIndexFiles=[{}]}}", + snapshot_id_, column_name_, row_range_start_, row_range_end_, fmt::join(ranges, ", "), + fmt::join(files, ", ")); +} + +Result FullTextSearchSplit::Serialize( + const std::shared_ptr& split, const std::shared_ptr& pool) { + auto index_split = std::dynamic_pointer_cast(split); + if (!index_split) { + return Status::Invalid("invalid full-text search split, cannot cast to an index split"); + } + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); + out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); + out.WriteValue(IndexFullTextSearchSplitImpl::VERSION); + out.WriteValue(index_split->SnapshotId()); + out.WriteString(index_split->ColumnName()); + out.WriteValue(index_split->RowRangeStart()); + out.WriteValue(index_split->RowRangeEnd()); + const std::vector& search_row_ranges = index_split->SearchRowRanges(); + out.WriteValue(static_cast(search_row_ranges.size())); + for (const auto& range : search_row_ranges) { + out.WriteValue(range.from); + out.WriteValue(range.to); + } + IndexFileMetaSerializer serializer(pool); + PAIMON_RETURN_NOT_OK(serializer.SerializeList(index_split->FullTextIndexFiles(), &out)); + PAIMON_UNIQUE_PTR bytes = + MemorySegmentUtils::CopyToBytes(out.Segments(), 0, out.CurrentSize(), pool.get()); + return std::string(bytes->data(), bytes->size()); +} + +Result> FullTextSearchSplit::Deserialize( + const char* buffer, size_t length, const std::shared_ptr& pool) { + if (buffer == nullptr) { + return Status::Invalid("cannot deserialize a full-text search split from a null buffer"); + } + if (length > static_cast(std::numeric_limits::max())) { + return Status::Invalid( + fmt::format("invalid full-text search split, {} bytes exceed the supported maximum {}", + length, std::numeric_limits::max())); + } + auto input_stream = + std::make_shared(buffer, static_cast(length)); + DataInputStream in(input_stream); + PAIMON_ASSIGN_OR_RAISE(int64_t magic, in.ReadValue()); + if (magic != IndexFullTextSearchSplitImpl::MAGIC) { + return Status::Invalid("invalid full-text search split, unknown magic number"); + } + PAIMON_ASSIGN_OR_RAISE(int32_t version, in.ReadValue()); + if (version < 1 || version > IndexFullTextSearchSplitImpl::VERSION) { + return Status::Invalid( + fmt::format("unsupported IndexFullTextSearchSplit version: {}", version)); + } + PAIMON_ASSIGN_OR_RAISE(int64_t snapshot_id, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(std::string column_name, in.ReadString()); + PAIMON_ASSIGN_OR_RAISE(int64_t row_range_start, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int64_t row_range_end, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int32_t range_count, in.ReadValue()); + PAIMON_RETURN_NOT_OK( + CheckEntryCount(in, range_count, kSearchRowRangeBytes, "search row ranges")); + std::vector search_row_ranges; + search_row_ranges.reserve(range_count); + for (int32_t i = 0; i < range_count; ++i) { + PAIMON_ASSIGN_OR_RAISE(int64_t from, in.ReadValue()); + PAIMON_ASSIGN_OR_RAISE(int64_t to, in.ReadValue()); + if (from > to) { + return Status::Invalid(fmt::format( + "invalid full-text search split, search row range [{}, {}] ends before it starts", + from, to)); + } + search_row_ranges.emplace_back(from, to); + } + PAIMON_ASSIGN_OR_RAISE(std::vector> full_text_index_files, + ReadIndexFiles(&in, pool)); + PAIMON_ASSIGN_OR_RAISE(int64_t pos, in.GetPos()); + PAIMON_ASSIGN_OR_RAISE(int64_t stream_length, in.Length()); + if (pos != stream_length) { + return Status::Invalid( + fmt::format("invalid full-text search split, remaining {} bytes after deserializing", + stream_length - pos)); + } + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + snapshot_id, column_name, row_range_start, row_range_end, + std::move(full_text_index_files), std::move(search_row_ranges))); + return std::shared_ptr(std::move(split)); +} + +} // namespace paimon diff --git a/src/paimon/core/table/source/full_text_search_split_test.cpp b/src/paimon/core/table/source/full_text_search_split_test.cpp new file mode 100644 index 000000000..606e4438a --- /dev/null +++ b/src/paimon/core/table/source/full_text_search_split_test.cpp @@ -0,0 +1,377 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "paimon/table/source/full_text_search_split.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gtest/gtest.h" +#include "paimon/common/data/binary_row.h" +#include "paimon/common/data/binary_section.h" +#include "paimon/common/io/memory_segment_output_stream.h" +#include "paimon/common/memory/memory_segment_utils.h" +#include "paimon/core/index/global_index_meta.h" +#include "paimon/core/index/index_file_meta.h" +#include "paimon/core/index/index_file_meta_serializer.h" +#include "paimon/core/table/source/index_full_text_search_split_impl.h" +#include "paimon/memory/bytes.h" +#include "paimon/memory/memory_pool.h" +#include "paimon/testing/utils/testharness.h" +#include "paimon/utils/range.h" + +namespace paimon::test { +namespace { +std::shared_ptr NewIndexFile( + const std::string& file_name, const std::string& index_type, const Range& range, + int32_t index_field_id, const std::optional>& extra_field_ids, + const std::shared_ptr& pool) { + GlobalIndexMeta meta(range.from, range.to, index_field_id, extra_field_ids, + std::make_shared(R"({"tokenizer":"default"})", pool.get())); + return std::make_shared(index_type, file_name, /*file_size=*/128, + /*row_count=*/range.Count(), /*dv_ranges=*/std::nullopt, + /*external_path=*/std::nullopt, meta); +} + +std::string ToBytes(MemorySegmentOutputStream* out, const std::shared_ptr& pool) { + PAIMON_UNIQUE_PTR bytes = + MemorySegmentUtils::CopyToBytes(out->Segments(), 0, out->CurrentSize(), pool.get()); + return std::string(bytes->data(), bytes->size()); +} + +std::string FromHex(const std::string& hex) { + std::string bytes; + for (size_t i = 0; i + 1 < hex.size(); i += 2) { + bytes.push_back(static_cast(std::stoi(hex.substr(i, 2), nullptr, 16))); + } + return bytes; +} + +std::string SplitBytes(const std::function& write_body, + const std::shared_ptr& pool) { + const int64_t snapshot_id = 1; + const int64_t row_range_start = 0; + const int64_t row_range_end = 9; + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool); + out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); + out.WriteValue(IndexFullTextSearchSplitImpl::VERSION); + out.WriteValue(snapshot_id); + out.WriteString("f0"); + out.WriteValue(row_range_start); + out.WriteValue(row_range_end); + write_body(&out); + return ToBytes(&out, pool); +} +} // namespace + +class FullTextSearchSplitTest : public ::testing::Test { + protected: + std::shared_ptr pool_ = GetDefaultPool(); +}; + +TEST_F(FullTextSearchSplitTest, TestSerializeRoundTrip) { + std::vector> files = { + NewIndexFile("full-text-0.index", "full-text", Range(100, 199), /*index_field_id=*/2, + /*extra_field_ids=*/std::vector({1, 3}), pool_), + NewIndexFile("full-text-1.index", "full-text", Range(100, 199), /*index_field_id=*/2, + /*extra_field_ids=*/std::vector({1, 3}), pool_)}; + ASSERT_OK_AND_ASSIGN(std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create( + /*snapshot_id=*/7, "content", /*row_range_start=*/100, + /*row_range_end=*/199, files, {Range(100, 119), Range(150, 199)})); + + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + const std::string version_1_index_files = FromHex( + "00000002" + "000000C8" + "0030000000000000090000004000000011000000500000008000000000000000" + "6400000000000000000000000000000000000000000000006000000068000000" + "66756C6C2D7465787400000000000000" + "66756C6C2D746578742D302E696E64657800000000000000" + "00200000000000006400000000000000C7000000000000000200000000000000" + "100000003800000017000000480000000000000000000000" + "02000000000000000100000003000000" + "7B22746F6B656E697A6572223A2264656661756C74227D00" + "000000C8" + "0030000000000000090000004000000011000000500000008000000000000000" + "6400000000000000000000000000000000000000000000006000000068000000" + "66756C6C2D7465787400000000000000" + "66756C6C2D746578742D312E696E64657800000000000000" + "00200000000000006400000000000000C7000000000000000200000000000000" + "100000003800000017000000480000000000000000000000" + "02000000000000000100000003000000" + "7B22746F6B656E697A6572223A2264656661756C74227D00"); + std::string version_1_bytes = "PFTSPLT1" + + FromHex( + "00000001" + "0000000000000007" + "0007") + + "content" + + FromHex( + "0000000000000064" + "00000000000000C7" + "00000002" + "0000000000000064" + "0000000000000077" + "0000000000000096" + "00000000000000C7") + + version_1_index_files; + ASSERT_EQ(bytes, version_1_bytes); + ASSERT_OK_AND_ASSIGN(std::shared_ptr result, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + auto index_split = std::dynamic_pointer_cast(result); + ASSERT_TRUE(index_split); + ASSERT_EQ(index_split->SnapshotId(), 7); + ASSERT_EQ(index_split->ColumnName(), "content"); + ASSERT_EQ(index_split->RowRangeStart(), 100); + ASSERT_EQ(index_split->RowRangeEnd(), 199); + ASSERT_EQ(index_split->SearchRowRanges(), + std::vector({Range(100, 119), Range(150, 199)})); + auto impl = std::dynamic_pointer_cast(result); + ASSERT_TRUE(impl); + ASSERT_EQ(*impl, *split) << impl->ToString() << " vs " << split->ToString(); + ASSERT_EQ(impl->FullTextIndexFiles().size(), 2u); + + ASSERT_OK_AND_ASSIGN(std::string bytes_again, FullTextSearchSplit::Serialize(result, pool_)); + ASSERT_EQ(bytes_again, bytes); + + std::string longest_column(std::numeric_limits::max(), 'c'); + ASSERT_OK_AND_ASSIGN(split, IndexFullTextSearchSplitImpl::Create( + /*snapshot_id=*/7, longest_column, /*row_range_start=*/100, + /*row_range_end=*/199, files, {Range(100, 199)})); + ASSERT_OK_AND_ASSIGN(bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_OK_AND_ASSIGN(result, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + index_split = std::dynamic_pointer_cast(result); + ASSERT_TRUE(index_split); + ASSERT_EQ(index_split->ColumnName(), longest_column); +} + +TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsInvalidBytes) { + std::vector> files = {NewIndexFile( + "full-text-0.index", "full-text", Range(0, 9), /*index_field_id=*/1, std::nullopt, pool_)}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, {Range(0, 9)})); + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + + std::string trailing = bytes + std::string(1, '\0'); + ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Deserialize(trailing.data(), trailing.size(), pool_), + "remaining 1 bytes after deserializing"); + for (size_t size = 0; size < bytes.size(); ++size) { + ASSERT_NOK(FullTextSearchSplit::Deserialize(bytes.data(), size, pool_)) << size; + } + ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Deserialize(nullptr, 0, pool_), "null buffer"); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(bytes.data(), std::numeric_limits::max(), pool_), + "exceed the supported maximum"); + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + out.WriteValue(12345); + std::string unknown_magic = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(unknown_magic.data(), unknown_magic.size(), pool_), + "unknown magic number"); + } + { + MemorySegmentOutputStream out(MemorySegmentOutputStream::DEFAULT_SEGMENT_SIZE, pool_); + out.WriteValue(IndexFullTextSearchSplitImpl::MAGIC); + out.WriteValue(IndexFullTextSearchSplitImpl::VERSION + 1); + std::string newer_version = ToBytes(&out, pool_); + ASSERT_NOK_WITH_MSG( + FullTextSearchSplit::Deserialize(newer_version.data(), newer_version.size(), pool_), + "unsupported IndexFullTextSearchSplit version: 2"); + } + ASSERT_NOK_WITH_MSG(FullTextSearchSplit::Serialize(nullptr, pool_), + "cannot cast to an index split"); +} + +TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptFileLayout) { + std::vector> files = { + NewIndexFile("full-text-0.index", "full-text", Range(0, 9), /*index_field_id=*/2, + /*extra_field_ids=*/std::vector({1}), pool_)}; + ASSERT_OK_AND_ASSIGN( + std::shared_ptr split, + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 0, 9, files, {Range(0, 9)})); + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_OK(FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + + IndexFileMetaSerializer file_serializer(pool_); + ASSERT_OK_AND_ASSIGN(BinaryRow file_row, file_serializer.ToRow(files[0])); + auto slot_of = [](size_t row_start, int32_t arity, size_t field) { + return row_start + BinaryRow::CalculateBitSetWidthInBytes(arity) + field * 8; + }; + auto offset_in_slot = [&bytes](size_t slot) { + int64_t offset_and_size = 0; + std::memcpy(&offset_and_size, bytes.data() + slot, sizeof(offset_and_size)); + return static_cast(offset_and_size >> 32); + }; + auto deserialize_with = [&](size_t position, const auto& value) { + std::string corrupt = bytes; + std::memcpy(corrupt.data() + position, &value, sizeof(value)); + return FullTextSearchSplit::Deserialize(corrupt.data(), corrupt.size(), pool_); + }; + const size_t index_type_field = 0; + const size_t global_index_field = 6; + const size_t extra_field_ids_field = 3; + const size_t file_start = bytes.size() - file_row.GetSizeInBytes(); + const size_t index_type_slot = slot_of(file_start, file_row.GetFieldCount(), index_type_field); + const size_t global_index_slot = + slot_of(file_start, file_row.GetFieldCount(), global_index_field); + + const int64_t outside_row = (static_cast(1000) << 32) | 9; + ASSERT_NOK_WITH_MSG(deserialize_with(index_type_slot, outside_row), + "string of 9 bytes at offset 1000 exceeds"); + const int64_t long_inline = + BinarySection::HIGHEST_FIRST_BIT | (static_cast(100) << 56); + ASSERT_NOK_WITH_MSG(deserialize_with(index_type_slot, long_inline), + "inline string of 100 bytes"); + ASSERT_NOK_WITH_MSG(deserialize_with(global_index_slot, outside_row), + "of 9 bytes at offset 1000 exceeds"); + + const size_t global_index_start = file_start + offset_in_slot(global_index_slot); + const size_t extra_field_ids_slot = + slot_of(global_index_start, GlobalIndexMeta::NUM_FIELDS, extra_field_ids_field); + const size_t extra_field_ids_offset = offset_in_slot(extra_field_ids_slot); + const int64_t short_array = (static_cast(extra_field_ids_offset) << 32) | 2; + ASSERT_NOK_WITH_MSG(deserialize_with(extra_field_ids_slot, short_array), + "binary array of 2 bytes"); + const size_t extra_field_ids_start = global_index_start + extra_field_ids_offset; + const int32_t element_count = 1000; + ASSERT_NOK_WITH_MSG(deserialize_with(extra_field_ids_start, element_count), + "binary array of 1000 elements"); +} + +TEST_F(FullTextSearchSplitTest, TestDeserializeRejectsCorruptEntries) { + auto deserialize = [this](const std::function& write_body) { + std::string bytes = SplitBytes(write_body, pool_); + return FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_); + }; + auto write_one_range = [](MemorySegmentOutputStream* out) { + const int32_t range_count = 1; + const int64_t from = 0; + const int64_t to = 9; + out->WriteValue(range_count); + out->WriteValue(from); + out->WriteValue(to); + }; + + auto negative_ranges = [](MemorySegmentOutputStream* out) { out->WriteValue(-1); }; + ASSERT_NOK_WITH_MSG(deserialize(negative_ranges), "-1 search row ranges"); + auto too_many_ranges = [](MemorySegmentOutputStream* out) { out->WriteValue(1000); }; + ASSERT_NOK_WITH_MSG(deserialize(too_many_ranges), + "1000 search row ranges exceed the remaining 0 bytes"); + auto reversed_range = [](MemorySegmentOutputStream* out) { + const int32_t range_count = 1; + const int64_t from = 9; + const int64_t to = 0; + out->WriteValue(range_count); + out->WriteValue(from); + out->WriteValue(to); + }; + ASSERT_NOK_WITH_MSG(deserialize(reversed_range), + "search row range [9, 0] ends before it starts"); + + auto negative_files = [&](MemorySegmentOutputStream* out) { + write_one_range(out); + out->WriteValue(-1); + }; + ASSERT_NOK_WITH_MSG(deserialize(negative_files), "-1 index files"); + auto too_many_files = [&](MemorySegmentOutputStream* out) { + write_one_range(out); + out->WriteValue(std::numeric_limits::max()); + }; + ASSERT_NOK_WITH_MSG(deserialize(too_many_files), + "2147483647 index files exceed the remaining 0 bytes"); + auto negative_file_length = [&](MemorySegmentOutputStream* out) { + write_one_range(out); + out->WriteValue(1); + out->WriteValue(-1); + }; + ASSERT_NOK_WITH_MSG(deserialize(negative_file_length), "invalid binary row length -1"); + auto empty_file = [&](MemorySegmentOutputStream* out) { + write_one_range(out); + out->WriteValue(1); + out->WriteValue(0); + }; + ASSERT_NOK_WITH_MSG(deserialize(empty_file), "binary row of 0 bytes without its"); +} + +TEST_F(FullTextSearchSplitTest, TestCreateValidatesArguments) { + std::vector> files = { + NewIndexFile("full-text-0.index", "full-text", Range(10, 19), /*index_field_id=*/1, + std::nullopt, pool_)}; + auto create = [&](const std::vector>& index_files, + const std::vector& search_row_ranges) { + return IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", 10, 19, index_files, + search_row_ranges); + }; + ASSERT_OK(create(files, {Range(10, 11), Range(13, 19)})); + ASSERT_NOK_WITH_MSG(create({}, {Range(10, 19)}), "at least one index file"); + ASSERT_NOK_WITH_MSG(create(files, {}), "at least one search row range"); + ASSERT_NOK_WITH_MSG(create(files, {Range(9, 19)}), "outside physical index range [10, 19]"); + ASSERT_NOK_WITH_MSG(create(files, {Range(10, 20)}), "outside physical index range [10, 19]"); + ASSERT_NOK_WITH_MSG(create(files, {Range(15, 19), Range(10, 12)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(create(files, {Range(10, 12), Range(13, 19)}), + "not sorted, disjoint and non-adjacent"); + ASSERT_NOK_WITH_MSG(create(files, {Range(10, 15), Range(15, 19)}), + "not sorted, disjoint and non-adjacent"); + + auto other_range = NewIndexFile("full-text-1.index", "full-text", Range(10, 29), + /*index_field_id=*/1, std::nullopt, pool_); + ASSERT_NOK_WITH_MSG(create({files[0], other_range}, {Range(10, 19)}), + "not the physical index range"); + auto other_type = NewIndexFile("lucene-0.index", "lucene-fts", Range(10, 19), + /*index_field_id=*/1, std::nullopt, pool_); + ASSERT_NOK_WITH_MSG(create({files[0], other_type}, {Range(10, 19)}), "different indexes"); + auto empty_extra_fields = NewIndexFile("full-text-4.index", "full-text", Range(10, 19), + /*index_field_id=*/1, std::vector(), pool_); + ASSERT_OK(create({files[0], empty_extra_fields}, {Range(10, 19)})); + auto other_fields = NewIndexFile("full-text-2.index", "full-text", Range(10, 19), + /*index_field_id=*/1, std::vector({2}), pool_); + ASSERT_NOK_WITH_MSG(create({files[0], other_fields}, {Range(10, 19)}), "different indexes"); + auto no_global_meta = std::make_shared( + "full-text", "full-text-3.index", /*file_size=*/128, /*row_count=*/10, + /*dv_ranges=*/std::nullopt, /*external_path=*/std::nullopt); + ASSERT_NOK_WITH_MSG(create({no_global_meta}, {Range(10, 19)}), "global index files"); + ASSERT_NOK_WITH_MSG( + IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "", 10, 19, files, {Range(10, 19)}), + "requires a column name"); + std::string too_long_column(std::numeric_limits::max() + 1, 'c'); + ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, too_long_column, 10, + 19, files, {Range(10, 19)}), + "column name of at most 65535 bytes, but got 65536 bytes"); + std::vector> invalid_ranges = { + {-1, 19}, {19, 10}, {0, std::numeric_limits::max()}}; + for (const auto& [start, end] : invalid_ranges) { + ASSERT_NOK_WITH_MSG(IndexFullTextSearchSplitImpl::Create(/*snapshot_id=*/1, "f0", start, + end, files, {Range(10, 19)}), + "invalid physical index range"); + } +} + +} // namespace paimon::test diff --git a/src/paimon/core/table/source/index_full_text_search_split_impl.h b/src/paimon/core/table/source/index_full_text_search_split_impl.h new file mode 100644 index 000000000..f2cd8cb62 --- /dev/null +++ b/src/paimon/core/table/source/index_full_text_search_split_impl.h @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include +#include +#include +#include + +#include "paimon/core/index/index_file_meta.h" +#include "paimon/result.h" +#include "paimon/table/source/full_text_search_split.h" +#include "paimon/utils/range.h" + +namespace paimon { +/// The implementation of `IndexFullTextSearchSplit`, which also holds the full-text index files of +/// the shard. +class IndexFullTextSearchSplitImpl : public IndexFullTextSearchSplit { + public: + /// "PFTSPLT1" in ASCII. + static constexpr int64_t MAGIC = 0x50465453504C5431L; + /// TODO: once row filters (#405) are supported, add the scalar index files that pre-filter rows + /// in a new version, as version 2 of Java `IndexFullTextSearchSplit` does. + static constexpr int32_t VERSION = 1; + + /// Creates a split. + /// @param snapshot_id The snapshot the split is planned from. + /// @param column_name The searched table column; not empty and at most 65535 bytes. + /// @param row_range_start The first global row id of the index shard; not negative. + /// @param row_range_end The last global row id of the index shard, inclusive; the row + /// count of the shard must fit in int64. + /// @param full_text_index_files The files of the shard: at least one, all of one index + /// (`DataEvolutionFullTextScan::IndexIdentity()`), and each with a + /// global index meta of the shard range. + /// @param search_row_ranges The sorted, disjoint and non-adjacent row id ranges to rank, at + /// least one and all within the shard range. + /// @return The split, or `Invalid` if an argument breaks these rules. + static Result> Create( + int64_t snapshot_id, const std::string& column_name, int64_t row_range_start, + int64_t row_range_end, std::vector> full_text_index_files, + std::vector search_row_ranges); + + int64_t SnapshotId() const override { + return snapshot_id_; + } + + const std::string& ColumnName() const override { + return column_name_; + } + + int64_t RowRangeStart() const override { + return row_range_start_; + } + + int64_t RowRangeEnd() const override { + return row_range_end_; + } + + const std::vector& SearchRowRanges() const override { + return search_row_ranges_; + } + + const std::vector>& FullTextIndexFiles() const { + return full_text_index_files_; + } + + bool operator==(const IndexFullTextSearchSplitImpl& other) const; + + std::string ToString() const; + + private: + IndexFullTextSearchSplitImpl(int64_t snapshot_id, const std::string& column_name, + int64_t row_range_start, int64_t row_range_end, + std::vector> full_text_index_files, + std::vector search_row_ranges) + : snapshot_id_(snapshot_id), + column_name_(column_name), + row_range_start_(row_range_start), + row_range_end_(row_range_end), + full_text_index_files_(std::move(full_text_index_files)), + search_row_ranges_(std::move(search_row_ranges)) {} + + int64_t snapshot_id_; + std::string column_name_; + int64_t row_range_start_; + int64_t row_range_end_; + std::vector> full_text_index_files_; + std::vector search_row_ranges_; +}; + +} // namespace paimon diff --git a/src/paimon/global_index/full_text/full_text_global_index.h b/src/paimon/global_index/full_text/full_text_global_index.h index 08a280e87..52e88d711 100644 --- a/src/paimon/global_index/full_text/full_text_global_index.h +++ b/src/paimon/global_index/full_text/full_text_global_index.h @@ -50,6 +50,10 @@ class FullTextGlobalIndex : public GlobalIndexer { const std::optional>& row_ranges, const std::shared_ptr& pool) const override; + bool SupportsFullTextSearch() const override { + return true; + } + private: std::map options_; }; diff --git a/src/paimon/global_index/full_text/full_text_global_index_test.cpp b/src/paimon/global_index/full_text/full_text_global_index_test.cpp index 15da34ef1..d361e62bd 100644 --- a/src/paimon/global_index/full_text/full_text_global_index_test.cpp +++ b/src/paimon/global_index/full_text/full_text_global_index_test.cpp @@ -368,6 +368,7 @@ TEST_F(FullTextGlobalIndexTest, TestWriteAndSearch) { ASSERT_OK_AND_ASSIGN(std::unique_ptr indexer, GlobalIndexerFactory::Get("full-text", options)); ASSERT_TRUE(dynamic_cast(indexer.get())); + ASSERT_TRUE(indexer->SupportsFullTextSearch()); ASSERT_OK_AND_ASSIGN(std::shared_ptr writer, indexer->CreateWriter("f0", CreateArrowSchema(data_type_).get(), CreateFileManager(), pool_)); diff --git a/src/paimon/global_index/lucene/lucene_global_index.h b/src/paimon/global_index/lucene/lucene_global_index.h index 254eec326..4495a71b8 100644 --- a/src/paimon/global_index/lucene/lucene_global_index.h +++ b/src/paimon/global_index/lucene/lucene_global_index.h @@ -47,6 +47,10 @@ class LuceneGlobalIndex : public GlobalIndexer { const std::optional>& row_ranges, const std::shared_ptr& pool) const override; + bool SupportsFullTextSearch() const override { + return true; + } + private: std::map options_; }; diff --git a/src/paimon/global_index/lucene/lucene_global_index_test.cpp b/src/paimon/global_index/lucene/lucene_global_index_test.cpp index 2be7aa02d..905c345c7 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_test.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_test.cpp @@ -164,6 +164,7 @@ TEST_P(LuceneGlobalIndexTest, TestSimple) { {"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.read.buffer-size", std::to_string(read_buffer_size)}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}; + ASSERT_TRUE(LuceneGlobalIndex(options).SupportsFullTextSearch()); std::shared_ptr array = arrow::ipc::internal::json::ArrayFromJSON(data_type_, R"([ ["This is an test document."], diff --git a/test/inte/global_index_test.cpp b/test/inte/global_index_test.cpp index a978b2999..19c606ef0 100644 --- a/test/inte/global_index_test.cpp +++ b/test/inte/global_index_test.cpp @@ -21,6 +21,8 @@ #include #include "arrow/type.h" +#include "fmt/format.h" +#include "fmt/ranges.h" #include "gtest/gtest.h" #include "paimon/common/factories/io_hook.h" #include "paimon/common/global_index/bitmap/bitmap_global_index_factory.h" @@ -28,13 +30,17 @@ #include "paimon/common/global_index/sorted_index_file_meta.h" #include "paimon/common/global_index/union_global_index_reader.h" #include "paimon/common/table/special_fields.h" +#include "paimon/common/utils/path_util.h" #include "paimon/common/utils/scope_guard.h" #include "paimon/core/global_index/global_index_scan_impl.h" #include "paimon/core/global_index/indexed_split_impl.h" #include "paimon/core/table/source/data_split_impl.h" +#include "paimon/core/utils/snapshot_manager.h" +#include "paimon/core/utils/tag_manager.h" #include "paimon/defs.h" #include "paimon/executor.h" #include "paimon/fs/file_system.h" +#include "paimon/fs/local/local_file_system.h" #include "paimon/global_index/bitmap_global_index_result.h" #include "paimon/global_index/bitmap_scored_global_index_result.h" #include "paimon/global_index/global_index_reader.h" @@ -45,6 +51,8 @@ #include "paimon/predicate/predicate_builder.h" #include "paimon/result.h" #include "paimon/status.h" +#include "paimon/table/source/full_text_search_builder.h" +#include "paimon/table/source/full_text_search_split.h" #include "paimon/testing/utils/binary_row_generator.h" #include "paimon/testing/utils/counting_cache_test_utils.h" #include "paimon/testing/utils/deletion_vector_test_helper.h" @@ -74,9 +82,31 @@ class CountingGlobalIndexFileManager : public GlobalIndexFileManager { mutable std::atomic open_count_ = 0; }; +class DeleteAfterListFileSystem final : public LocalFileSystem { + public: + explicit DeleteAfterListFileSystem(std::string deleted_path) + : deleted_path_(std::move(deleted_path)), + listed_directory_name_(PathUtil::GetName(PathUtil::GetParentDirPath(deleted_path_))) {} + + Status ListDir(const std::string& directory, + std::vector* file_status_list) const override { + PAIMON_RETURN_NOT_OK(LocalFileSystem::ListDir(directory, file_status_list)); + if (PathUtil::GetName(directory) == listed_directory_name_ && !deleted_.exchange(true)) { + return LocalFileSystem::Delete(deleted_path_); + } + return Status::OK(); + } + + private: + std::string deleted_path_; + std::string listed_directory_name_; + mutable std::atomic deleted_ = false; +}; + } // namespace using ScoredRows = std::vector>; +using PlannedShards = std::vector>>; // string: FileFormat, bool: UseSpecificFileSystem using ParamType = std::tuple; /// This is a sdk end-to-end test for global index. @@ -234,6 +264,76 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter } } + std::unique_ptr NewFullTextSearchBuilder( + const std::string& table_path, const std::string& query, int32_t limit, + const std::map& options = {}) const { + auto builder = std::make_unique(table_path); + builder->SetQuery("f0", query).SetLimit(limit).SetOptions(options).WithFileSystem(fs_); + return builder; + } + + static Result ToScoredRows(const std::shared_ptr& result) { + PAIMON_ASSIGN_OR_RAISE(std::unique_ptr iter, + result->CreateScoredIterator()); + ScoredRows rows; + while (iter->HasNext()) { + rows.push_back(iter->NextWithScore()); + } + return rows; + } + + Result SearchTable( + const std::string& table_path, const std::string& query, int32_t limit, + const std::vector>& partition_filters = {}, + const std::map& options = {}) const { + std::unique_ptr builder = + NewFullTextSearchBuilder(table_path, query, limit, options); + builder->SetPartitionFilter(partition_filters); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr result, + builder->ExecuteLocal()); + return ToScoredRows(result); + } + + Status TagSnapshot(const std::string& table_path, int64_t snapshot_id, + const std::string& tag_name) const { + std::shared_ptr fs = dir_->GetFileSystem(); + std::string snapshot_json; + PAIMON_RETURN_NOT_OK(fs->ReadFile(SnapshotManager(fs, table_path).SnapshotPath(snapshot_id), + &snapshot_json)); + return fs->WriteFile(TagManager(fs, table_path).TagPath(tag_name), snapshot_json, + /*overwrite=*/false); + } + + Result PlanShards(const std::string& table_path, + const std::string& query) const { + PAIMON_ASSIGN_OR_RAISE( + std::unique_ptr scan, + NewFullTextSearchBuilder(table_path, query, /*limit=*/1)->NewFullTextScan()); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr plan, scan->CreatePlan()); + PlannedShards shards; + for (const auto& split : plan->Splits()) { + auto index_split = std::dynamic_pointer_cast(split); + if (!index_split || index_split->ColumnName() != "f0" || + index_split->SnapshotId() != plan->SnapshotId()) { + return Status::Invalid("unexpected full-text search split"); + } + shards.emplace_back(Range(index_split->RowRangeStart(), index_split->RowRangeEnd()), + index_split->SearchRowRanges()); + } + return shards; + } + + void CheckTableSearchMatchesReader(const std::string& table_path, const std::string& query, + const std::shared_ptr& reader, + int32_t max_limit) const { + for (int32_t limit = 1; limit <= max_limit; ++limit) { + ASSERT_OK_AND_ASSIGN(ScoredRows expected_rows, + SearchFullText(reader, query, limit, std::nullopt)); + ASSERT_OK_AND_ASSIGN(ScoredRows rows, SearchTable(table_path, query, limit)); + ASSERT_EQ(rows, expected_rows) << "limit=" << limit; + } + } + void CheckSearchOverlappingIndexRanges(const std::string& index_type, const std::map& index_options, const std::string& query) { @@ -278,6 +378,11 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_EQ(top_rows[0].first, 3); CheckFullTextTopK(readers[0], query, RoaringBitmap64::From({1, 2, 3}), {1, 2, 3}); + ASSERT_OK_AND_ASSIGN(PlannedShards shards, PlanShards(table_path, query)); + ASSERT_EQ(shards, + PlannedShards({{Range(0, 2), {Range(0, 2)}}, {Range(1, 3), {Range(3, 3)}}})); + CheckTableSearchMatchesReader(table_path, query, readers[0], /*max_limit=*/5); + ASSERT_OK_AND_ASSIGN(RowRangeIndex row_range_index, RowRangeIndex::Create({Range(3, 3)})); ASSERT_OK_AND_ASSIGN(readers, global_index_scan->CreateReaders("f0", row_range_index)); ASSERT_EQ(readers.size(), 1u); @@ -330,6 +435,12 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter /*executor=*/nullptr, pool_); }; ASSERT_OK_AND_ASSIGN(std::shared_ptr scan_before_deletion, create_scan()); + std::string query = R"({"match":{"query":"apple"}})"; + ASSERT_OK_AND_ASSIGN( + std::unique_ptr full_text_scan, + NewFullTextSearchBuilder(table_path, query, /*limit=*/100)->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan_before_deletion, + full_text_scan->CreatePlan()); ASSERT_OK_AND_ASSIGN(auto split, ScanData(table_path, /*partition_filters=*/{})); ASSERT_OK_AND_ASSIGN(std::vector anchor_file_names, @@ -344,7 +455,6 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK(Commit(table_path, {dv_commit_msg})); ASSERT_OK_AND_ASSIGN(std::shared_ptr scan, create_scan()); - std::string query = R"({"match":{"query":"apple"}})"; ASSERT_OK_AND_ASSIGN( auto readers_before_deletion, scan_before_deletion->CreateReaders("f0", /*row_range_index=*/std::nullopt)); @@ -372,6 +482,49 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter CheckFullTextTopK(readers[0], query, RoaringBitmap64::From({0, 5, 8}), {8}); CheckFullTextTopK(readers[0], query, RoaringBitmap64::From({0, 1}), {}); + CheckTableSearchMatchesReader(table_path, query, readers[0], /*max_limit=*/6); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr full_text_read, + NewFullTextSearchBuilder(table_path, query, /*limit=*/100)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr pinned_result, + full_text_read->Read(*plan_before_deletion)); + ASSERT_OK_AND_ASSIGN(ScoredRows pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + + std::shared_ptr file_system = dir_->GetFileSystem(); + int64_t snapshot_before_deletion = plan_before_deletion->SnapshotId().value(); + std::string snapshot_path = + SnapshotManager(file_system, table_path).SnapshotPath(snapshot_before_deletion); + std::string tag_path = TagManager(file_system, table_path).TagPath("before-deletion"); + ASSERT_OK(TagSnapshot(table_path, snapshot_before_deletion, "before-deletion")); + ASSERT_OK(file_system->Delete(snapshot_path)); + ASSERT_OK_AND_ASSIGN(pinned_result, full_text_read->Read(*plan_before_deletion)); + ASSERT_OK_AND_ASSIGN(pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + std::string unrelated_tag_path = TagManager(file_system, table_path).TagPath("a-unrelated"); + ASSERT_OK(file_system->WriteFile(unrelated_tag_path, "", /*overwrite=*/false)); + std::unique_ptr racing_builder = + NewFullTextSearchBuilder(table_path, query, /*limit=*/100); + racing_builder->WithFileSystem( + std::make_shared(unrelated_tag_path)); + ASSERT_OK_AND_ASSIGN(std::unique_ptr racing_read, + racing_builder->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(pinned_result, racing_read->Read(*plan_before_deletion)); + ASSERT_OK_AND_ASSIGN(pinned_rows, ToScoredRows(pinned_result)); + ASSERT_EQ(pinned_rows, rows_before_deletion); + ASSERT_OK_AND_ASSIGN(bool unrelated_tag_exists, file_system->Exists(unrelated_tag_path)); + ASSERT_FALSE(unrelated_tag_exists); + ASSERT_OK(file_system->Delete(tag_path)); + ASSERT_NOK_WITH_MSG(full_text_read->Read(*plan_before_deletion), + "planned from no longer exists"); + ASSERT_OK(file_system->WriteFile(snapshot_path, "not a snapshot", /*overwrite=*/false)); + Result> unreadable_snapshot_result = + full_text_read->Read(*plan_before_deletion); + ASSERT_FALSE(unreadable_snapshot_result.ok()); + ASSERT_EQ(unreadable_snapshot_result.status().ToString().find("no longer exists"), + std::string::npos); + ASSERT_OK(file_system->Delete(snapshot_path)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr index_result, readers[0]->VisitFullTextSearch( std::make_shared("f0", query, /*limit=*/5))); @@ -391,6 +544,183 @@ class GlobalIndexTest : public ::testing::Test, public ::testing::WithParamInter ASSERT_OK(ReadData(table_path, {"f1"}, expected_array, /*predicate=*/nullptr, plan)); } + static std::shared_ptr AppleRows(const arrow::FieldVector& fields, int32_t from, + int32_t to) { + std::vector rows; + for (int32_t row = from; row < to; ++row) { + std::vector terms(static_cast(row + 1), "apple"); + rows.push_back(fmt::format(R"(["{}", {}])", fmt::join(terms, " "), row)); + } + return arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_(fields), fmt::format("[{}]", fmt::join(rows, ",\n"))) + .ValueOrDie(); + } + + void CheckTableFullTextSearch(const std::string& index_type, + const std::map& index_options) { + arrow::FieldVector fields = {arrow::field("f0", arrow::utf8()), + arrow::field("f1", arrow::int32())}; + auto schema = arrow::schema(fields); + std::map options = {{Options::FILE_FORMAT, file_format_}, + {Options::FILE_SYSTEM, "local"}, + {Options::ROW_TRACKING_ENABLED, "true"}, + {Options::DATA_EVOLUTION_ENABLED, "true"}}; + CreateTable(/*partition_keys=*/{}, schema, options); + std::string table_path = PathUtil::JoinPath(dir_->Str(), "foo.db/bar"); + ASSERT_OK_AND_ASSIGN(auto commit_msgs1, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 0, 5))); + ASSERT_OK(Commit(table_path, commit_msgs1)); + ASSERT_OK_AND_ASSIGN(auto commit_msgs2, WriteArray(table_path, schema->field_names(), + AppleRows(fields, 5, 10))); + ASSERT_OK(Commit(table_path, commit_msgs2)); + std::string query = R"({"match":{"query":"apple"}})"; + + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, + Range(0, 4))); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr first_shard_scan, + NewFullTextSearchBuilder(table_path, query, /*limit=*/10)->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_shard_plan, + first_shard_scan->CreatePlan()); + ASSERT_TRUE(first_shard_plan->SnapshotId()); + ASSERT_EQ(first_shard_plan->Splits().size(), 1u); + ASSERT_OK_AND_ASSIGN(ScoredRows indexed_rows, + SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, "fast"}, + {Options::GLOBAL_INDEX_SEARCH_MODE, "full"}})); + ASSERT_EQ(indexed_rows.size(), 5u); + for (const auto& [row_id, score] : indexed_rows) { + ASSERT_LT(row_id, 5) << score; + } + for (const char* mode : {"full", "detail"}) { + ASSERT_NOK_WITH_MSG( + SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, + {{Options::FULL_TEXT_INDEX_SEARCH_MODE, mode}}), + "full and detail are not supported yet"); + } + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", index_type, index_options, + Range(5, 9))); + + ASSERT_OK_AND_ASSIGN(PlannedShards shards, PlanShards(table_path, query)); + ASSERT_EQ(shards, + PlannedShards({{Range(0, 4), {Range(0, 4)}}, {Range(5, 9), {Range(5, 9)}}})); + ASSERT_OK_AND_ASSIGN(std::shared_ptr global_index_scan, + GlobalIndexScan::Create(table_path, /*snapshot_id=*/std::nullopt, + /*partitions=*/std::nullopt, /*options=*/{}, + fs_, /*executor=*/nullptr, pool_)); + ASSERT_OK_AND_ASSIGN( + auto readers, global_index_scan->CreateReaders("f0", /*row_range_index=*/std::nullopt)); + ASSERT_EQ(readers.size(), 1u); + CheckTableSearchMatchesReader(table_path, query, readers[0], /*max_limit=*/11); + + ASSERT_OK_AND_ASSIGN( + std::unique_ptr scan, + NewFullTextSearchBuilder(table_path, query, /*limit=*/3)->NewFullTextScan()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr plan, scan->CreatePlan()); + std::vector> received_splits; + for (const auto& split : plan->Splits()) { + ASSERT_OK_AND_ASSIGN(std::string bytes, FullTextSearchSplit::Serialize(split, pool_)); + ASSERT_OK_AND_ASSIGN( + std::shared_ptr received_split, + FullTextSearchSplit::Deserialize(bytes.data(), bytes.size(), pool_)); + received_splits.push_back(std::move(received_split)); + } + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read, + NewFullTextSearchBuilder(table_path, query, /*limit=*/3)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr top_result, + read->Read(received_splits)); + ASSERT_OK_AND_ASSIGN(ScoredRows top_rows, ToScoredRows(top_result)); + ASSERT_EQ(top_rows.size(), 3u); + ASSERT_OK_AND_ASSIGN(ScoredRows local_top_rows, + SearchTable(table_path, query, /*limit=*/3)); + ASSERT_EQ(top_rows, local_top_rows); + + std::string first_shard_snapshot_id = + std::to_string(first_shard_plan->SnapshotId().value()); + ASSERT_OK_AND_ASSIGN(ScoredRows first_shard_rows, + SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, + {{Options::SCAN_SNAPSHOT_ID, first_shard_snapshot_id}})); + ASSERT_EQ(first_shard_rows.size(), 5u); + for (const auto& [row_id, score] : first_shard_rows) { + ASSERT_LT(row_id, 5) << score; + } + ASSERT_OK_AND_ASSIGN(ScoredRows snapshot_mode_rows, + SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, + {{Options::SCAN_MODE, "from-snapshot"}, + {Options::SCAN_SNAPSHOT_ID, first_shard_snapshot_id}})); + ASSERT_EQ(snapshot_mode_rows, first_shard_rows); + ASSERT_NOK_WITH_MSG(SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, + {{Options::SCAN_MODE, "latest-full"}, + {Options::SCAN_SNAPSHOT_ID, first_shard_snapshot_id}}), + "scan.snapshot-id must be null when you use latest-full for scan.mode"); + ASSERT_OK(TagSnapshot(table_path, first_shard_plan->SnapshotId().value(), "first-shard")); + ASSERT_OK_AND_ASSIGN(ScoredRows tag_rows, + SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, + {{Options::SCAN_TAG_NAME, "first-shard"}})); + ASSERT_EQ(tag_rows, first_shard_rows); + ASSERT_OK_AND_ASSIGN(ScoredRows latest_rows, SearchTable(table_path, query, /*limit=*/10)); + ASSERT_EQ(latest_rows.size(), 10u); + ASSERT_OK_AND_ASSIGN(ScoredRows timestamp_rows, + SearchTable(table_path, query, /*limit=*/10, /*partition_filters=*/{}, + {{Options::SCAN_TIMESTAMP_MILLIS, + std::to_string(std::numeric_limits::max())}})); + ASSERT_EQ(timestamp_rows, latest_rows); + ASSERT_OK_AND_ASSIGN( + std::unique_ptr read_ten, + NewFullTextSearchBuilder(table_path, query, /*limit=*/10)->NewFullTextRead()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr first_shard_result, + read_ten->Read(*first_shard_plan)); + ASSERT_OK_AND_ASSIGN(ScoredRows first_shard_plan_rows, ToScoredRows(first_shard_result)); + ASSERT_EQ(first_shard_plan_rows, first_shard_rows); + + std::vector expected_rows; + for (const auto& [row_id, score] : top_rows) { + expected_rows.push_back(fmt::format("[0, {}, {}]", row_id, score)); + } + auto expected_array = + arrow::ipc::internal::json::ArrayFromJSON( + arrow::struct_({SpecialFields::ValueKind().ArrowField(), fields[1], + SpecialFields::IndexScore().ArrowField()}), + fmt::format("[{}]", fmt::join(expected_rows, ",\n"))) + .ValueOrDie(); + ASSERT_OK_AND_ASSIGN(auto data_plan, + ScanGlobalIndexAndData(table_path, /*predicate=*/nullptr, + /*options=*/{}, top_result)); + ASSERT_OK(ReadData(table_path, {"f1", "_INDEX_SCORE"}, expected_array, + /*predicate=*/nullptr, data_plan)); + + ASSERT_OK(WriteIndex(table_path, /*partition_filters=*/{}, "f0", "bitmap", /*options=*/{}, + Range(0, 9))); + ASSERT_OK_AND_ASSIGN(shards, PlanShards(table_path, query)); + ASSERT_EQ(shards, + PlannedShards({{Range(0, 4), {Range(0, 4)}}, {Range(5, 9), {Range(5, 9)}}})); + ASSERT_OK_AND_ASSIGN(ScoredRows rows_with_bitmap, + SearchTable(table_path, query, /*limit=*/3)); + ASSERT_EQ(rows_with_bitmap, top_rows); + } + + void CheckTableSearchWithPartitionFilter(const std::string& table_path, + const std::string& query) const { + auto row_ids_of = [](const ScoredRows& rows) { + std::vector row_ids; + for (const auto& row : rows) { + row_ids.push_back(row.first); + } + return row_ids; + }; + ASSERT_OK_AND_ASSIGN(ScoredRows rows, + SearchTable(table_path, query, /*limit=*/10, {{{"f1", "10"}}})); + ASSERT_EQ(row_ids_of(rows), std::vector({0, 1})); + ASSERT_OK_AND_ASSIGN(rows, SearchTable(table_path, query, /*limit=*/10, {{{"f1", "20"}}})); + ASSERT_EQ(row_ids_of(rows), std::vector({2})); + ASSERT_OK_AND_ASSIGN( + rows, SearchTable(table_path, query, /*limit=*/10, {{{"f1", "10"}}, {{"f1", "20"}}})); + ASSERT_EQ(row_ids_of(rows), std::vector({0, 1, 2})); + ASSERT_OK_AND_ASSIGN(rows, SearchTable(table_path, query, /*limit=*/10)); + ASSERT_EQ(row_ids_of(rows), std::vector({0, 1, 2})); + } + Result> ScanGlobalIndexAndData( const std::string& table_path, const std::shared_ptr& predicate, const std::map& options = {}, @@ -3042,6 +3372,8 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadLuceneIndexWithPartition) { CheckFullTextTopK(readers[0], query, std::nullopt, {0, 1, 2}); CheckFullTextTopK(readers[0], query, RoaringBitmap64::From({1, 2, 3}), {1, 2}); } + CheckTableSearchWithPartitionFilter(table_path, + R"({"match":{"query":"document","operator":"And"}})"); } TEST_P(GlobalIndexTest, TestLuceneSearchOverlappingIndexRanges) { @@ -3060,6 +3392,14 @@ TEST_P(GlobalIndexTest, TestLuceneSearchSkipsRowsDeletedByDeletionVectors) { {{"lucene-fts.write.omit-term-freq-and-position", "false"}, {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); } + +TEST_P(GlobalIndexTest, TestLuceneTableFullTextSearch) { + auto tmp_dir = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(tmp_dir); + CheckTableFullTextSearch("lucene-fts", + {{"lucene-fts.write.omit-term-freq-and-position", "false"}, + {"lucene-fts.write.tmp.directory", tmp_dir->Str()}}); +} #endif #ifdef PAIMON_ENABLE_FULL_TEXT @@ -3201,6 +3541,7 @@ TEST_P(GlobalIndexTest, TestWriteCommitScanReadFullTextIndexWithPartition) { CheckFullTextTopK(readers[0], query, std::nullopt, {0, 1, 2}); CheckFullTextTopK(readers[0], query, RoaringBitmap64::From({1, 2, 3}), {1, 2}); } + CheckTableSearchWithPartitionFilter(table_path, R"({"match":{"query":"document"}})"); } TEST_P(GlobalIndexTest, TestFullTextSearchOverlappingIndexRanges) { @@ -3211,6 +3552,10 @@ TEST_P(GlobalIndexTest, TestFullTextSearchOverlappingIndexRanges) { TEST_P(GlobalIndexTest, TestFullTextSearchSkipsRowsDeletedByDeletionVectors) { CheckSearchSkipsDeletedRows("full-text", {{"full-text.tokenizer", "default"}}); } + +TEST_P(GlobalIndexTest, TestFullTextTableFullTextSearch) { + CheckTableFullTextSearch("full-text", {{"full-text.tokenizer", "default"}}); +} #endif TEST_P(GlobalIndexTest, TestBTreeWriteCommitScanReadIndex) {