From 5ce292182eaf0a7b6fc605e65681d7538751f654 Mon Sep 17 00:00:00 2001 From: Nicholas Jiang Date: Sun, 1 Nov 2026 20:08:13 -0800 Subject: [PATCH] perf(lucene): load Jieba dictionaries once per process Every Lucene global index reader and writer constructed its own cppjieba::Jieba, reloading about 11.6MB of dictionaries each time. A table-level full-text search creates one reader per shard, so every query paid this cost once per shard. After the table-level full-text search tests were added in #461, paimon-global-index-test on tsan-x86_64 grew from about 1400s to about 3500s and pushed the job past its 120-minute timeout. Add LuceneUtils::GetJieba(), which loads each dictionary directory once per process and shares a const segmenter with all readers and writers. Segmentation only calls const cppjieba methods, which keep no mutable state and are safe to call concurrently. The cache is keyed by the canonical absolute path, so a relative PAIMON_JIEBA_DICT_DIR resolves against the current working directory instead of reusing dictionaries loaded from another one, and a directory that cannot be resolved returns an IOError instead of aborting in cppjieba. Document how a relative directory is resolved and that dictionary changes take effect only after the process restarts. On tsan-x86_64, this cuts paimon-global-index-test from about 3500s to about 160s, and the job finishes in about an hour instead of timing out. --- docs/source/user_guide/global_index.rst | 2 +- src/paimon/global_index/lucene/CMakeLists.txt | 1 + .../global_index/lucene/jieba_analyzer.cpp | 2 +- .../global_index/lucene/jieba_analyzer.h | 4 +- .../lucene/lucene_global_index_reader.cpp | 7 +- .../lucene/lucene_global_index_reader.h | 4 +- .../lucene/lucene_global_index_writer.cpp | 7 +- .../global_index/lucene/lucene_utils.cpp | 30 ++++ src/paimon/global_index/lucene/lucene_utils.h | 12 ++ .../global_index/lucene/lucene_utils_test.cpp | 132 ++++++++++++++++++ 10 files changed, 185 insertions(+), 16 deletions(-) create mode 100644 src/paimon/global_index/lucene/lucene_utils_test.cpp diff --git a/docs/source/user_guide/global_index.rst b/docs/source/user_guide/global_index.rst index b9168c5e0..5901e5c71 100644 --- a/docs/source/user_guide/global_index.rst +++ b/docs/source/user_guide/global_index.rst @@ -144,7 +144,7 @@ a column name containing only such whitespace selects the indexed field. - **Environment Variable**: ``PAIMON_JIEBA_DICT_DIR`` - - **Description**: Specifies the directory containing Jieba dictionary files for Chinese text tokenization. At runtime, the system first checks this environment variable; if not set, it falls back to the compile-time ``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is available, will fail with an error. + - **Description**: Specifies the directory containing Jieba dictionary files for Chinese text tokenization. At runtime, the system first checks this environment variable; if not set, it falls back to the compile-time ``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is available, will fail with an error. A relative directory is resolved against the current working directory. Each dictionary directory is loaded once per process and shared by all Lucene index readers and writers, so changes to its dictionary files take effect only after the process restarts. Full-Text Index (Experimental) ------------------------------ diff --git a/src/paimon/global_index/lucene/CMakeLists.txt b/src/paimon/global_index/lucene/CMakeLists.txt index 43d85b3ad..36548daea 100644 --- a/src/paimon/global_index/lucene/CMakeLists.txt +++ b/src/paimon/global_index/lucene/CMakeLists.txt @@ -58,6 +58,7 @@ if(PAIMON_ENABLE_LUCENE) lucene_directory_test.cpp lucene_global_index_test.cpp lucene_filter_test.cpp + lucene_utils_test.cpp EXTRA_INCLUDES ${LUCENE_INCLUDE_DIR} STATIC_LINK_LIBS diff --git a/src/paimon/global_index/lucene/jieba_analyzer.cpp b/src/paimon/global_index/lucene/jieba_analyzer.cpp index e0a71a81a..ca0d10dd6 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.cpp +++ b/src/paimon/global_index/lucene/jieba_analyzer.cpp @@ -24,7 +24,7 @@ namespace paimon::lucene { JiebaTokenizerContext::JiebaTokenizerContext(const std::string& _tokenize_mode, bool _with_position, - const std::shared_ptr& _jieba, + const std::shared_ptr& _jieba, const std::shared_ptr& _pool, int32_t _buffer_size) : pool(_pool), diff --git a/src/paimon/global_index/lucene/jieba_analyzer.h b/src/paimon/global_index/lucene/jieba_analyzer.h index 2af534ac7..896ed44e3 100644 --- a/src/paimon/global_index/lucene/jieba_analyzer.h +++ b/src/paimon/global_index/lucene/jieba_analyzer.h @@ -26,7 +26,7 @@ namespace paimon::lucene { struct JiebaTokenizerContext { JiebaTokenizerContext(const std::string& _tokenize_mode, bool _with_position, - const std::shared_ptr& _jieba, + const std::shared_ptr& _jieba, const std::shared_ptr& _pool, int32_t _buffer_size = kReadBufferSize); @@ -34,7 +34,7 @@ struct JiebaTokenizerContext { std::string tokenize_mode; bool with_position; int32_t buffer_size; - std::shared_ptr jieba; + std::shared_ptr jieba; static inline const int32_t kReadBufferSize = 5 * 1024 * 1024; static inline const int32_t kMaxWordLen = 1024; diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp index c8f28b9ca..ec5958892 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.cpp @@ -530,11 +530,8 @@ Result> LuceneGlobalIndexReader::Create Lucene::IndexReaderPtr reader = Lucene::IndexReader::open(lucene_dir, /*read_only=*/true); Lucene::IndexSearcherPtr searcher = Lucene::newLucene(reader); - PAIMON_ASSIGN_OR_RAISE(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir()); - auto jieba = std::make_shared( - dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", - dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", - dictionary_dir + "/stop_words.utf8"); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr jieba, + LuceneUtils::GetJieba()); // priority: read options > write options > kDefaultJiebaTokenizeMode PAIMON_ASSIGN_OR_RAISE( diff --git a/src/paimon/global_index/lucene/lucene_global_index_reader.h b/src/paimon/global_index/lucene/lucene_global_index_reader.h index 59a378e08..88768668c 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_reader.h +++ b/src/paimon/global_index/lucene/lucene_global_index_reader.h @@ -121,7 +121,7 @@ class LuceneGlobalIndexReader : public GlobalIndexReader { LuceneGlobalIndexReader(const std::wstring& wfield_name, const Lucene::IndexSearcherPtr& searcher, const std::string& tokenize_mode, - const std::shared_ptr& jieba) + const std::shared_ptr& jieba) : wfield_name_(wfield_name), searcher_(searcher), tokenize_mode_(tokenize_mode), @@ -142,6 +142,6 @@ class LuceneGlobalIndexReader : public GlobalIndexReader { std::wstring wfield_name_; Lucene::IndexSearcherPtr searcher_; std::string tokenize_mode_; - std::shared_ptr jieba_; + std::shared_ptr jieba_; }; } // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp index 133ddf23d..7477302c1 100644 --- a/src/paimon/global_index/lucene/lucene_global_index_writer.cpp +++ b/src/paimon/global_index/lucene/lucene_global_index_writer.cpp @@ -76,11 +76,8 @@ Result> LuceneGlobalIndexWriter::Create std::string tokenize_mode, OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode, std::string(kDefaultJiebaTokenizeMode))); - PAIMON_ASSIGN_OR_RAISE(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir()); - auto jieba = std::make_shared( - dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", - dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", - dictionary_dir + "/stop_words.utf8"); + PAIMON_ASSIGN_OR_RAISE(std::shared_ptr jieba, + LuceneUtils::GetJieba()); JiebaTokenizerContext jieba_context(tokenize_mode, /*with_position=*/!omit_term_freq_and_positions, jieba, pool); diff --git a/src/paimon/global_index/lucene/lucene_utils.cpp b/src/paimon/global_index/lucene/lucene_utils.cpp index 960a39716..53f3f5f65 100644 --- a/src/paimon/global_index/lucene/lucene_utils.cpp +++ b/src/paimon/global_index/lucene/lucene_utils.cpp @@ -16,6 +16,13 @@ * limitations under the License. */ #include "paimon/global_index/lucene/lucene_utils.h" + +#include +#include +#include +#include + +#include "cppjieba/Jieba.hpp" namespace paimon::lucene { Result LuceneUtils::GetJiebaDictionaryDir() { @@ -30,4 +37,27 @@ Result LuceneUtils::GetJiebaDictionaryDir() { fmt::format("cannot get dictionary dir for jieba, must set env {}", kJiebaDictDirEnv)); } +Result> LuceneUtils::GetJieba() { + PAIMON_ASSIGN_OR_RAISE(std::string configured_dir, GetJiebaDictionaryDir()); + std::error_code ec; + std::string dictionary_dir = std::filesystem::canonical(configured_dir, ec).string(); + if (ec) { + return Status::IOError(fmt::format("cannot resolve dictionary dir {} for jieba: {}", + configured_dir, ec.message())); + } + static std::mutex mutex; + static std::map> dir_to_jieba; + // The lock is held while loading, so concurrent callers wait for one load of a directory + // instead of each loading it. + std::lock_guard lock(mutex); + std::shared_ptr& jieba = dir_to_jieba[dictionary_dir]; + if (!jieba) { + jieba = std::make_shared( + dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8", + dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8", + dictionary_dir + "/stop_words.utf8"); + } + return jieba; +} + } // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_utils.h b/src/paimon/global_index/lucene/lucene_utils.h index c3dea2dc5..4c2df9f30 100644 --- a/src/paimon/global_index/lucene/lucene_utils.h +++ b/src/paimon/global_index/lucene/lucene_utils.h @@ -16,10 +16,16 @@ * limitations under the License. */ #pragma once +#include +#include + #include "fmt/format.h" #include "lucene++/StringUtils.h" #include "paimon/global_index/lucene/lucene_defs.h" #include "paimon/result.h" +namespace cppjieba { +class Jieba; +} // namespace cppjieba namespace paimon::lucene { class LuceneUtils { public: @@ -37,5 +43,11 @@ class LuceneUtils { } static Result GetJiebaDictionaryDir(); + + /// Returns the Jieba segmenter of the dictionaries in `GetJiebaDictionaryDir()`, which is + /// resolved to a canonical absolute path. Loading the dictionaries is slow, so each directory + /// is loaded on first use, cached until the process exits and shared by all Lucene index + /// readers and writers; its const methods are safe to call from many threads. + static Result> GetJieba(); }; } // namespace paimon::lucene diff --git a/src/paimon/global_index/lucene/lucene_utils_test.cpp b/src/paimon/global_index/lucene/lucene_utils_test.cpp new file mode 100644 index 000000000..ee83f2f8e --- /dev/null +++ b/src/paimon/global_index/lucene/lucene_utils_test.cpp @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "paimon/global_index/lucene/lucene_utils.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cppjieba/Jieba.hpp" +#include "fmt/format.h" +#include "gtest/gtest.h" +#include "paimon/common/utils/path_util.h" +#include "paimon/fs/file_system.h" +#include "paimon/testing/utils/testharness.h" +namespace paimon::lucene::test { +class LuceneUtilsTest : public ::testing::Test { + public: + void SetUp() override { + const char* dict_dir_env = std::getenv(kJiebaDictDirEnv); + if (dict_dir_env != nullptr) { + original_dict_dir_env_ = dict_dir_env; + } + ASSERT_OK_AND_ASSIGN(working_dir_, PathUtil::GetWorkingDirectory()); + ASSERT_OK_AND_ASSIGN(std::string dict_dir, LuceneUtils::GetJiebaDictionaryDir()); + default_dict_dir_ = std::filesystem::canonical(dict_dir); + test_dir_ = paimon::test::UniqueTestDirectory::Create(); + ASSERT_TRUE(test_dir_); + } + + void TearDown() override { + std::error_code ec; + std::filesystem::current_path(working_dir_, ec); + if (original_dict_dir_env_) { + setenv(kJiebaDictDirEnv, original_dict_dir_env_->c_str(), /*overwrite=*/1); + } else { + unsetenv(kJiebaDictDirEnv); + } + } + + std::string CopyDictionary(const std::string& relative_dir) const { + std::string dir = PathUtil::JoinPath(test_dir_->Str(), relative_dir); + EXPECT_TRUE(paimon::test::TestUtil::CopyDirectory(default_dict_dir_, dir)); + return dir; + } + + protected: + std::optional original_dict_dir_env_; + std::string working_dir_; + std::filesystem::path default_dict_dir_; + std::unique_ptr test_dir_; +}; + +TEST_F(LuceneUtilsTest, TestGetJiebaSharesOneSegmenter) { + std::string dict_dir = CopyDictionary("dict"); + setenv(kJiebaDictDirEnv, dict_dir.c_str(), /*overwrite=*/1); + + constexpr size_t kThreadNum = 4; + std::vector> jiebas(kThreadNum); + std::vector threads; + threads.reserve(kThreadNum); + for (size_t i = 0; i < kThreadNum; ++i) { + threads.emplace_back([&jiebas, i]() { + ASSERT_OK_AND_ASSIGN(jiebas[i], LuceneUtils::GetJieba()); + std::vector words; + jiebas[i]->CutForSearch("我爱机器学习", words); + std::vector expected = {"我", "爱", "机器", "学习"}; + ASSERT_EQ(words, expected); + }); + } + for (auto& thread : threads) { + thread.join(); + } + ASSERT_TRUE(jiebas[0]); + for (const auto& jieba : jiebas) { + ASSERT_EQ(jieba, jiebas[0]); + } + + std::weak_ptr cached_jieba = jiebas[0]; + jiebas.clear(); + ASSERT_FALSE(cached_jieba.expired()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr jieba, LuceneUtils::GetJieba()); + ASSERT_EQ(jieba, cached_jieba.lock()); +} + +TEST_F(LuceneUtilsTest, TestGetJiebaResolvesRelativeDictionaryDir) { + std::string relative_dir = default_dict_dir_.filename().string(); + setenv(kJiebaDictDirEnv, relative_dir.c_str(), /*overwrite=*/1); + std::filesystem::current_path(test_dir_->Str()); + ASSERT_NOK_WITH_MSG(LuceneUtils::GetJieba(), + fmt::format("cannot resolve dictionary dir {} for jieba", relative_dir)); + + std::string user_dict = PathUtil::JoinPath(CopyDictionary(relative_dir), "user.dict.utf8"); + ASSERT_OK(test_dir_->GetFileSystem()->WriteFile(user_dict, "机器学习\n", /*overwrite=*/true)); + ASSERT_OK_AND_ASSIGN(std::shared_ptr jieba_with_user_word, + LuceneUtils::GetJieba()); + std::filesystem::current_path(default_dict_dir_.parent_path()); + ASSERT_OK_AND_ASSIGN(std::shared_ptr default_jieba, + LuceneUtils::GetJieba()); + ASSERT_NE(jieba_with_user_word, default_jieba); + + std::vector words; + jieba_with_user_word->Cut("机器学习", words); + ASSERT_EQ(words, std::vector({"机器学习"})); + default_jieba->Cut("机器学习", words); + ASSERT_EQ(words, std::vector({"机器", "学习"})); + + setenv(kJiebaDictDirEnv, default_dict_dir_.c_str(), /*overwrite=*/1); + ASSERT_OK_AND_ASSIGN(std::shared_ptr jieba, LuceneUtils::GetJieba()); + ASSERT_EQ(jieba, default_jieba); +} + +} // namespace paimon::lucene::test