perf(lucene): load Jieba dictionaries once per process

docs/source/user_guide/global_index.rst

@@ -144,7 +144,7 @@ a column name containing only such whitespace selects the indexed field.
 
 - **Environment Variable**: ``PAIMON_JIEBA_DICT_DIR``
 
-  - **Description**: Specifies the directory containing Jieba dictionary files for Chinese text tokenization. At runtime, the system first checks this environment variable; if not set, it falls back to the compile-time ``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is available, will fail with an error.
+  - **Description**: Specifies the directory containing Jieba dictionary files for Chinese text tokenization. At runtime, the system first checks this environment variable; if not set, it falls back to the compile-time ``JIEBA_TEST_DICT_DIR`` macro (only available in test builds). If neither is available, will fail with an error. A relative directory is resolved against the current working directory. Each dictionary directory is loaded once per process and shared by all Lucene index readers and writers, so changes to its dictionary files take effect only after the process restarts.
 
 Full-Text Index (Experimental)
 ------------------------------

src/paimon/global_index/lucene/CMakeLists.txt

@@ -58,6 +58,7 @@ if(PAIMON_ENABLE_LUCENE)
                         lucene_directory_test.cpp
                         lucene_global_index_test.cpp
                         lucene_filter_test.cpp
+                        lucene_utils_test.cpp
                         EXTRA_INCLUDES
                         ${LUCENE_INCLUDE_DIR}
                         STATIC_LINK_LIBS

src/paimon/global_index/lucene/jieba_analyzer.cpp

@@ -24,7 +24,7 @@
 
 namespace paimon::lucene {
 JiebaTokenizerContext::JiebaTokenizerContext(const std::string& _tokenize_mode, bool _with_position,
-                                             const std::shared_ptr<cppjieba::Jieba>& _jieba,
+                                             const std::shared_ptr<const cppjieba::Jieba>& _jieba,
                                              const std::shared_ptr<MemoryPool>& _pool,
                                              int32_t _buffer_size)
     : pool(_pool),

src/paimon/global_index/lucene/jieba_analyzer.h

@@ -26,15 +26,15 @@
 namespace paimon::lucene {
 struct JiebaTokenizerContext {
     JiebaTokenizerContext(const std::string& _tokenize_mode, bool _with_position,
-                          const std::shared_ptr<cppjieba::Jieba>& _jieba,
+                          const std::shared_ptr<const cppjieba::Jieba>& _jieba,
                           const std::shared_ptr<MemoryPool>& _pool,
                           int32_t _buffer_size = kReadBufferSize);
 
     std::shared_ptr<MemoryPool> pool;
     std::string tokenize_mode;
     bool with_position;
     int32_t buffer_size;
-    std::shared_ptr<cppjieba::Jieba> jieba;
+    std::shared_ptr<const cppjieba::Jieba> jieba;
 
     static inline const int32_t kReadBufferSize = 5 * 1024 * 1024;
     static inline const int32_t kMaxWordLen = 1024;

src/paimon/global_index/lucene/lucene_global_index_reader.cpp

@@ -530,11 +530,8 @@ Result<std::shared_ptr<LuceneGlobalIndexReader>> LuceneGlobalIndexReader::Create
         Lucene::IndexReaderPtr reader = Lucene::IndexReader::open(lucene_dir, /*read_only=*/true);
         Lucene::IndexSearcherPtr searcher = Lucene::newLucene<Lucene::IndexSearcher>(reader);
 
-        PAIMON_ASSIGN_OR_RAISE(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir());
-        auto jieba = std::make_shared<cppjieba::Jieba>(
-            dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8",
-            dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8",
-            dictionary_dir + "/stop_words.utf8");
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const cppjieba::Jieba> jieba,
+                               LuceneUtils::GetJieba());
 
         // priority: read options > write options > kDefaultJiebaTokenizeMode
         PAIMON_ASSIGN_OR_RAISE(

src/paimon/global_index/lucene/lucene_global_index_reader.h

@@ -121,7 +121,7 @@ class LuceneGlobalIndexReader : public GlobalIndexReader {
     LuceneGlobalIndexReader(const std::wstring& wfield_name,
                             const Lucene::IndexSearcherPtr& searcher,
                             const std::string& tokenize_mode,
-                            const std::shared_ptr<cppjieba::Jieba>& jieba)
+                            const std::shared_ptr<const cppjieba::Jieba>& jieba)
         : wfield_name_(wfield_name),
           searcher_(searcher),
           tokenize_mode_(tokenize_mode),
@@ -142,6 +142,6 @@ class LuceneGlobalIndexReader : public GlobalIndexReader {
     std::wstring wfield_name_;
     Lucene::IndexSearcherPtr searcher_;
     std::string tokenize_mode_;
-    std::shared_ptr<cppjieba::Jieba> jieba_;
+    std::shared_ptr<const cppjieba::Jieba> jieba_;
 };
 }  // namespace paimon::lucene

src/paimon/global_index/lucene/lucene_global_index_writer.cpp

@@ -76,11 +76,8 @@ Result<std::shared_ptr<LuceneGlobalIndexWriter>> LuceneGlobalIndexWriter::Create
             std::string tokenize_mode,
             OptionsUtils::GetValueFromMap(options, kJiebaTokenizeMode,
                                           std::string(kDefaultJiebaTokenizeMode)));
-        PAIMON_ASSIGN_OR_RAISE(std::string dictionary_dir, LuceneUtils::GetJiebaDictionaryDir());
-        auto jieba = std::make_shared<cppjieba::Jieba>(
-            dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8",
-            dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8",
-            dictionary_dir + "/stop_words.utf8");
+        PAIMON_ASSIGN_OR_RAISE(std::shared_ptr<const cppjieba::Jieba> jieba,
+                               LuceneUtils::GetJieba());
         JiebaTokenizerContext jieba_context(tokenize_mode,
                                             /*with_position=*/!omit_term_freq_and_positions, jieba,
                                             pool);

src/paimon/global_index/lucene/lucene_utils.cpp

@@ -16,6 +16,13 @@
  * limitations under the License.
  */
 #include "paimon/global_index/lucene/lucene_utils.h"
+
+#include <filesystem>
+#include <map>
+#include <mutex>
+#include <system_error>
+
+#include "cppjieba/Jieba.hpp"
 namespace paimon::lucene {
 
 Result<std::string> LuceneUtils::GetJiebaDictionaryDir() {
@@ -30,4 +37,27 @@ Result<std::string> LuceneUtils::GetJiebaDictionaryDir() {
         fmt::format("cannot get dictionary dir for jieba, must set env {}", kJiebaDictDirEnv));
 }
 
+Result<std::shared_ptr<const cppjieba::Jieba>> LuceneUtils::GetJieba() {
+    PAIMON_ASSIGN_OR_RAISE(std::string configured_dir, GetJiebaDictionaryDir());
+    std::error_code ec;
+    std::string dictionary_dir = std::filesystem::canonical(configured_dir, ec).string();
+    if (ec) {
+        return Status::IOError(fmt::format("cannot resolve dictionary dir {} for jieba: {}",
+                                           configured_dir, ec.message()));
+    }
+    static std::mutex mutex;
+    static std::map<std::string, std::shared_ptr<const cppjieba::Jieba>> dir_to_jieba;
+    // The lock is held while loading, so concurrent callers wait for one load of a directory
+    // instead of each loading it.
+    std::lock_guard<std::mutex> lock(mutex);
+    std::shared_ptr<const cppjieba::Jieba>& jieba = dir_to_jieba[dictionary_dir];
+    if (!jieba) {
+        jieba = std::make_shared<cppjieba::Jieba>(
+            dictionary_dir + "/jieba.dict.utf8", dictionary_dir + "/hmm_model.utf8",
+            dictionary_dir + "/user.dict.utf8", dictionary_dir + "/idf.utf8",
+            dictionary_dir + "/stop_words.utf8");
+    }
+    return jieba;
+}
+
 }  // namespace paimon::lucene

src/paimon/global_index/lucene/lucene_utils.h

@@ -16,10 +16,16 @@
  * limitations under the License.
  */
 #pragma once
+#include <memory>
+#include <string>
+
 #include "fmt/format.h"
 #include "lucene++/StringUtils.h"
 #include "paimon/global_index/lucene/lucene_defs.h"
 #include "paimon/result.h"
+namespace cppjieba {
+class Jieba;
+}  // namespace cppjieba
 namespace paimon::lucene {
 class LuceneUtils {
  public:
@@ -37,5 +43,11 @@ class LuceneUtils {
     }
 
     static Result<std::string> GetJiebaDictionaryDir();
+
+    /// Returns the Jieba segmenter of the dictionaries in `GetJiebaDictionaryDir()`, which is
+    /// resolved to a canonical absolute path. Loading the dictionaries is slow, so each directory
+    /// is loaded on first use, cached until the process exits and shared by all Lucene index
+    /// readers and writers; its const methods are safe to call from many threads.
+    static Result<std::shared_ptr<const cppjieba::Jieba>> GetJieba();
 };
 }  // namespace paimon::lucene

src/paimon/global_index/lucene/lucene_utils_test.cpp

@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "paimon/global_index/lucene/lucene_utils.h"
+
+#include <cstdlib>
+#include <filesystem>
+#include <memory>
+#include <optional>
+#include <string>
+#include <system_error>
+#include <thread>
+#include <vector>
+
+#include "cppjieba/Jieba.hpp"
+#include "fmt/format.h"
+#include "gtest/gtest.h"
+#include "paimon/common/utils/path_util.h"
+#include "paimon/fs/file_system.h"
+#include "paimon/testing/utils/testharness.h"
+namespace paimon::lucene::test {
+class LuceneUtilsTest : public ::testing::Test {
+ public:
+    void SetUp() override {
+        const char* dict_dir_env = std::getenv(kJiebaDictDirEnv);
+        if (dict_dir_env != nullptr) {
+            original_dict_dir_env_ = dict_dir_env;
+        }
+        ASSERT_OK_AND_ASSIGN(working_dir_, PathUtil::GetWorkingDirectory());
+        ASSERT_OK_AND_ASSIGN(std::string dict_dir, LuceneUtils::GetJiebaDictionaryDir());
+        default_dict_dir_ = std::filesystem::canonical(dict_dir);
+        test_dir_ = paimon::test::UniqueTestDirectory::Create();
+        ASSERT_TRUE(test_dir_);
+    }
+
+    void TearDown() override {
+        std::error_code ec;
+        std::filesystem::current_path(working_dir_, ec);
+        if (original_dict_dir_env_) {
+            setenv(kJiebaDictDirEnv, original_dict_dir_env_->c_str(), /*overwrite=*/1);
+        } else {
+            unsetenv(kJiebaDictDirEnv);
+        }
+    }
+
+    std::string CopyDictionary(const std::string& relative_dir) const {
+        std::string dir = PathUtil::JoinPath(test_dir_->Str(), relative_dir);
+        EXPECT_TRUE(paimon::test::TestUtil::CopyDirectory(default_dict_dir_, dir));
+        return dir;
+    }
+
+ protected:
+    std::optional<std::string> original_dict_dir_env_;
+    std::string working_dir_;
+    std::filesystem::path default_dict_dir_;
+    std::unique_ptr<paimon::test::UniqueTestDirectory> test_dir_;
+};
+
+TEST_F(LuceneUtilsTest, TestGetJiebaSharesOneSegmenter) {
+    std::string dict_dir = CopyDictionary("dict");
+    setenv(kJiebaDictDirEnv, dict_dir.c_str(), /*overwrite=*/1);
+
+    constexpr size_t kThreadNum = 4;
+    std::vector<std::shared_ptr<const cppjieba::Jieba>> jiebas(kThreadNum);
+    std::vector<std::thread> threads;
+    threads.reserve(kThreadNum);
+    for (size_t i = 0; i < kThreadNum; ++i) {
+        threads.emplace_back([&jiebas, i]() {
+            ASSERT_OK_AND_ASSIGN(jiebas[i], LuceneUtils::GetJieba());
+            std::vector<std::string> words;
+            jiebas[i]->CutForSearch("我爱机器学习", words);
+            std::vector<std::string> expected = {"我", "爱", "机器", "学习"};
+            ASSERT_EQ(words, expected);
+        });
+    }
+    for (auto& thread : threads) {
+        thread.join();
+    }
+    ASSERT_TRUE(jiebas[0]);
+    for (const auto& jieba : jiebas) {
+        ASSERT_EQ(jieba, jiebas[0]);
+    }
+
+    std::weak_ptr<const cppjieba::Jieba> cached_jieba = jiebas[0];
+    jiebas.clear();
+    ASSERT_FALSE(cached_jieba.expired());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<const cppjieba::Jieba> jieba, LuceneUtils::GetJieba());
+    ASSERT_EQ(jieba, cached_jieba.lock());
+}
+
+TEST_F(LuceneUtilsTest, TestGetJiebaResolvesRelativeDictionaryDir) {
+    std::string relative_dir = default_dict_dir_.filename().string();
+    setenv(kJiebaDictDirEnv, relative_dir.c_str(), /*overwrite=*/1);
+    std::filesystem::current_path(test_dir_->Str());
+    ASSERT_NOK_WITH_MSG(LuceneUtils::GetJieba(),
+                        fmt::format("cannot resolve dictionary dir {} for jieba", relative_dir));
+
+    std::string user_dict = PathUtil::JoinPath(CopyDictionary(relative_dir), "user.dict.utf8");
+    ASSERT_OK(test_dir_->GetFileSystem()->WriteFile(user_dict, "机器学习\n", /*overwrite=*/true));
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<const cppjieba::Jieba> jieba_with_user_word,
+                         LuceneUtils::GetJieba());
+    std::filesystem::current_path(default_dict_dir_.parent_path());
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<const cppjieba::Jieba> default_jieba,
+                         LuceneUtils::GetJieba());
+    ASSERT_NE(jieba_with_user_word, default_jieba);
+
+    std::vector<std::string> words;
+    jieba_with_user_word->Cut("机器学习", words);
+    ASSERT_EQ(words, std::vector<std::string>({"机器学习"}));
+    default_jieba->Cut("机器学习", words);
+    ASSERT_EQ(words, std::vector<std::string>({"机器", "学习"}));
+
+    setenv(kJiebaDictDirEnv, default_dict_dir_.c_str(), /*overwrite=*/1);
+    ASSERT_OK_AND_ASSIGN(std::shared_ptr<const cppjieba::Jieba> jieba, LuceneUtils::GetJieba());
+    ASSERT_EQ(jieba, default_jieba);
+}
+
+}  // namespace paimon::lucene::test