Merge branch 'main' into perf/manifest-row-range-cache

Author: lxy-9602

.devcontainer/Dockerfile.template

@@ -71,7 +71,7 @@ RUN apt update && \
     vim \
     wget \
     sudo \
-    # ---- additions for tantivy-fts migration (Rust + Sanitizer + LLVM) ----
+    # ---- additions for the Rust FFI builds (Rust + Sanitizer + LLVM) ----
     clang \
     clang-format \
     clang-tidy \

.devcontainer/centos7/Dockerfile

@@ -14,10 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# CentOS 7 cross-build verification image for paimon-cpp + tantivy-fts.
+# CentOS 7 cross-build verification image for paimon-cpp + the full-text index.
 #
 # Purpose:
-#   Prove the tantivy-fts stack builds on the OLDEST reasonable Linux target
+#   Prove the full-text index stack builds on the OLDEST reasonable Linux target
 #   (glibc 2.17, EOL 2024-06-30). The default Ubuntu 24.04 dev container
 #   proves nothing about glibc compatibility; this image does.
 #
@@ -95,7 +95,7 @@ enabled=0\n' > /etc/yum.repos.d/CentOS-Base.repo \
 # EPEL provides ninja-build and a newer python3 than the base 3.6.
 # SCL (Software Collections) provides devtoolset-11 (gcc 11) and rh-python38
 # without overriding the system gcc/python. CentOS 7's default gcc 4.8 is
-# too old for C++17/20 used by lucene++ and our tantivy wrapper.
+# too old for C++17/20 used by lucene++ and our full-text wrapper.
 #
 # Same story as CentOS-Base.repo: both epel + SCL default to mirrorlist
 # endpoints that are effectively dead; overwrite with aliyun URLs that we

.devcontainer/centos7/run.sh

@@ -107,12 +107,12 @@ case "${cmd}" in
                 -DPAIMON_ENABLE_LUMINA=OFF \
                 -DPAIMON_ENABLE_JINDO=OFF \
                 -DPAIMON_ENABLE_LUCENE=ON \
-                -DPAIMON_ENABLE_TANTIVY=ON \
+                -DPAIMON_ENABLE_FULL_TEXT=ON \
                 -DPAIMON_ENABLE_ORC=ON \
                 -DPAIMON_ENABLE_AVRO=ON
             cmake --build build-centos7 -j "$(nproc)"
             ctest --test-dir build-centos7 \
-                -R "paimon-lucene-index-test|paimon-global-index-test|paimon-tantivy-.*-test" \
+                -R "paimon-lucene-index-test|paimon-global-index-test|paimon-full-text-index-test" \
                 --output-on-failure
         '
         ;;

.devcontainer/devcontainer.json.template

@@ -48,11 +48,11 @@
 		"source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly",
 		"source=paimon-cargo-registry,target=/home/paimon/.cargo/registry,type=volume",
 		"source=paimon-cargo-git,target=/home/paimon/.cargo/git,type=volume",
-		"source=paimon-rust-target,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume",
+		"source=paimon-rust-target,target=${containerWorkspaceFolder}/crates/lance_ffi/target,type=volume",
 		"source=paimon-build,target=${containerWorkspaceFolder}/build,type=volume",
 		"source=paimon-ccache,target=/home/paimon/.ccache,type=volume"
 	],
-	"postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
+	"postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/lance_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
 	"customizations": {
 		"vscode": {
 			"extensions": [
@@ -66,7 +66,7 @@
 			"settings": {
 				"editor.formatOnSave": true,
 				"rust-analyzer.linkedProjects": [
-					"crates/tantivy_ffi/Cargo.toml"
+					"crates/lance_ffi/Cargo.toml"
 				]
 			}
 		}

.devcontainer/x86_64/devcontainer.json.template

@@ -54,11 +54,11 @@
 		"source=${localEnv:HOME}/.ssh,target=/home/paimon/.ssh,type=bind,readonly",
 		"source=paimon-cargo-registry-amd64,target=/home/paimon/.cargo/registry,type=volume",
 		"source=paimon-cargo-git-amd64,target=/home/paimon/.cargo/git,type=volume",
-		"source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/crates/tantivy_ffi/target,type=volume",
+		"source=paimon-rust-target-amd64,target=${containerWorkspaceFolder}/crates/lance_ffi/target,type=volume",
 		"source=paimon-build-amd64,target=${containerWorkspaceFolder}/build,type=volume",
 		"source=paimon-ccache-amd64,target=/home/paimon/.ccache,type=volume"
 	],
-	"postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/tantivy_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
+	"postCreateCommand": "sudo chown -R paimon:paimon ${containerWorkspaceFolder}/build ${containerWorkspaceFolder}/crates/lance_ffi/target /home/paimon/.ccache /home/paimon/.cargo/registry /home/paimon/.cargo/git 2>/dev/null || true; cargo install cbindgen --locked || true; rustup component add rust-src rust-analyzer clippy rustfmt || true",
 	"customizations": {
 		"vscode": {
 			"extensions": [

.github/workflows/build_and_test.yaml

@@ -67,6 +67,8 @@ jobs:
           - name: gcc-debug-x86_64
             cc: gcc-14
             cxx: g++-14
+            # Also read the full-text archives written by Paimon C++ with the Python reader.
+            build_args: --verify_full_text_archives
           - name: clang-debug-x86_64
             fetch_depth: '0' # fetch the PR target branch history for clang-tidy
             timeout_minutes: 180

CMakeLists.txt

@@ -19,7 +19,7 @@
 
 cmake_minimum_required(VERSION 3.22)
 # 3.22 is the minimum required by Corrosion-rs (used for the Rust<->C++ FFI
-# integration, see crates/tantivy_ffi). Ubuntu 24.04 ships CMake 3.28 and
+# integration, see crates/lance_ffi). Ubuntu 24.04 ships CMake 3.28 and
 # CentOS 8+/RHEL 9+ ship 3.20+, so older distributions need a newer CMake
 # installation.
 message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
@@ -69,8 +69,8 @@ option(PAIMON_ENABLE_NETWORK_TESTS
        "Whether to enable tests that access real remote services over the network" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
 option(PAIMON_ENABLE_LUCENE "Whether to enable lucene index" OFF)
-option(PAIMON_ENABLE_TANTIVY
-       "Whether to enable tantivy-fulltext global index (Rust FFI, experimental)" OFF)
+option(PAIMON_ENABLE_FULL_TEXT
+       "Whether to enable full-text global index (Rust FFI, experimental)" OFF)
 option(PAIMON_ENABLE_REST
        "Whether to enable the rest catalog (requires libcurl and OpenSSL)" OFF)
 if(PAIMON_ENABLE_ORC)
@@ -126,8 +126,8 @@ if(PAIMON_ENABLE_LUCENE)
     add_definitions(-DPAIMON_ENABLE_LUCENE)
 endif()
 
-if(PAIMON_ENABLE_TANTIVY)
-    add_definitions(-DPAIMON_ENABLE_TANTIVY)
+if(PAIMON_ENABLE_FULL_TEXT)
+    add_definitions(-DPAIMON_ENABLE_FULL_TEXT)
 endif()
 
 add_definitions(-DSNAPPY_CODEC_AVAILABLE)
@@ -333,7 +333,7 @@ add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL)
 add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL)
 
 # ---- Rust FFI crates via Corrosion-rs -------------------------------------------
-if(PAIMON_ENABLE_LANCE OR PAIMON_ENABLE_TANTIVY)
+if(PAIMON_ENABLE_LANCE)
     include(CorrosionFetch)
 endif()
 
@@ -346,13 +346,6 @@ if(PAIMON_ENABLE_LANCE)
     corrosion_set_env_vars(paimon_lance_ffi "PROTOC=${PAIMON_LANCE_PROTOC}")
 endif()
 
-if(PAIMON_ENABLE_TANTIVY)
-    corrosion_import_crate(MANIFEST_PATH crates/tantivy_ffi/Cargo.toml CRATES
-                           paimon_tantivy_ffi)
-    corrosion_experimental_cbindgen(TARGET paimon_tantivy_ffi HEADER_NAME
-                                    paimon_tantivy_ffi.h)
-endif()
-
 if(PAIMON_ENABLE_LUCENE)
     set(PAIMON_DICT_DEST "share/paimon/dict")
 
@@ -532,6 +525,13 @@ if(PAIMON_BUILD_TESTS)
                                            paimon_lucene_index_shared)
         list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
+    if(PAIMON_ENABLE_FULL_TEXT)
+        paimon_link_libraries_whole_archive(PAIMON_FULL_TEXT_INDEX_STATIC_LINK_LIBS
+                                            paimon_full_text_index_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_full_text_index_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+    endif()
 endif()
 
 if(PAIMON_BUILD_BENCHMARKS)
@@ -586,8 +586,8 @@ if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(src/paimon/indexer/lumina)
 endif()
 add_subdirectory(src/paimon/global_index/lucene)
-if(PAIMON_ENABLE_TANTIVY)
-    add_subdirectory(src/paimon/global_index/tantivy)
+if(PAIMON_ENABLE_FULL_TEXT)
+    add_subdirectory(src/paimon/global_index/full_text)
 endif()
 add_subdirectory(src/paimon/testing/mock)
 add_subdirectory(src/paimon/testing/utils)

ci/scripts/build_paimon.sh

@@ -18,7 +18,7 @@
 set -eux
 
 usage() {
-    echo "Usage: $0 --source_dir <path> [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type <type>] [--lint_git_target_commit <commit-or-branch>] [--install_smoke]"
+    echo "Usage: $0 --source_dir <path> [--enable_asan] [--enable_ubsan] [--enable_tsan] [--check_clang_tidy] [--build_type <type>] [--lint_git_target_commit <commit-or-branch>] [--install_smoke] [--verify_full_text_archives]"
 }
 
 source_dir=""
@@ -29,6 +29,7 @@ check_clang_tidy="false"
 build_type="Debug"
 lint_git_target_commit="origin/main"
 install_smoke="false"
+verify_full_text_archives="false"
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
@@ -79,6 +80,10 @@ while [[ $# -gt 0 ]]; do
             install_smoke="true"
             shift
             ;;
+        --verify_full_text_archives)
+            verify_full_text_archives="true"
+            shift
+            ;;
         -h | --help)
             usage
             exit 0
@@ -128,22 +133,26 @@ mkdir -p "${build_dir}"
 pushd "${build_dir}"
 
 ENABLE_LUMINA="ON"
-ENABLE_TANTIVY="ON"
+ENABLE_FULL_TEXT="ON"
 ENABLE_LANCE="ON"
 if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then
     ENABLE_LUMINA="OFF"
-    ENABLE_TANTIVY="OFF" # tantivy-fts (Rust FFI) is not built on the gcc-8 image.
+    ENABLE_FULL_TEXT="OFF" # The full-text index (Rust FFI) is not built on the gcc-8 image.
 fi
 if [[ "${enable_tsan}" == "true" ]]; then
-    ENABLE_TANTIVY="OFF" # Tantivy's Rust library is not TSAN-instrumented.
-    ENABLE_LANCE="OFF"   # Lance's Rust library is not TSAN-instrumented.
+    ENABLE_FULL_TEXT="OFF" # The full-text Rust library is not TSAN-instrumented.
+    ENABLE_LANCE="OFF"     # Lance's Rust library is not TSAN-instrumented.
 fi
 # CI always builds natively, so the host architecture is the target architecture.
 host_arch=$(uname -m)
 if [[ "${host_arch}" != "x86_64" ]]; then
     ENABLE_LUMINA="OFF"
     echo "=== Lumina disabled: no prebuilt artifacts for ${host_arch} ==="
 fi
+if [[ "${verify_full_text_archives}" == "true" && "${ENABLE_FULL_TEXT}" != "ON" ]]; then
+    echo "--verify_full_text_archives requires the full-text index, which this build disables" >&2
+    exit 1
+fi
 
 CMAKE_ARGS=(
     "-G Ninja"
@@ -156,7 +165,7 @@ CMAKE_ARGS=(
     "-DPAIMON_ENABLE_S3=ON"
     "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
     "-DPAIMON_ENABLE_LUCENE=ON"
-    "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}"
+    "-DPAIMON_ENABLE_FULL_TEXT=${ENABLE_FULL_TEXT}"
     "-DPAIMON_ENABLE_REST=ON"
     "-DPAIMON_LINT_GIT_TARGET_COMMIT=${lint_git_target_commit}"
 )
@@ -173,8 +182,25 @@ fi
 
 cmake "${CMAKE_ARGS[@]}" "${source_dir}"
 cmake --build . -- -j "${build_jobs}"
+if [[ "${verify_full_text_archives}" == "true" ]]; then
+    # TestCrossReadFixtures copies the full-text archives written by Paimon C++ here.
+    full_text_archive_dir="${build_dir}/full_text_archives"
+    mkdir -p "${full_text_archive_dir}"
+    export PAIMON_FULL_TEXT_ARCHIVE_OUTPUT_DIR="${full_text_archive_dir}"
+fi
 ctest --output-on-failure -j "${build_jobs}"
 
+if [[ "${verify_full_text_archives}" == "true" ]]; then
+    # Read them with the Python binding of the same engine and compare the search results with
+    # those of the checked-in fixtures.
+    full_text_venv="${build_dir}/full_text_venv"
+    python3 -m venv "${full_text_venv}"
+    "${full_text_venv}/bin/pip" install paimon-ftindex==0.1.0
+    "${full_text_venv}/bin/python" \
+        "${source_dir}/test/test_data/full_text_fixtures/generate_fixtures.py" \
+        --verify "${full_text_archive_dir}"
+fi
+
 if [[ "${check_clang_tidy}" == "true" ]]; then
     cmake --build . --target check-clang-tidy
 fi

ci/scripts/setup_rust.sh

@@ -15,8 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Install the Rust toolchain used by the Lance, Mosaic, and tantivy-fts FFI builds, plus
-# cbindgen required by Lance and tantivy-fts.
+# Install the Rust toolchain used by the Lance, Mosaic, and full-text index FFI builds, plus
+# cbindgen required by Lance.
 #
 # The dev container (see .devcontainer/) already has these preinstalled;
 # this script is for the GitHub Actions runners and is called before ci/scripts/build_paimon.sh.
@@ -26,8 +26,8 @@
 set -eux
 
 RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}
-# 1.88.0 is the minimum required by transitive crates (e.g. time 0.3.47).
-RUST_VERSION=${RUST_VERSION:-1.88.0}
+# 1.90.0 is the minimum required by paimon-full-text 0.1.0 (rust-version in its Cargo.toml).
+RUST_VERSION=${RUST_VERSION:-1.90.0}
 CBINDGEN_VERSION=${CBINDGEN_VERSION:-0.29.2}
 PROTOC_VERSION=27.4
 

cmake_modules/CorrosionFetch.cmake

@@ -14,8 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Pull Corrosion-rs via FetchContent so we can import crates/tantivy_ffi as
-# CMake targets for the tantivy-fulltext global index.
+# Pull Corrosion-rs via FetchContent so we can import crates/lance_ffi as
+# CMake targets for the Lance file format.
 #
 # Pinned to v0.5.2 (stable release). Requires CMake >= 3.22.