feat(format): support Lance file format (#315)

Author: wangyong9999

.codespell_ignore

@@ -9,3 +9,4 @@ VisitIn
 NotIn
 Collet
 convertor
+inout

.github/workflows/build_and_test.yaml

@@ -53,7 +53,7 @@ jobs:
   build-and-test:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.runner || 'ubuntu-24.04' }}
-    timeout-minutes: 120
+    timeout-minutes: ${{ matrix.timeout_minutes || 120 }}
     strategy:
       fail-fast: false
       matrix:
@@ -69,6 +69,7 @@ jobs:
             cxx: g++-14
           - name: clang-debug-x86_64
             fetch_depth: '0' # fetch the PR target branch history for clang-tidy
+            timeout_minutes: 180
             build_args: >-
               --check_clang_tidy
               --lint_git_target_commit "origin/${{ github.base_ref || github.event.repository.default_branch }}"
@@ -106,7 +107,7 @@ jobs:
         uses: ./.github/actions/setup-ccache
         with:
           cache-key-prefix: ccache-${{ matrix.name }}
-      - name: Install Rust toolchain (Mosaic and tantivy-fts)
+      - name: Install Rust and protobuf tools
         shell: bash
         run: ci/scripts/setup_rust.sh
       - name: Install build and debugging dependencies

.github/workflows/gcc8_test.yaml

@@ -60,7 +60,7 @@ jobs:
           ls -la
       - name: Checkout paimon-cpp
         uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-      - name: Install Rust toolchain (Mosaic)
+      - name: Install Rust and protobuf tools
         shell: bash
         run: ci/scripts/setup_rust.sh
       - name: Setup ccache

.github/workflows/release_candidate.yaml

@@ -131,7 +131,7 @@ jobs:
           name: source-archive
           path: release/ci
 
-      - name: Install Rust toolchain (Mosaic and tantivy-fts)
+      - name: Install Rust and protobuf tools
         shell: bash
         run: ci/scripts/setup_rust.sh
 

.gitignore

@@ -66,4 +66,4 @@ FlameGraph
 third_party/*.tar.gz
 
 # Rust / Cargo build artifacts
-crates/tantivy_ffi/target/
+crates/*/target/

CMakeLists.txt

@@ -61,6 +61,7 @@ option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_MOSAIC "Whether to enable mosaic file format (Rust FFI)" OFF)
+option(PAIMON_ENABLE_LANCE "Whether to enable lance file format (Rust FFI)" OFF)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
 option(PAIMON_ENABLE_OSS "Whether to enable OSS SDK v2 file system" OFF)
 option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
@@ -93,6 +94,9 @@ endif()
 if(PAIMON_ENABLE_MOSAIC)
     add_definitions(-DPAIMON_ENABLE_MOSAIC)
 endif()
+if(PAIMON_ENABLE_LANCE)
+    add_definitions(-DPAIMON_ENABLE_LANCE)
+endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
@@ -328,13 +332,21 @@ set(PAIMON_SHARED_PRIVATE_LINK_LIBS ${PAIMON_STATIC_LINK_LIBS})
 add_subdirectory(third_party/roaring_bitmap EXCLUDE_FROM_ALL)
 add_subdirectory(third_party/xxhash EXCLUDE_FROM_ALL)
 
-# ---- tantivy-fulltext Rust FFI via Corrosion-rs --------------------------------
-# Corrosion wraps the Cargo crate as a CMake target named `paimon_tantivy_ffi`.
-# `corrosion_experimental_cbindgen` runs cbindgen from CMake and writes the
-# header to a stable path; it also adds that path to the target's INTERFACE
-# include dirs so C++ consumers pick it up via target_link_libraries.
-if(PAIMON_ENABLE_TANTIVY)
+# ---- Rust FFI crates via Corrosion-rs -------------------------------------------
+if(PAIMON_ENABLE_LANCE OR PAIMON_ENABLE_TANTIVY)
     include(CorrosionFetch)
+endif()
+
+if(PAIMON_ENABLE_LANCE)
+    find_program(PAIMON_LANCE_PROTOC protoc REQUIRED)
+    corrosion_import_crate(MANIFEST_PATH crates/lance_ffi/Cargo.toml CRATES
+                           paimon_lance_ffi)
+    corrosion_experimental_cbindgen(TARGET paimon_lance_ffi HEADER_NAME
+                                    paimon_lance_ffi.h)
+    corrosion_set_env_vars(paimon_lance_ffi "PROTOC=${PAIMON_LANCE_PROTOC}")
+endif()
+
+if(PAIMON_ENABLE_TANTIVY)
     corrosion_import_crate(MANIFEST_PATH crates/tantivy_ffi/Cargo.toml CRATES
                            paimon_tantivy_ffi)
     corrosion_experimental_cbindgen(TARGET paimon_tantivy_ffi HEADER_NAME
@@ -403,6 +415,11 @@ if(PAIMON_BUILD_TESTS OR PAIMON_BUILD_BENCHMARKS)
                                             paimon_mosaic_file_format_static)
     endif()
 
+    if(PAIMON_ENABLE_LANCE)
+        paimon_link_libraries_whole_archive(PAIMON_LANCE_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_lance_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_whole_archive(PAIMON_ORC_FILE_FORMAT_STATIC_LINK_LIBS
                                             paimon_orc_file_format_static)
@@ -462,6 +479,14 @@ if(PAIMON_BUILD_TESTS)
                                             paimon_mosaic_file_format_static)
     endif()
 
+    if(PAIMON_ENABLE_LANCE)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_lance_file_format_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+        paimon_link_libraries_whole_archive(PAIMON_LANCE_FILE_FORMAT_STATIC_LINK_LIBS
+                                            paimon_lance_file_format_static)
+    endif()
+
     if(PAIMON_ENABLE_ORC)
         paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
                                            paimon_orc_file_format_shared)
@@ -554,6 +579,9 @@ add_subdirectory(src/paimon/format/avro)
 if(PAIMON_ENABLE_MOSAIC)
     add_subdirectory(src/paimon/format/mosaic)
 endif()
+if(PAIMON_ENABLE_LANCE)
+    add_subdirectory(src/paimon/format/lance)
+endif()
 if(PAIMON_ENABLE_LUMINA)
     add_subdirectory(src/paimon/global_index/lumina)
 endif()

ci/scripts/build_paimon.sh

@@ -129,12 +129,14 @@ pushd "${build_dir}"
 
 ENABLE_LUMINA="ON"
 ENABLE_TANTIVY="ON"
+ENABLE_LANCE="ON"
 if [[ "${CC:-}" == *"gcc-8"* ]] || [[ "${CXX:-}" == *"g++-8"* ]]; then
     ENABLE_LUMINA="OFF"
     ENABLE_TANTIVY="OFF" # tantivy-fts (Rust FFI) is not built on the gcc-8 image.
 fi
 if [[ "${enable_tsan}" == "true" ]]; then
     ENABLE_TANTIVY="OFF" # Tantivy's Rust library is not TSAN-instrumented.
+    ENABLE_LANCE="OFF"   # Lance's Rust library is not TSAN-instrumented.
 fi
 # CI always builds natively, so the host architecture is the target architecture.
 host_arch=$(uname -m)
@@ -148,6 +150,7 @@ CMAKE_ARGS=(
     "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_MOSAIC=ON"
+    "-DPAIMON_ENABLE_LANCE=${ENABLE_LANCE}"
     "-DPAIMON_ENABLE_JINDO=ON"
     "-DPAIMON_ENABLE_OSS=ON"
     "-DPAIMON_ENABLE_S3=ON"

ci/scripts/setup_rust.sh

@@ -15,8 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-# Install the Rust toolchain used by the Mosaic and tantivy-fts FFI builds, plus cbindgen required
-# by tantivy-fts.
+# Install the Rust toolchain used by the Lance, Mosaic, and tantivy-fts FFI builds, plus
+# cbindgen required by Lance and tantivy-fts.
 #
 # The dev container (see .devcontainer/) already has these preinstalled;
 # this script is for the GitHub Actions runners and is called before ci/scripts/build_paimon.sh.
@@ -29,6 +29,7 @@ RUSTUP_VERSION=${RUSTUP_VERSION:-1.29.0}
 # 1.88.0 is the minimum required by transitive crates (e.g. time 0.3.47).
 RUST_VERSION=${RUST_VERSION:-1.88.0}
 CBINDGEN_VERSION=${CBINDGEN_VERSION:-0.29.2}
+PROTOC_VERSION=27.4
 
 # Install rustup + default toolchain if cargo isn't on PATH yet.
 if ! command -v cargo >/dev/null 2>&1; then
@@ -44,6 +45,37 @@ rustup toolchain install "${RUST_VERSION}" --profile minimal
 rustup default "${RUST_VERSION}"
 rustup component add rustfmt clippy
 
+# Lance's protobuf definitions use proto3 optional fields, which require protoc >= 3.12.
+if ! command -v protoc >/dev/null 2>&1 || \
+   ! protoc --experimental_allow_proto3_optional --version >/dev/null 2>&1; then
+    case "$(uname -m)" in
+        x86_64)
+            protoc_arch="x86_64"
+            protoc_sha256="20a977d023a47a7f27557aa144eb2c06baa6f623784e34d8a13d9abb6f6bc6c0"
+            ;;
+        aarch64|arm64)
+            protoc_arch="aarch_64"
+            protoc_sha256="2399fa9e634880e70a8aa760b1b164ea0a8e8acac3bf541e732f9b3ade312cc9"
+            ;;
+        *)
+            echo "Unsupported architecture for protoc: $(uname -m)" >&2
+            exit 1
+            ;;
+    esac
+    protoc_home="${HOME}/.local/protoc-${PROTOC_VERSION}"
+    protoc_archive="/tmp/protoc-${PROTOC_VERSION}-${protoc_arch}.zip"
+    if [[ ! -x "${protoc_home}/bin/protoc" ]]; then
+        curl --proto '=https' --tlsv1.2 -fL \
+            "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${protoc_arch}.zip" \
+            -o "${protoc_archive}"
+        echo "${protoc_sha256}  ${protoc_archive}" | sha256sum --check
+        mkdir -p "${protoc_home}"
+        unzip -oq "${protoc_archive}" -d "${protoc_home}"
+    fi
+    export PATH="${protoc_home}/bin:${PATH}"
+    echo "${protoc_home}/bin" >> "${GITHUB_PATH:-/dev/null}" || true
+fi
+
 # cbindgen is used by the crate's build.rs to emit the C header that the
 # C++ side includes. Corrosion will also run cbindgen at CMake configure
 # time; both paths need it available.
@@ -54,3 +86,4 @@ fi
 rustc --version
 cargo --version
 cbindgen --version
+protoc --version