From 018fd5caf5bddf6027ff93e2622a696e6d31f217 Mon Sep 17 00:00:00 2001 From: seokjin0414 Date: Mon, 24 Aug 2026 14:05:54 +0200 Subject: [PATCH 1/2] [rust] Check in generated protobuf code and drop protoc requirement --- .github/workflows/client-integration.yml | 20 - .github/workflows/gateway-ci.yml | 14 - .github/workflows/python-release.yml | 39 - .github/workflows/rust-build-and-test.yml | 43 +- .github/workflows/rust-license-and-format.yml | 5 - .github/workflows/rust-release.yml | 12 +- fluss-gateway/Cargo.lock | 79 +- fluss-gateway/DEPENDENCIES.rust.tsv | 8 - fluss-rpc/src/main/proto/FlussApi.proto | 6 + fluss-rust/Cargo.lock | 8 +- fluss-rust/Cargo.toml | 2 +- fluss-rust/DEVELOPMENT.md | 4 +- fluss-rust/bindings/cpp/BUILD.bazel | 20 +- fluss-rust/bindings/cpp/CMakeLists.txt | 28 +- fluss-rust/bindings/cpp/bazel/cpp/deps.bzl | 1 - .../bazel-consumer/build/MODULE.bazel | 1 - .../bazel-consumer/system/MODULE.bazel | 3 +- .../bindings/cpp/scripts/ensure_protoc.sh | 277 --- fluss-rust/crates/fluss/Cargo.toml | 3 - fluss-rust/crates/fluss/build.rs | 42 - fluss-rust/crates/fluss/gen/Cargo.toml | 33 + fluss-rust/crates/fluss/gen/src/main.rs | 84 + fluss-rust/crates/fluss/proto/FlussApi.proto | 1430 ++++++++++++ fluss-rust/crates/fluss/regen.sh | 27 + fluss-rust/crates/fluss/src/lib.rs | 2 +- fluss-rust/crates/fluss/src/proto/fluss.rs | 1935 +++++++++++++++++ .../crates/fluss/tests/vendored_proto.rs | 36 + fluss-rust/docs/cpp-bazel-usage.md | 39 +- fluss-rust/docs/cpp-cmake-usage.md | 22 +- fluss-rust/scripts/vendor-proto.sh | 46 - .../docs/developer-guide/contributing.md | 14 +- .../release/verifying-a-release-candidate.md | 2 +- .../docs/user-guide/rust/installation.md | 12 +- website/community/dev/building.md | 6 +- website/community/dev/ide-setup.md | 2 +- .../creating-a-fluss-release.mdx | 2 +- .../how-to-release/pre-rc-checklist.md | 2 +- website/docs/apis/rust/installation.md | 12 +- 38 files changed, 3628 insertions(+), 693 deletions(-) delete mode 100755 fluss-rust/bindings/cpp/scripts/ensure_protoc.sh delete mode 100644 fluss-rust/crates/fluss/build.rs create mode 100644 fluss-rust/crates/fluss/gen/Cargo.toml create mode 100644 fluss-rust/crates/fluss/gen/src/main.rs create mode 100644 fluss-rust/crates/fluss/proto/FlussApi.proto create mode 100755 fluss-rust/crates/fluss/regen.sh create mode 100644 fluss-rust/crates/fluss/src/proto/fluss.rs create mode 100644 fluss-rust/crates/fluss/tests/vendored_proto.rs delete mode 100755 fluss-rust/scripts/vendor-proto.sh diff --git a/.github/workflows/client-integration.yml b/.github/workflows/client-integration.yml index 853ebf8ec39..0695794803e 100644 --- a/.github/workflows/client-integration.yml +++ b/.github/workflows/client-integration.yml @@ -162,10 +162,6 @@ jobs: path: /tmp - name: Load server image run: docker load -i /tmp/fluss-dev.tar - - name: Install protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -205,10 +201,6 @@ jobs: python-version: ${{ matrix.python }} - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 - - name: Install protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -260,10 +252,6 @@ jobs: python-version: "3.12" - name: Install uv uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 - - name: Install protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -300,10 +288,6 @@ jobs: path: /tmp - name: Load server image run: docker load -i /tmp/fluss-dev.tar - - name: Install protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install Apache Arrow C++ run: | sudo apt-get install -y -V ca-certificates lsb-release wget @@ -366,10 +350,6 @@ jobs: with: otp-version: ${{ env.OTP_VERSION }} elixir-version: ${{ env.ELIXIR_VERSION }} - - name: Install protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: diff --git a/.github/workflows/gateway-ci.yml b/.github/workflows/gateway-ci.yml index f5c2aa42d8a..d38c3af63ef 100644 --- a/.github/workflows/gateway-ci.yml +++ b/.github/workflows/gateway-ci.yml @@ -61,14 +61,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install protobuf compiler - run: brew install protobuf - if: runner.os == 'macOS' - - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - if: runner.os == 'Linux' - - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -105,9 +97,6 @@ jobs: - name: Install the MSRV toolchain run: rustup toolchain install 1.88.0 --profile minimal - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -132,9 +121,6 @@ jobs: with: tool: cargo-deny@0.14.22 - - name: Install protobuf compiler - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - name: Check dependency licenses (Apache-compatible) run: cargo deny --locked check licenses diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index ddbc4f0cf96..fd0af316062 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -48,14 +48,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install protoc - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - # Vendor the canonical proto so the sdist builds standalone from source. - - name: Vendor canonical proto into the crate - working-directory: fluss-rust - run: scripts/vendor-proto.sh - - uses: PyO3/maturin-action@v1 with: working-directory: fluss-rust/bindings/python @@ -82,20 +74,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install protoc (Linux) - if: runner.os == 'Linux' - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - - name: Install protoc (macOS) - if: runner.os == 'macOS' - run: brew install protobuf - - - name: Install protoc (Windows) - if: runner.os == 'Windows' - run: choco install protoc -y - shell: pwsh - - # Install protoc in manylinux container (x86_64/aarch64); script shared via YAML anchor - uses: PyO3/maturin-action@v1 with: working-directory: fluss-rust/bindings/python @@ -103,20 +81,6 @@ jobs: command: build args: --release -o dist -i python3.9 manylinux: ${{ matrix.manylinux || 'auto' }} - before-script-linux: &protoc-install | - set -e - ARCH=$(uname -m) - case "$ARCH" in - x86_64) ZIP=protoc-27.1-linux-x86_64.zip ;; - aarch64) ZIP=protoc-27.1-linux-aarch_64.zip ;; - *) echo "Unsupported arch $ARCH"; exit 1 ;; - esac - curl -sLO "https://github.com/protocolbuffers/protobuf/releases/download/v27.1/${ZIP}" - python3 -c "import zipfile; zipfile.ZipFile('${ZIP}').extractall('/tmp/protoc_install')" - chmod +x /tmp/protoc_install/bin/protoc - rm -f "${ZIP}" - export PATH="/tmp/protoc_install/bin:$PATH" - export PROTOC=/tmp/protoc_install/bin/protoc - uses: PyO3/maturin-action@v1 with: working-directory: fluss-rust/bindings/python @@ -124,7 +88,6 @@ jobs: command: build args: --release -o dist -i python3.10 manylinux: ${{ matrix.manylinux || 'auto' }} - before-script-linux: *protoc-install - uses: PyO3/maturin-action@v1 with: working-directory: fluss-rust/bindings/python @@ -132,7 +95,6 @@ jobs: command: build args: --release -o dist -i python3.11 manylinux: ${{ matrix.manylinux || 'auto' }} - before-script-linux: *protoc-install - uses: PyO3/maturin-action@v1 with: working-directory: fluss-rust/bindings/python @@ -140,7 +102,6 @@ jobs: command: build args: --release -o dist -i python3.12 manylinux: ${{ matrix.manylinux || 'auto' }} - before-script-linux: *protoc-install - name: Upload wheels uses: actions/upload-artifact@v7 diff --git a/.github/workflows/rust-build-and-test.yml b/.github/workflows/rust-build-and-test.yml index d59fadce592..18d552ea343 100644 --- a/.github/workflows/rust-build-and-test.yml +++ b/.github/workflows/rust-build-and-test.yml @@ -62,11 +62,6 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Install protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -80,3 +75,41 @@ jobs: env: RUST_LOG: DEBUG RUST_BACKTRACE: full + + proto-up-to-date: + timeout-minutes: 15 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Install protoc + uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 + with: + # Pinned: this job compares regenerated output byte-for-byte. + version: "27.1" + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Rust Cache + uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 + with: + workspaces: fluss-rust + + - name: Regenerate protobuf code + run: ./crates/fluss/regen.sh + + - name: Verify the checked-in code is up to date + id: verify + run: | + git diff --exit-code || { + echo "::error::Checked-in protobuf code is out of date with fluss-rpc/src/main/proto/FlussApi.proto. Run fluss-rust/crates/fluss/regen.sh and commit the result, or commit the files from this job's regenerated-proto artifact." + exit 1 + } + + - name: Upload regenerated files + if: failure() && steps.verify.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: regenerated-proto + path: | + fluss-rust/crates/fluss/src/proto/fluss.rs + fluss-rust/crates/fluss/proto/FlussApi.proto diff --git a/.github/workflows/rust-license-and-format.yml b/.github/workflows/rust-license-and-format.yml index 2c2d4f6b413..d8996d27704 100644 --- a/.github/workflows/rust-license-and-format.yml +++ b/.github/workflows/rust-license-and-format.yml @@ -75,11 +75,6 @@ jobs: - name: Check dependency licenses (Apache-compatible) run: cargo deny check licenses - - name: Install protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - name: Rust Cache uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: diff --git a/.github/workflows/rust-release.yml b/.github/workflows/rust-release.yml index d2f8901400a..c1e228607af 100644 --- a/.github/workflows/rust-release.yml +++ b/.github/workflows/rust-release.yml @@ -42,19 +42,11 @@ jobs: - uses: ./.github/actions/verify-tag-version - - name: Install protoc - run: sudo apt-get update && sudo apt-get install -y protobuf-compiler - - # build.rs reads the canonical proto from the in-repo fluss-rpc, which is - # outside the published crate; vendor it so the crate publishes standalone. - - name: Vendor canonical proto into the crate - run: scripts/vendor-proto.sh - - name: Dry run (crates/fluss) - run: cargo publish -p fluss-rs --dry-run --allow-dirty + run: cargo publish -p fluss-rs --dry-run --locked - name: Publish fluss-rs to crates.io if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, '-') - run: cargo publish -p fluss-rs --allow-dirty + run: cargo publish -p fluss-rs --locked env: CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/fluss-gateway/Cargo.lock b/fluss-gateway/Cargo.lock index 4c08f394a8f..b94e7604f72 100644 --- a/fluss-gateway/Cargo.lock +++ b/fluss-gateway/Cargo.lock @@ -777,12 +777,6 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" -[[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - [[package]] name = "flatbuffers" version = "25.12.19" @@ -846,7 +840,6 @@ dependencies = [ "parking_lot", "parse-display", "prost", - "prost-build", "rand 0.9.5", "scopeguard", "serde", @@ -861,12 +854,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "foldhash" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" - [[package]] name = "foldhash" version = "0.2.0" @@ -1055,22 +1042,13 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash 0.1.5", -] - [[package]] name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "foldhash 0.2.0", + "foldhash", ] [[package]] @@ -1631,12 +1609,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - [[package]] name = "num-bigint" version = "0.4.8" @@ -1795,17 +1767,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1851,16 +1812,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn 2.0.119", -] - [[package]] name = "proc-macro2" version = "1.0.107" @@ -1880,25 +1831,6 @@ dependencies = [ "prost-derive", ] -[[package]] -name = "prost-build" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" -dependencies = [ - "heck", - "itertools", - "log", - "multimap", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn 2.0.119", - "tempfile", -] - [[package]] name = "prost-derive" version = "0.14.4" @@ -1912,15 +1844,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "prost-types" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" -dependencies = [ - "prost", -] - [[package]] name = "quanta" version = "0.12.6" diff --git a/fluss-gateway/DEPENDENCIES.rust.tsv b/fluss-gateway/DEPENDENCIES.rust.tsv index ec9bc96c26d..1ba94a43853 100644 --- a/fluss-gateway/DEPENDENCIES.rust.tsv +++ b/fluss-gateway/DEPENDENCIES.rust.tsv @@ -64,11 +64,9 @@ equivalent@1.0.2 X X errno@0.3.14 X X fastrand@2.5.0 X X find-msvc-tools@0.1.11 X X -fixedbitset@0.5.7 X X flatbuffers@25.12.19 X fluss-gateway@1.0.0 X fluss-rs@1.0.0 X -foldhash@0.1.5 X foldhash@0.2.0 X form_urlencoded@1.2.2 X X funty@2.0.0 X @@ -88,7 +86,6 @@ getrandom@0.4.3 X X gloo-timers@0.3.0 X X half@2.7.1 X X hashbrown@0.14.5 X X -hashbrown@0.15.5 X X hashbrown@0.16.1 X X hashbrown@0.17.1 X X heck@0.5.0 X X @@ -144,7 +141,6 @@ metrics-exporter-prometheus@0.17.2 X metrics-util@0.20.4 X mime@0.3.17 X X mio@1.2.2 X -multimap@0.10.1 X X num-bigint@0.4.8 X X num-bigint@0.5.1 X X num-complex@0.4.6 X X @@ -160,19 +156,15 @@ parse-display@0.10.0 X X parse-display-derive@0.10.0 X X paste@1.0.15 X X percent-encoding@2.3.2 X X -petgraph@0.8.3 X X pin-project-lite@0.2.17 X X pkg-config@0.3.34 X X portable-atomic@1.15.0 X X portable-atomic-util@0.2.7 X X potential_utf@0.1.5 X ppv-lite86@0.2.21 X X -prettyplease@0.2.37 X X proc-macro2@1.0.107 X X prost@0.14.4 X -prost-build@0.14.4 X prost-derive@0.14.4 X -prost-types@0.14.4 X quanta@0.12.6 X quick-xml@0.38.4 X quote@1.0.47 X X diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto index fa762da93e3..d03acb92fbe 100644 --- a/fluss-rpc/src/main/proto/FlussApi.proto +++ b/fluss-rpc/src/main/proto/FlussApi.proto @@ -27,6 +27,12 @@ option optimize_for = LITE_RUNTIME; * 2. DO NOT USE "enum" type */ +/** +* NOTICE: The Rust client checks in the code generated from this file. After changing it, run +* fluss-rust/crates/fluss/regen.sh and commit the result; the proto-up-to-date job of the Rust +* CI fails otherwise. + */ + // the message indicates a failed request of a server failure. message ErrorResponse { required int32 error_code = 1; diff --git a/fluss-rust/Cargo.lock b/fluss-rust/Cargo.lock index 961b4f221b9..7e716e90e56 100644 --- a/fluss-rust/Cargo.lock +++ b/fluss-rust/Cargo.lock @@ -1129,7 +1129,6 @@ dependencies = [ "parking_lot", "parse-display 0.10.0", "prost", - "prost-build", "rand 0.9.3", "scopeguard", "serde", @@ -1304,6 +1303,13 @@ dependencies = [ "slab", ] +[[package]] +name = "gen" +version = "0.1.0" +dependencies = [ + "prost-build", +] + [[package]] name = "generic-array" version = "0.14.7" diff --git a/fluss-rust/Cargo.toml b/fluss-rust/Cargo.toml index b7534d62064..3a82f8fca51 100644 --- a/fluss-rust/Cargo.toml +++ b/fluss-rust/Cargo.toml @@ -28,7 +28,7 @@ keywords = ["fluss", "streaming-storage", "datalake"] [workspace] resolver = "2" -members = ["crates/fluss", "crates/fluss-test-cluster", "crates/examples", "bindings/python", "bindings/cpp", "bindings/elixir/native/fluss_nif"] +members = ["crates/fluss", "crates/fluss/gen", "crates/fluss-test-cluster", "crates/examples", "bindings/python", "bindings/cpp", "bindings/elixir/native/fluss_nif"] [workspace.dependencies] fluss = { package = "fluss-rs", version = "1.0.0", path = "crates/fluss", features = ["storage-all"] } diff --git a/fluss-rust/DEVELOPMENT.md b/fluss-rust/DEVELOPMENT.md index c10f9d81e96..4a823c967e1 100644 --- a/fluss-rust/DEVELOPMENT.md +++ b/fluss-rust/DEVELOPMENT.md @@ -4,16 +4,16 @@ Welcome to the development guide of `fluss-rust`! This project builds `fluss-rus ## Pre-requisites -- protobuf - rust You can install these using your favourite package / version manager. Example installation using mise: ```bash -mise install protobuf mise install rust ``` +`protobuf` (the `protoc` compiler) is only needed when regenerating the protobuf code after the canonical `FlussApi.proto` changes (see `crates/fluss/regen.sh`). Regular builds use the checked-in `crates/fluss/src/proto/fluss.rs`. + ## IDE Setup We recommend [RustRover](https://www.jetbrains.com/rust/) IDE to work with fluss-rust code base. diff --git a/fluss-rust/bindings/cpp/BUILD.bazel b/fluss-rust/bindings/cpp/BUILD.bazel index d247baf18cd..a3873872f8e 100644 --- a/fluss-rust/bindings/cpp/BUILD.bazel +++ b/fluss-rust/bindings/cpp/BUILD.bazel @@ -34,7 +34,7 @@ config_setting( values = {"compilation_mode": "opt"}, ) -_PROTOC_SETUP_SNIPPET = """ +_CARGO_SETUP_SNIPPET = """ set -e if [ -n "$${CARGO:-}" ]; then if [ ! -x "$$CARGO" ]; then @@ -49,20 +49,6 @@ _PROTOC_SETUP_SNIPPET = """ exit 1 fi fi - if [ -n "$${PROTOC:-}" ]; then - if [ ! -x "$$PROTOC" ]; then - echo "Error: PROTOC is set but not executable: $$PROTOC" >&2 - exit 1 - fi - export PROTOC - else - PROTOC_BIN=$$(command -v protoc || true) - if [ -z "$$PROTOC_BIN" ]; then - echo "Error: protoc not found in PATH and PROTOC is not set" >&2 - exit 1 - fi - export PROTOC="$$PROTOC_BIN" - fi """ genrule( @@ -78,7 +64,7 @@ genrule( "src/lib.rs_debug.h", "cxxbridge/rust/cxx_debug.h", ], - cmd = _PROTOC_SETUP_SNIPPET + """ + cmd = _CARGO_SETUP_SNIPPET + """ EXECROOT=$$(pwd) OUTPUT_LIB=$(location rust_lib_debug.a) OUTPUT_CC=$(location rust_bridge_cc_debug.cc) @@ -144,7 +130,7 @@ genrule( "src/lib.rs_release.h", "cxxbridge/rust/cxx_release.h", ], - cmd = _PROTOC_SETUP_SNIPPET + """ + cmd = _CARGO_SETUP_SNIPPET + """ EXECROOT=$$(pwd) OUTPUT_LIB=$(location rust_lib_release.a) OUTPUT_CC=$(location rust_bridge_cc_release.cc) diff --git a/fluss-rust/bindings/cpp/CMakeLists.txt b/fluss-rust/bindings/cpp/CMakeLists.txt index c72b3675bd5..a8d339d0385 100644 --- a/fluss-rust/bindings/cpp/CMakeLists.txt +++ b/fluss-rust/bindings/cpp/CMakeLists.txt @@ -31,7 +31,6 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(FLUSS_CPP_DEP_MODE "system" CACHE STRING "Dependency provisioning mode for fluss-cpp (system|build)") set_property(CACHE FLUSS_CPP_DEP_MODE PROPERTY STRINGS system build) set(FLUSS_CPP_ARROW_VERSION "19.0.1" CACHE STRING "Arrow C++ version baseline for fluss-cpp") -set(FLUSS_CPP_PROTOBUF_VERSION "3.25.5" CACHE STRING "Protobuf/protoc version baseline for fluss-cpp") set(FLUSS_CPP_ARROW_SYSTEM_ROOT "" CACHE PATH "Optional Arrow installation prefix for system mode") set(FLUSS_CPP_ARROW_SOURCE_URL "https://github.com/apache/arrow/archive/refs/tags/apache-arrow-19.0.1.tar.gz" @@ -64,11 +63,6 @@ if (NOT FLUSS_CPP_DEP_MODE STREQUAL "system" AND NOT FLUSS_CPP_DEP_MODE STREQUAL message(FATAL_ERROR "Unsupported FLUSS_CPP_DEP_MODE='${FLUSS_CPP_DEP_MODE}'. Expected 'system' or 'build'.") endif() -find_program(FLUSS_PROTOC_EXECUTABLE NAMES protoc) -if (NOT FLUSS_PROTOC_EXECUTABLE) - message(FATAL_ERROR "protoc not found. Install protoc or set it in PATH. (Fluss baseline: ${FLUSS_CPP_PROTOBUF_VERSION})") -endif() - if (DEFINED ENV{CARGO} AND NOT "$ENV{CARGO}" STREQUAL "" AND EXISTS "$ENV{CARGO}") set(FLUSS_CARGO_EXECUTABLE "$ENV{CARGO}") else() @@ -81,27 +75,7 @@ if (NOT FLUSS_CARGO_EXECUTABLE) message(FATAL_ERROR "cargo not found. Install Rust toolchain or set CARGO/PATH.") endif() -execute_process( - COMMAND ${FLUSS_PROTOC_EXECUTABLE} --version - OUTPUT_VARIABLE FLUSS_PROTOC_VERSION_OUTPUT - OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_QUIET -) -string(REGEX MATCH "([0-9]+\\.[0-9]+\\.[0-9]+)" FLUSS_PROTOC_VERSION "${FLUSS_PROTOC_VERSION_OUTPUT}") -set(FLUSS_PROTOC_VERSION_NORM "${FLUSS_PROTOC_VERSION}") -set(FLUSS_CPP_PROTOBUF_VERSION_NORM "${FLUSS_CPP_PROTOBUF_VERSION}") -string(REGEX REPLACE "^3\\." "" FLUSS_PROTOC_VERSION_NORM "${FLUSS_PROTOC_VERSION_NORM}") -string(REGEX REPLACE "^3\\." "" FLUSS_CPP_PROTOBUF_VERSION_NORM "${FLUSS_CPP_PROTOBUF_VERSION_NORM}") -if (FLUSS_PROTOC_VERSION AND - NOT FLUSS_PROTOC_VERSION VERSION_EQUAL FLUSS_CPP_PROTOBUF_VERSION AND - NOT FLUSS_PROTOC_VERSION_NORM VERSION_EQUAL FLUSS_CPP_PROTOBUF_VERSION_NORM) - message(WARNING - "protoc version (${FLUSS_PROTOC_VERSION}) does not match Fluss baseline " - "(${FLUSS_CPP_PROTOBUF_VERSION}). Build may still work, but this is outside the tested baseline.") -endif() - message(STATUS "Fluss C++ dependency mode: ${FLUSS_CPP_DEP_MODE}") -message(STATUS "Fluss C++ protoc executable: ${FLUSS_PROTOC_EXECUTABLE} (${FLUSS_PROTOC_VERSION_OUTPUT})") message(STATUS "Fluss C++ cargo executable: ${FLUSS_CARGO_EXECUTABLE}") if (FLUSS_CPP_DEP_MODE STREQUAL "system") @@ -196,7 +170,7 @@ if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") endif() add_custom_target(cargo_build - COMMAND ${CMAKE_COMMAND} -E env PROTOC=${FLUSS_PROTOC_EXECUTABLE} ${FLUSS_CARGO_EXECUTABLE} build --manifest-path ${CARGO_MANIFEST} ${CARGO_BUILD_FLAGS} + COMMAND ${FLUSS_CARGO_EXECUTABLE} build --manifest-path ${CARGO_MANIFEST} ${CARGO_BUILD_FLAGS} BYPRODUCTS ${RUST_BRIDGE_CPP} ${RUST_LIB} ${RUST_HEADER_FILE} DEPENDS ${RUST_SOURCE_FILE} USES_TERMINAL diff --git a/fluss-rust/bindings/cpp/bazel/cpp/deps.bzl b/fluss-rust/bindings/cpp/bazel/cpp/deps.bzl index 6dd5e1b635c..08aee1915a4 100644 --- a/fluss-rust/bindings/cpp/bazel/cpp/deps.bzl +++ b/fluss-rust/bindings/cpp/bazel/cpp/deps.bzl @@ -127,7 +127,6 @@ _ARROW_BUILD_VERSIONS = { _config_tag = tag_class(attrs = { "mode": attr.string(default = "build"), "arrow_cpp_version": attr.string(default = "19.0.1"), - "protobuf_version": attr.string(default = "3.25.5"), "ep_cmake_ranlib": attr.string(default = "ranlib"), "ep_cmake_ar": attr.string(default = "ar"), "ep_cmake_nm": attr.string(default = "nm"), diff --git a/fluss-rust/bindings/cpp/examples/bazel-consumer/build/MODULE.bazel b/fluss-rust/bindings/cpp/examples/bazel-consumer/build/MODULE.bazel index f31165c1cdf..1f45c59f629 100644 --- a/fluss-rust/bindings/cpp/examples/bazel-consumer/build/MODULE.bazel +++ b/fluss-rust/bindings/cpp/examples/bazel-consumer/build/MODULE.bazel @@ -30,7 +30,6 @@ local_path_override( fluss_cpp = use_extension("@fluss-cpp//bindings/cpp/bazel/cpp:deps.bzl", "cpp_sdk") fluss_cpp.config( mode = "build", - protobuf_version = "3.25.5", arrow_cpp_version = "19.0.1", ep_cmake_ranlib = "/usr/bin/ranlib", ep_cmake_ar = "/usr/bin/ar", diff --git a/fluss-rust/bindings/cpp/examples/bazel-consumer/system/MODULE.bazel b/fluss-rust/bindings/cpp/examples/bazel-consumer/system/MODULE.bazel index 2a4d6a65846..d03591943f4 100644 --- a/fluss-rust/bindings/cpp/examples/bazel-consumer/system/MODULE.bazel +++ b/fluss-rust/bindings/cpp/examples/bazel-consumer/system/MODULE.bazel @@ -28,11 +28,10 @@ local_path_override( path = "../../../../../", ) -# Intended interface for preinstalled protoc + Arrow C++ environments. +# Intended interface for preinstalled Arrow C++ environments. fluss_cpp = use_extension("@fluss-cpp//bindings/cpp/bazel/cpp:deps.bzl", "cpp_sdk") fluss_cpp.config( mode = "system", - protobuf_version = "3.25.5", arrow_cpp_version = "19.0.1", # Adjust these paths for your environment. # Ubuntu 22.04 (apt / custom package) commonly uses lib/x86_64-linux-gnu. diff --git a/fluss-rust/bindings/cpp/scripts/ensure_protoc.sh b/fluss-rust/bindings/cpp/scripts/ensure_protoc.sh deleted file mode 100755 index 3210bcc7a51..00000000000 --- a/fluss-rust/bindings/cpp/scripts/ensure_protoc.sh +++ /dev/null @@ -1,277 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -PROTOBUF_BASELINE_VERSION="${PROTOBUF_BASELINE_VERSION:-3.25.5}" -if [[ -n "${XDG_CACHE_HOME:-}" ]]; then - _PROTOC_DEFAULT_CACHE_BASE="${XDG_CACHE_HOME}" -elif [[ -n "${HOME:-}" ]]; then - _PROTOC_DEFAULT_CACHE_BASE="${HOME}/.cache" -else - _PROTOC_DEFAULT_CACHE_BASE="/tmp" -fi - -_PROTOC_UNAME_S="$(uname -s | tr '[:upper:]' '[:lower:]')" -case "${_PROTOC_UNAME_S}" in - linux*) - _PROTOC_DEFAULT_OS="linux" - ;; - darwin*) - _PROTOC_DEFAULT_OS="osx" - ;; - *) - echo "ERROR: unsupported host OS '${_PROTOC_UNAME_S}'. Please set PROTOC_OS explicitly." >&2 - exit 1 - ;; -esac - -_PROTOC_UNAME_M="$(uname -m)" -case "${_PROTOC_UNAME_M}" in - x86_64|amd64) - _PROTOC_DEFAULT_ARCH="x86_64" - ;; - aarch64|arm64) - _PROTOC_DEFAULT_ARCH="aarch_64" - ;; - *) - echo "ERROR: unsupported host arch '${_PROTOC_UNAME_M}'. Please set PROTOC_ARCH explicitly." >&2 - exit 1 - ;; -esac - -PROTOC_INSTALL_ROOT="${PROTOC_INSTALL_ROOT:-${_PROTOC_DEFAULT_CACHE_BASE}/fluss-cpp-tools}" -PROTOC_OS="${PROTOC_OS:-${_PROTOC_DEFAULT_OS}}" -PROTOC_ARCH="${PROTOC_ARCH:-${_PROTOC_DEFAULT_ARCH}}" -PROTOC_FORCE_INSTALL="${PROTOC_FORCE_INSTALL:-0}" -PROTOC_PRINT_PATH_ONLY="${PROTOC_PRINT_PATH_ONLY:-0}" -PROTOC_ALLOW_INSECURE_DOWNLOAD="${PROTOC_ALLOW_INSECURE_DOWNLOAD:-0}" -PROTOC_SKIP_CHECKSUM_VERIFY="${PROTOC_SKIP_CHECKSUM_VERIFY:-0}" - -usage() { - cat <<'EOF' -Usage: bindings/cpp/scripts/ensure_protoc.sh [--print-path] - -Ensures a protoc binary matching the configured protobuf baseline is available. -Installs into a local cache directory (default: \$XDG_CACHE_HOME/fluss-cpp-tools or -\$HOME/.cache/fluss-cpp-tools) and prints -the protoc path on stdout. - -Env vars: - PROTOBUF_BASELINE_VERSION Baseline protobuf version (default: 3.25.5) - PROTOC_INSTALL_ROOT Local cache root (default: XDG/HOME cache dir) - PROTOC_OS protoc package OS (default: auto-detect host: linux/osx) - PROTOC_ARCH protoc package arch (default: auto-detect host: x86_64/aarch_64) - PROTOC_FORCE_INSTALL 1 to force re-download - PROTOC_ALLOW_INSECURE_DOWNLOAD - 1 to disable TLS verification (not recommended) - PROTOC_SKIP_CHECKSUM_VERIFY - 1 to skip pinned archive checksum verification - BAZEL_PROXY_URL Optional proxy (sets curl/wget proxy envs if present) -EOF -} - -for arg in "$@"; do - case "$arg" in - --print-path) - PROTOC_PRINT_PATH_ONLY=1 - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "Unknown argument: $arg" >&2 - usage >&2 - exit 1 - ;; - esac -done - -setup_proxy_env() { - if [[ -n "${BAZEL_PROXY_URL:-}" ]]; then - export http_proxy="${http_proxy:-$BAZEL_PROXY_URL}" - export https_proxy="${https_proxy:-$BAZEL_PROXY_URL}" - export HTTP_PROXY="${HTTP_PROXY:-$http_proxy}" - export HTTPS_PROXY="${HTTPS_PROXY:-$https_proxy}" - fi -} - -normalize_version_for_protoc_release() { - local v="$1" - # Protobuf release packaging switched from v3.x.y to vX.Y for newer versions. - # For our current agreed baseline (3.25.5), the protoc archive/tag is 25.5. - if [[ "$v" =~ ^3\.([0-9]+\.[0-9]+)$ ]]; then - local stripped="${BASH_REMATCH[1]}" - local major="${stripped%%.*}" - if [[ "$major" -ge 21 ]]; then - echo "$stripped" - return 0 - fi - fi - echo "$v" -} - -version_matches_baseline() { - local actual="$1" - local baseline="$2" - local actual_norm baseline_norm - actual_norm="$(normalize_version_for_protoc_release "$actual")" - baseline_norm="$(normalize_version_for_protoc_release "$baseline")" - [[ "$actual" == "$baseline" || "$actual_norm" == "$baseline_norm" ]] -} - -lookup_protoc_archive_sha256() { - local release_version="$1" - local os="$2" - local arch="$3" - case "${release_version}:${os}:${arch}" in - 25.5:linux:aarch_64) - echo "dc715bb5aab2ebf9653d7d3efbe55e01a035e45c26f391ff6d9b7923e22914b7" - ;; - 25.5:linux:x86_64) - echo "e1ed237a17b2e851cf9662cb5ad02b46e70ff8e060e05984725bc4b4228c6b28" - ;; - 25.5:osx:aarch_64) - echo "781a6fc4c265034872cadc65e63dd3c0fc49245b70917821b60e2d457a6876ab" - ;; - 25.5:osx:x86_64) - echo "c5447e4f0d5caffb18d9ff21eae7bc7faf2bb2000083d6f49e5b6000b30fceae" - ;; - *) - return 1 - ;; - esac -} - -verify_download_sha256() { - local file="$1" - local expected="$2" - local actual="" - if command -v sha256sum >/dev/null 2>&1; then - actual="$(sha256sum "$file" | awk '{print $1}')" - elif command -v shasum >/dev/null 2>&1; then - actual="$(shasum -a 256 "$file" | awk '{print $1}')" - else - echo "ERROR: neither sha256sum nor shasum is available for checksum verification." >&2 - return 1 - fi - if [[ "$actual" != "$expected" ]]; then - echo "ERROR: protoc archive checksum mismatch." >&2 - echo " expected: $expected" >&2 - echo " actual: $actual" >&2 - return 1 - fi -} - -download_file() { - local url="$1" - local out="$2" - - if command -v curl >/dev/null 2>&1; then - local curl_args=(-fL) - if [[ "${PROTOC_ALLOW_INSECURE_DOWNLOAD}" == "1" ]]; then - curl_args+=(-k) - fi - curl "${curl_args[@]}" "$url" -o "$out" - return 0 - fi - - if command -v wget >/dev/null 2>&1; then - local wget_args=() - if [[ -n "${https_proxy:-}" || -n "${http_proxy:-}" ]]; then - wget_args+=(-e use_proxy=yes) - if [[ -n "${https_proxy:-}" ]]; then - wget_args+=(-e "https_proxy=${https_proxy}") - fi - if [[ -n "${http_proxy:-}" ]]; then - wget_args+=(-e "http_proxy=${http_proxy}") - fi - fi - if [[ "${PROTOC_ALLOW_INSECURE_DOWNLOAD}" == "1" ]]; then - wget_args+=(--no-check-certificate) - fi - wget "${wget_args[@]}" -O "$out" "$url" - return 0 - fi - - echo "ERROR: neither curl nor wget is available for downloading protoc." >&2 - return 1 -} - -ensure_zip_tools() { - command -v unzip >/dev/null 2>&1 || { - echo "ERROR: unzip not found." >&2 - exit 1 - } -} - -setup_proxy_env -ensure_zip_tools - -if command -v protoc >/dev/null 2>&1; then - existing_out="$(protoc --version 2>/dev/null || true)" - if [[ "$existing_out" =~ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then - existing_ver="${BASH_REMATCH[1]}" - if version_matches_baseline "$existing_ver" "$PROTOBUF_BASELINE_VERSION"; then - command -v protoc - exit 0 - fi - fi -fi - -PROTOC_RELEASE_VERSION="$(normalize_version_for_protoc_release "$PROTOBUF_BASELINE_VERSION")" -PROTOC_ARCHIVE="protoc-${PROTOC_RELEASE_VERSION}-${PROTOC_OS}-${PROTOC_ARCH}.zip" -PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_RELEASE_VERSION}/${PROTOC_ARCHIVE}" -PROTOC_PREFIX="${PROTOC_INSTALL_ROOT}/protoc-${PROTOC_RELEASE_VERSION}-${PROTOC_OS}-${PROTOC_ARCH}" -PROTOC_BIN="${PROTOC_PREFIX}/bin/protoc" - -if [[ "${PROTOC_FORCE_INSTALL}" != "1" && -x "${PROTOC_BIN}" ]]; then - if [[ "${PROTOC_PRINT_PATH_ONLY}" == "1" ]]; then - echo "${PROTOC_BIN}" - else - echo "${PROTOC_BIN}" - fi - exit 0 -fi - -mkdir -p "${PROTOC_INSTALL_ROOT}" -tmpdir="$(mktemp -d "${PROTOC_INSTALL_ROOT}/.protoc-download.XXXXXX")" -trap 'rm -rf "${tmpdir}"' EXIT - -archive_path="${tmpdir}/${PROTOC_ARCHIVE}" -download_file "${PROTOC_URL}" "${archive_path}" -if [[ "${PROTOC_SKIP_CHECKSUM_VERIFY}" != "1" ]]; then - if expected_sha256="$(lookup_protoc_archive_sha256 "${PROTOC_RELEASE_VERSION}" "${PROTOC_OS}" "${PROTOC_ARCH}")"; then - verify_download_sha256 "${archive_path}" "${expected_sha256}" - else - echo "ERROR: no pinned checksum for protoc archive ${PROTOC_ARCHIVE}. Set PROTOC_SKIP_CHECKSUM_VERIFY=1 to bypass." >&2 - exit 1 - fi -fi - -extract_dir="${tmpdir}/extract" -mkdir -p "${extract_dir}" -unzip -q "${archive_path}" -d "${extract_dir}" - -rm -rf "${PROTOC_PREFIX}" -mkdir -p "${PROTOC_PREFIX}" -cp -a "${extract_dir}/." "${PROTOC_PREFIX}/" -chmod +x "${PROTOC_BIN}" - -echo "${PROTOC_BIN}" diff --git a/fluss-rust/crates/fluss/Cargo.toml b/fluss-rust/crates/fluss/Cargo.toml index bf8ae57df75..1c00f7f0be5 100644 --- a/fluss-rust/crates/fluss/Cargo.toml +++ b/fluss-rust/crates/fluss/Cargo.toml @@ -86,6 +86,3 @@ fluss-test-cluster = { path = "../fluss-test-cluster" } # workspace `full` feature set; enable it for tests so the # `last_poll_seconds_ago` ticker loop can be driven deterministically. tokio = { workspace = true, features = ["test-util"] } - -[build-dependencies] -prost-build = "0.14" diff --git a/fluss-rust/crates/fluss/build.rs b/fluss-rust/crates/fluss/build.rs deleted file mode 100644 index 5fd20dbf628..00000000000 --- a/fluss-rust/crates/fluss/build.rs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::io::Result; -use std::path::Path; - -fn main() -> Result<()> { - let mut config = prost_build::Config::new(); - config.bytes([ - ".fluss.PbProduceLogReqForBucket.records", - ".fluss.PbPutKvReqForBucket.records", - ".fluss.PbLookupReqForBucket.keys", - ".fluss.PbPrefixLookupReqForBucket.keys", - ".fluss.ScanKvResponse.records", - ]); - // Published crates vendor the proto under proto/ (scripts/vendor-proto.sh); - // monorepo builds read the canonical proto directly from fluss-rpc. - let (proto, include_dir) = if Path::new("proto/FlussApi.proto").exists() { - ("proto/FlussApi.proto", "proto") - } else { - ( - "../../../fluss-rpc/src/main/proto/FlussApi.proto", - "../../../fluss-rpc/src/main/proto", - ) - }; - config.compile_protos(&[proto], &[include_dir])?; - Ok(()) -} diff --git a/fluss-rust/crates/fluss/gen/Cargo.toml b/fluss-rust/crates/fluss/gen/Cargo.toml new file mode 100644 index 00000000000..e617881389e --- /dev/null +++ b/fluss-rust/crates/fluss/gen/Cargo.toml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "gen" +description = "Code generation for fluss-rs" +version = "0.1.0" +edition = { workspace = true } +rust-version = { workspace = true } +authors = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +publish = false + +[dependencies] +# Pin the prost-build version so regenerating does not churn the checked-in +# src/proto/fluss.rs. Bump deliberately, then rerun ../regen.sh. +prost-build = "0.14.3" diff --git a/fluss-rust/crates/fluss/gen/src/main.rs b/fluss-rust/crates/fluss/gen/src/main.rs new file mode 100644 index 00000000000..18b6d0e2b88 --- /dev/null +++ b/fluss-rust/crates/fluss/gen/src/main.rs @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::error::Error; +use std::fs; +use std::path::Path; + +const HEADER: &str = "\ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// \"License\"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This file was generated from the canonical FlussApi.proto by +// fluss-rust/crates/fluss/regen.sh, and should not be edited by hand. + +"; + +fn main() -> Result<(), Box> { + let fluss_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .ok_or("the gen crate must live inside crates/fluss")?; + + // The canonical proto lives in fluss-rpc and is shared with the Java server. + // Regeneration always reads it and refreshes the copy vendored under proto/, + // so the published crate ships the schema next to the generated code. + let include_dir = fluss_dir.join("../../../fluss-rpc/src/main/proto"); + let proto = include_dir.join("FlussApi.proto"); + if !proto.exists() { + return Err(format!( + "canonical proto not found at {}; regeneration must run from the Fluss monorepo", + proto.display() + ) + .into()); + } + + let vendored_dir = fluss_dir.join("proto"); + fs::create_dir_all(&vendored_dir)?; + fs::copy(&proto, vendored_dir.join("FlussApi.proto"))?; + + let mut config = prost_build::Config::new(); + config.bytes([ + ".fluss.PbProduceLogReqForBucket.records", + ".fluss.PbPutKvReqForBucket.records", + ".fluss.PbLookupReqForBucket.keys", + ".fluss.PbPrefixLookupReqForBucket.keys", + ".fluss.ScanKvResponse.records", + ]); + let out_dir = fluss_dir.join("src/proto"); + fs::create_dir_all(&out_dir)?; + config.out_dir(&out_dir); + config.compile_protos(&[&proto], &[&include_dir])?; + + let generated_path = out_dir.join("fluss.rs"); + let generated = fs::read_to_string(&generated_path)?; + fs::write(&generated_path, format!("{HEADER}{generated}"))?; + Ok(()) +} diff --git a/fluss-rust/crates/fluss/proto/FlussApi.proto b/fluss-rust/crates/fluss/proto/FlussApi.proto new file mode 100644 index 00000000000..d03acb92fbe --- /dev/null +++ b/fluss-rust/crates/fluss/proto/FlussApi.proto @@ -0,0 +1,1430 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; + +package fluss; +option java_package = "org.apache.fluss.rpc.messages"; +option optimize_for = LITE_RUNTIME; + +/** +* NOTICE: To ensure a smooth upgrade to Proto3 syntax while maintaining backward compatibility in +* the future, please adhere to the following limitations: +* 1. DO NOT USE "default" keyword, proto3 removes the support of "default" +* 2. DO NOT USE "enum" type + */ + +/** +* NOTICE: The Rust client checks in the code generated from this file. After changing it, run +* fluss-rust/crates/fluss/regen.sh and commit the result; the proto-up-to-date job of the Rust +* CI fails otherwise. + */ + +// the message indicates a failed request of a server failure. +message ErrorResponse { + required int32 error_code = 1; + optional string error_message = 2; +} + +message ApiVersionsRequest { + required string client_software_name = 1; + required string client_software_version = 2; +} + +message ApiVersionsResponse { + repeated PbApiVersion api_versions = 1; + optional int32 server_type = 2; +} + +// get schema request and response +message GetTableSchemaRequest { + required PbTablePath table_path = 1; + optional int32 schema_id = 2; +} + +message GetTableSchemaResponse { + required int32 schema_id = 1; + required bytes schema_json = 2; +} + +// create database request and response +message CreateDatabaseRequest { + required string database_name = 1; + required bool ignore_if_exists = 2; + optional bytes database_json = 3; +} + +message CreateDatabaseResponse { +} + +// alter database request and response +message AlterDatabaseRequest { + required string database_name = 1; + required bool ignore_if_not_exists = 2; + repeated PbAlterConfig config_changes = 3; + optional string comment = 4; +} + +message AlterDatabaseResponse { +} + +// get table request and response +message GetDatabaseInfoRequest { + required string database_name = 1; +} + +message GetDatabaseInfoResponse { + required bytes database_json = 3; + required int64 created_time = 4; + required int64 modified_time = 5; +} + +// drop database request and response +message DropDatabaseRequest { + required string database_name = 1; + required bool ignore_if_not_exists = 2; + required bool cascade = 3; +} +message DropDatabaseResponse { +} + +// database exists request and response +message DatabaseExistsRequest { + required string database_name = 1; +} +message DatabaseExistsResponse { + required bool exists = 1; +} + +// list databases request and response +message ListDatabasesRequest { + optional bool include_summary = 1; +} + +message ListDatabasesResponse { + repeated string database_name = 1; + repeated PbDatabaseSummary database_summary = 2; +} + +// create table request and response +message CreateTableRequest { + required PbTablePath table_path = 1; + required bytes table_json = 2; + required bool ignore_if_exists = 3; +} + +message CreateTableResponse { +} + +// alter table request and response +message AlterTableRequest { + required PbTablePath table_path = 1; + required bool ignore_if_not_exists = 2; + repeated PbAlterConfig config_changes = 3; + repeated PbAddColumn add_columns = 4; + repeated PbDropColumn drop_columns = 5; + repeated PbRenameColumn rename_columns = 6; + repeated PbModifyColumn modify_columns = 7; +} + +message AlterTableResponse { +} + + + +// get table request and response +message GetTableInfoRequest { + required PbTablePath table_path = 1; +} + +message GetTableInfoResponse { + required int64 table_id = 1; + required int32 schema_id = 2; + required bytes table_json = 3; + required int64 created_time = 4; + required int64 modified_time = 5; + optional string remote_data_dir = 6; +} + +// list tables request and response +message ListTablesRequest { + required string database_name = 1; +} + +message ListTablesResponse { + repeated string table_name = 1; +} + +// drop table request and response +message DropTableRequest { + required PbTablePath table_path = 1; + required bool ignore_if_not_exists = 2; +} + +message DropTableResponse { +} + +// table exists request and response +message TableExistsRequest { + required PbTablePath table_path = 1; +} + +message TableExistsResponse { + required bool exists = 1; +} + +// metadata request and response, request send from client to each server. +message MetadataRequest { + repeated PbTablePath table_path = 1; + repeated PbPhysicalTablePath partitions_path = 2; + + // note: currently, we assume the partition ids must belong to the table_paths in the + // metadata request + // todo: we won't need the assumption after we introduce metadata cache in server + repeated int64 partitions_id = 3 [packed = true]; +} + +message MetadataResponse { + optional PbServerNode coordinator_server = 1; + repeated PbServerNode tablet_servers = 2; + repeated PbTableMetadata table_metadata = 3; + repeated PbPartitionMetadata partition_metadata = 4; +} + +// update metadata request and response, only send between server. +message UpdateMetadataRequest { + optional PbServerNode coordinator_server = 1; + repeated PbServerNode tablet_servers = 2; + repeated PbTableMetadata table_metadata = 3; + repeated PbPartitionMetadata partition_metadata = 4; + optional int32 coordinator_epoch = 5; +} + +message UpdateMetadataResponse { +} + +// produce log request and response +message ProduceLogRequest { + required int32 acks = 1; + required int64 table_id = 2; + required int32 timeout_ms = 3; + repeated PbProduceLogReqForBucket buckets_req = 4; +} + +message ProduceLogResponse { + repeated PbProduceLogRespForBucket buckets_resp = 1; +} + +// fetch log request and response +message FetchLogRequest { + required int32 follower_server_id = 1; // value -1 indicate the request from client. + required int32 max_bytes = 2; + repeated PbFetchLogReqForTable tables_req = 3; + optional int32 max_wait_ms = 4; + optional int32 min_bytes = 5; + optional int32 read_preference = 6; +} + +message FetchLogResponse { + repeated PbFetchLogRespForTable tables_resp = 1; +} + +// Aggregation mode constants for write operations (using int32 instead of enum for proto3 compatibility) +// AGG_MODE_AGGREGATE = 0: Data is aggregated through server-side merge engine (default) +// AGG_MODE_OVERWRITE = 1: Bypass merge engine, directly replace values (for undo recovery) +// AGG_MODE_LOCAL_AGGREGATE = 2: Client-side local aggregation (reserved for future implementation) + +// put kv request and response +message PutKvRequest { + required int32 acks = 1; + required int64 table_id = 2; + required int32 timeout_ms = 3; + // the indexes for the columns to write, + // if empty, means write all columns + repeated int32 target_columns = 4 [packed = true]; + repeated PbPutKvReqForBucket buckets_req = 5; + // Aggregation mode for this request (see AGG_MODE_* constants above) + // 0 = AGGREGATE (default), 1 = OVERWRITE, 2 = LOCAL_AGGREGATE (not yet supported) + optional int32 agg_mode = 6; +} + +message PutKvResponse { + repeated PbPutKvRespForBucket buckets_resp = 1; +} + +// lookup request and response +message LookupRequest { + required int64 table_id = 1; + repeated PbLookupReqForBucket buckets_req = 2; + optional bool insert_if_not_exists = 3; + optional int32 acks = 4; + optional int32 timeout_ms = 5; +} + +message LookupResponse { + repeated PbLookupRespForBucket buckets_resp = 1; +} + +// Prefix Lookup request and response +message PrefixLookupRequest { + required int64 table_id = 1; + repeated PbPrefixLookupReqForBucket buckets_req = 2; +} + +message PrefixLookupResponse { + repeated PbPrefixLookupRespForBucket buckets_resp = 1; +} + + +// limit scan request and response +message LimitScanRequest { + required int64 table_id = 2; + optional int64 partition_id = 3; + required int32 bucket_id = 4; + required int32 limit = 5; +} + +message LimitScanResponse{ + optional int32 error_code = 1; + optional string error_message = 2; + // flag to indicate the table type + optional bool is_log_table = 3; + // LogRecordBatch if is_log_table is true, otherwise KvRecordBatch + optional bytes records = 4; +} + + +// Full KV scan request and response. +// A new scan is initiated with bucket_scan_req; subsequent batches use scanner_id. +message PbScanReqForBucket { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + // If set, stops returning rows after this many records. + optional int64 limit = 4; +} + +message ScanKvRequest { + // Mutually exclusive: either scanner_id (continuation) or bucket_scan_req (new scan). + optional bytes scanner_id = 1; + optional PbScanReqForBucket bucket_scan_req = 2; + // Monotonically increasing sequence number for in-order delivery validation. + optional int32 call_seq_id = 3; + // Maximum number of bytes of record data to return in this batch. + optional int32 batch_size_bytes = 4; + // If true, the server closes the scanner session immediately. + optional bool close_scanner = 5; +} + +message ScanKvResponse { + optional int32 error_code = 1; + optional string error_message = 2; + // Opaque server-assigned session identifier; echoed on every response. + optional bytes scanner_id = 3; + // False when the scan is complete and the scanner session has been closed. + optional bool has_more_results = 4; + // Serialised DefaultValueRecordBatch; absent when the batch is empty. + optional bytes records = 5; + // Log high-watermark at the time the RocksDB snapshot was opened. + // Only set on the first response (new scan initiation). + optional int64 log_offset = 6; +} + + +// Get table statistics request and response. +// Sent to TabletServer to get per-bucket statistics. +message GetTableStatsRequest { + required int64 table_id = 1; + repeated PbTableStatsReqForBucket buckets_req = 2; + // Column indexes for column-level stats (e.g., ndv, sum). + // If empty, column stats are not collected. + // Reserved for future use. + repeated int32 target_columns = 3 [packed = true]; +} + +message GetTableStatsResponse { + repeated PbTableStatsRespForBucket buckets_resp = 1; +} + +// notify bucket leader and isr request +message NotifyLeaderAndIsrRequest { + required int32 coordinator_epoch = 1; + repeated PbNotifyLeaderAndIsrReqForBucket notify_buckets_leader_req = 2; +} + +// response for notify bucket leader and isr request +message NotifyLeaderAndIsrResponse { + repeated PbNotifyLeaderAndIsrRespForBucket notify_buckets_leader_resp = 1; +} + +// stop bucket replica request +message StopReplicaRequest { + required int32 coordinator_epoch = 1; + repeated PbStopReplicaReqForBucket stop_replicas_req = 2; +} + +// response for stop bucket replica request +message StopReplicaResponse { + repeated PbStopReplicaRespForBucket stop_replicas_resp = 1; +} + +// adjust isr request and response +message AdjustIsrRequest { + required int32 serverId = 1; + repeated PbAdjustIsrReqForTable tables_req = 2; +} + +message AdjustIsrResponse { + repeated PbAdjustIsrRespForTable tables_resp = 1; +} + +// list offsets request and response +message ListOffsetsRequest { + required int32 follower_server_id = 1; // value -1 indicate the request from client. + required int32 offset_type = 2; // value can be 0,1,2 (see ListOffsetsParam for more details) + required int64 table_id = 3; + optional int64 partition_id = 4; + repeated int32 bucket_id = 5 [packed = true]; // it is recommended to use packed for repeated numerics to get more efficient encoding + optional int64 startTimestamp = 6; +} +message ListOffsetsResponse { + repeated PbListOffsetsRespForBucket buckets_resp = 1; +} + +// commit kv snapshot request and response +message CommitKvSnapshotRequest { + required bytes completed_snapshot = 1; + required int32 coordinator_epoch = 2; + required int32 bucket_leader_epoch = 3; +} + +message CommitKvSnapshotResponse { +} + +// notify the log offset about kv snapshot +message NotifyKvSnapshotOffsetRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int32 coordinator_epoch = 4; + required int64 min_retain_offset = 5; +} + +message NotifyKvSnapshotOffsetResponse { +} + +message GetLatestKvSnapshotsRequest { + required PbTablePath table_path = 1; + optional string partition_name = 2; +} + +message GetLatestKvSnapshotsResponse { + required int64 table_id = 1; + // null if it is a non-partitioned table, otherwise, it must be not null + optional int64 partition_id = 2; + repeated PbKvSnapshot latest_snapshots = 3; +} + +message GetKvSnapshotMetadataRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int64 snapshot_id = 4; +} + +message GetKvSnapshotMetadataResponse { + required int64 log_offset = 1; + repeated PbRemotePathAndLocalFile snapshot_files = 2; +} + +message AcquireKvSnapshotLeaseRequest { + required string lease_id = 1; + required int64 lease_duration_ms = 2; + repeated PbKvSnapshotLeaseForTable snapshots_to_lease = 3; +} + +message AcquireKvSnapshotLeaseResponse { + repeated PbKvSnapshotLeaseForTable unavailable_snapshots = 1; +} + +message ReleaseKvSnapshotLeaseRequest { + required string lease_id = 1; + repeated PbTableBucket buckets_to_release = 2; +} + +message ReleaseKvSnapshotLeaseResponse {} + +message DropKvSnapshotLeaseRequest { + required string lease_id = 1; +} + +message DropKvSnapshotLeaseResponse {} + +message GetLakeSnapshotRequest { + required PbTablePath table_path = 1; + optional int64 snapshot_id = 2; // If not set, returns the latest snapshot + optional bool readable = 3; // If true, returns the latest readable snapshot +} + +message GetLakeSnapshotResponse { + required int64 table_id = 1; + required int64 snapshotId = 2; + repeated PbLakeSnapshotForBucket bucket_snapshots = 3; +} + +message GetFileSystemSecurityTokenRequest { +} + +message GetFileSystemSecurityTokenResponse { + required string schema = 1; + required bytes token = 2; + optional int64 expiration_time = 3; + repeated PbKeyValue addition_info = 4; +} + +// init writer request and response +message InitWriterRequest { + repeated PbTablePath table_path = 1; +} + +message InitWriterResponse { + required int64 writer_id = 1; +} + +message ListPartitionInfosRequest { + required PbTablePath table_path = 1; + optional PbPartitionSpec partial_partition_spec = 2; +} + +message ListPartitionInfosResponse { + repeated PbPartitionInfo partitions_info = 1; +} + +// list remote log manifest entries (one per bucket of a table or partition) +message ListRemoteLogManifestsRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; // required if table is partitioned +} + +message ListRemoteLogManifestsResponse { + repeated PbRemoteLogManifestEntry manifests = 1; +} + +message PbRemoteLogManifestEntry { + required PbTableBucket table_bucket = 1; + required string remote_log_manifest_path = 2; + required int64 remote_log_end_offset = 3; +} + +// list active KV snapshot dirs (retained_N + still-in-use) for a unit +message ListKvSnapshotsRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; +} + +message ListKvSnapshotsResponse { + required int64 table_id = 1; + // null if it is a non-partitioned table, otherwise, it must be not null + optional int64 partition_id = 2; + // Active snapshots = retained_N ∪ still-in-use; multiple entries per bucket allowed. + repeated PbKvSnapshot active_snapshots = 3; +} + +// create partition request and response +message CreatePartitionRequest { + required PbTablePath table_path = 1; + required PbPartitionSpec partition_spec = 2; + required bool ignore_if_not_exists = 3; +} + +message CreatePartitionResponse {} + +// drop partition request and response +message DropPartitionRequest { + required PbTablePath table_path = 1; + required PbPartitionSpec partition_spec = 2; + required bool ignore_if_not_exists = 3; +} + +message DropPartitionResponse {} + +// commit remote log manifest request and response +message CommitRemoteLogManifestRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required string remote_log_manifest_path = 4; + required int64 remote_log_start_offset = 5; + required int64 remote_log_end_offset = 6; + required int32 coordinator_epoch = 7; + required int32 bucket_leader_epoch = 8; + optional int64 highest_copied_end_offset = 9; +} + +message CommitRemoteLogManifestResponse { + required bool commitSuccess = 1; +} + +// notify remote log offsets request and response +message NotifyRemoteLogOffsetsRequest { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int32 coordinator_epoch = 4; + required int64 remote_start_offset = 5; + required int64 remote_end_offset = 6; + optional int64 highest_copied_end_offset = 7; +} + +message NotifyRemoteLogOffsetsResponse { +} + +message PrepareLakeTableSnapshotRequest { + repeated PbTableOffsets bucket_offsets = 1; + // If true, ignore previous table offsets and do not merge with existing lake table offsets. + // If false (default), the system will merge the provided bucket offsets with the previous + // lake table offsets to ensure continuity. This is useful when you want to replace all offsets + // instead of merging with existing ones. + optional bool ignore_previous_table_offsets = 2; +} + +message PrepareLakeTableSnapshotResponse { + repeated PbPrepareLakeTableRespForTable prepare_lake_table_resp = 1; +} + +message CommitLakeTableSnapshotRequest { + // Deprecated: PbLakeTableSnapshotInfo is no longer used for committing lake table snapshots. + // Currently, it is only used to allow the coordinator to notify tablet servers about the current + // synchronized log end offsets, which are then reported to metrics. In the future, we plan to + // have tiering directly report to metrics, and this field will be fully removed. + // Still reserve it for protocol compatibility. + repeated PbLakeTableSnapshotInfo tables_req = 1; + // The metadata for lake table snapshots to be committed. Each entry contains the table ID, + // snapshot ID, and the file paths where the snapshot data (containing bucket log end offset + // information) is stored. The tiered_snapshot_file_path points to the file storing tiered log + // end offsets, while readable_snapshot_file_path (if present) points to the file storing readable + // log end offsets. + repeated PbLakeTableSnapshotMetadata lake_table_snapshot_metadata = 2; +} + +message CommitLakeTableSnapshotResponse { + repeated PbCommitLakeTableSnapshotRespForTable table_resp = 1; +} + +message PbCommitLakeTableSnapshotRespForTable { + optional int32 error_code = 1; + optional string error_message = 2; + required int64 table_id = 3; +} + +message NotifyLakeTableOffsetRequest { + required int32 coordinator_epoch = 1; + repeated PbNotifyLakeTableOffsetReqForBucket notify_buckets_req = 2; +} + +message PbNotifyLakeTableOffsetReqForBucket { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; + required int64 snapshot_id = 4; + optional int64 log_start_offset = 5; + optional int64 log_end_offset = 6; + optional int64 max_timestamp = 7; +} + +message NotifyLakeTableOffsetResponse { +} + +message AuthenticateRequest { + required string protocol = 1; + required bytes token = 2; +} + +message AuthenticateResponse { + optional bytes challenge = 1; +} + +// acl related requests and responses +message ListAclsRequest{ + required PbAclFilter acl_filter = 1; +} + +message ListAclsResponse{ + repeated PbAclInfo acl = 1; +} + +message CreateAclsRequest{ + repeated PbAclInfo acl = 1; +} + +message CreateAclsResponse{ + repeated PbCreateAclRespInfo aclRes = 1; +} + +message DropAclsRequest{ + repeated PbAclFilter acl_filter = 1; +} + +message DropAclsResponse{ + repeated PbDropAclsFilterResult filter_results = 1; +} + +message LakeTieringHeartbeatRequest { + repeated PbHeartbeatReqForTable tiering_tables = 1; + repeated PbHeartbeatReqForTable finished_tables = 2; + repeated PbHeartbeatReqForTable failed_tables = 3; + // whether to request a table + optional bool request_table = 4; + // Table IDs that were force finished due to reaching the maximum tiering duration. + // When a table's tiering operation exceeds the max duration (data lake freshness), + // it will be force finished to prevent it from blocking other tables' tiering operations. + // These table IDs must also appear in finished_tables. The coordinator uses this field to + // distinguish between normally finished tables and force finished tables. + repeated int64 force_finished_tables = 5; +} + +message LakeTieringHeartbeatResponse { + // coordinator epoch + required int32 coordinator_epoch = 1; + // the returned table to tier, empty when no table is needed by the lake tiering service + optional PbLakeTieringTableInfo tiering_table = 2; + repeated PbHeartbeatRespForTable tiering_table_resp = 3; + repeated PbHeartbeatRespForTable finished_table_resp = 4; + repeated PbHeartbeatRespForTable failed_table_resp = 5; +} + +message ControlledShutdownRequest { + required int32 tablet_server_id = 1; + required int32 tablet_server_epoch = 2; +} + +message ControlledShutdownResponse { + repeated PbTableBucket remaining_leader_buckets = 1; +} + +message DescribeClusterConfigsRequest{ +} + + +message DescribeClusterConfigsResponse{ + repeated PbDescribeConfig configs = 1; +} + +message AlterClusterConfigsRequest{ + repeated PbAlterConfig alter_configs = 1; +} + +message AlterClusterConfigsResponse{ +} + +message AddServerTagRequest { + repeated int32 server_ids = 1 [packed = true]; + required int32 server_tag = 2; +} + +message AddServerTagResponse { +} + +message RemoveServerTagRequest { + repeated int32 server_ids = 1 [packed = true]; + required int32 server_tag = 2; +} + +message RemoveServerTagResponse { +} + +message RebalanceRequest { + repeated int32 goals = 1 [packed = true]; +} + +message RebalanceResponse { + required string rebalance_id = 1; +} + +message ListRebalanceProgressRequest { + optional string rebalance_id = 1; +} + +message ListRebalanceProgressResponse { + optional string rebalance_id = 1; + optional int32 rebalance_status = 2; + repeated PbRebalanceProgressForTable table_progress = 3; +} + +message CancelRebalanceRequest { + optional string rebalance_id = 1; +} + +message CancelRebalanceResponse { +} + +// ------------------------------------------------------------------------------------------ +// Producer Offset Management +// ------------------------------------------------------------------------------------------ + +// Register producer offsets request and response +message RegisterProducerOffsetsRequest { + required string producer_id = 1; + repeated PbProducerTableOffsets table_offsets = 2; + optional int64 ttl_ms = 3; +} + +message RegisterProducerOffsetsResponse { + // Result of registration: 0 = CREATED (new snapshot), 1 = ALREADY_EXISTS (snapshot existed) + optional int32 result = 1; +} + +// Get producer offsets request and response +message GetProducerOffsetsRequest { + required string producer_id = 1; +} + +message GetProducerOffsetsResponse { + optional string producer_id = 1; + optional int64 expiration_time = 2; + repeated PbProducerTableOffsets table_offsets = 3; +} + +// Delete producer offsets request and response +message DeleteProducerOffsetsRequest { + required string producer_id = 1; +} + +message DeleteProducerOffsetsResponse { +} + +message GetClusterHealthRequest { } + +message GetClusterHealthResponse { + required int32 num_replicas = 1; + required int32 in_sync_replicas = 2; + required int32 num_leader_replicas = 3; + required int32 active_leader_replicas = 4; + required int32 status = 5; // PbClusterHealthStatus: GREEN=0, YELLOW=1, RED=2, UNKNOWN=3 +} + + +// --------------- Inner classes ---------------- +message PbApiVersion { + required int32 api_key = 1; + required int32 min_version = 2; + required int32 max_version = 3; +} + +message PbTablePath { + required string database_name = 1; + required string table_name = 2; +} + +message PbPhysicalTablePath { + required string database_name = 1; + required string table_name = 2; + optional string partition_name = 3; +} + +// For MetadataResponse, host and port are still used for all versions. +// For UpdateMetadataRequest, +// * versions <= 0.6: host and port are used. +// * versions >= 0.7: listeners is used to replace host and port. +// For MetadataResponse and UpdateMetadataRequest: Fluss versions >= 0.7: we add rack for each tabletServer +message PbServerNode { + required int32 node_id = 1; + required string host = 2; + required int32 port = 3; + optional string listeners = 4; + optional string rack = 5; +} + +message PbTableMetadata { + required PbTablePath table_path = 1; + required int64 table_id = 2; + required int32 schema_id = 3; + // TODO Starting from version 0.9, fluss-client no longer get table info from metadata response. However, for backward + // compatibility, we will still retain the field in the request. Once metadataV2 is introduced, table info will be + // officially omitted from the new metadata response. + required bytes table_json = 4; + repeated PbBucketMetadata bucket_metadata = 5; + required int64 created_time = 6; + required int64 modified_time = 7; + optional string remote_data_dir = 8; + + // TODO add a new filed 'deleted_table' to indicate this table is deleted in UpdateMetadataRequest. + // trace by: https://github.com/apache/fluss/issues/981 +} + +message PbPartitionMetadata { + required int64 table_id = 1; + // the partition name and id for the partition + required string partition_name = 2; + required int64 partition_id = 3; + repeated PbBucketMetadata bucket_metadata = 4; +} + +message PbBucketMetadata { + required int32 bucket_id = 1; + // optional as some time the leader may not elected yet + optional int32 leader_id = 2; + repeated int32 replica_id = 3 [packed = true]; + // TODO: Add isr here. + optional int32 leader_epoch = 4; +} + +message PbProduceLogReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required bytes records = 3; +} + +message PbProduceLogRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + optional int64 base_offset = 5; +} + +message PbFetchLogReqForTable { + required int64 table_id = 1; + required bool projection_pushdown_enabled = 2; + repeated int32 projected_fields = 3 [packed = true]; + repeated PbFetchLogReqForBucket buckets_req = 4; + // filter_predicate and filter_schema_id MUST be set together. + // filter_predicate is the serialized predicate; filter_schema_id identifies + // the schema against which the predicate was compiled. + optional PbPredicate filter_predicate = 5; + optional int32 filter_schema_id = 6; +} + +message PbFetchLogReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + // TODO leader epoch + required int64 fetch_offset = 3; + required int32 max_fetch_bytes = 4; +} + +message PbFetchLogRespForTable { + required int64 table_id = 1; + repeated PbFetchLogRespForBucket buckets_resp = 2; +} +message PbFetchLogRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + optional int64 high_watermark = 5; + optional int64 log_start_offset = 6; // TODO now we don't introduce log start offset, but remain it in protobuf + optional PbRemoteLogFetchInfo remote_log_fetch_info = 7; + optional bytes records = 8; + // The offset the server scanned up to while filtering out batches, which may accompany a + // non-empty records field when only trailing batches were filtered. The client should start + // its next fetch from the later of this offset and the end of the records it received. + optional int64 filtered_end_offset = 9; +} + +message PbPutKvReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required bytes records = 3; +} + +message PbPutKvRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + // the log end offset (LEO) of the changelog after this write + // this is the offset of the next record to be written, used for exactly-once semantics + optional int64 log_end_offset = 5; + // backpressure signal from RocksDB write stall detection: + // 0 = normal (no pressure), (0,1) = DELAYED zone (normalized L0 ratio) + // introduced in v1.0 + optional float pressure = 6; +} + +message PbLookupReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + repeated bytes keys = 3; + // The original partition name for historical lookup. It is unset for normal lookup. + optional string original_partition_name = 4; +} + +message PbLookupRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + repeated PbValue values = 5; + // The original partition name for historical lookup. It is unset for normal lookup. + optional string original_partition_name = 6; +} + +message PbValue { + // optional, if empty, means no value + optional bytes values = 1; +} + +message PbValueList { + repeated bytes values = 1; +} + +message PbPrefixLookupReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + repeated bytes keys = 3; +} + +message PbPrefixLookupRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + repeated PbValueList value_lists = 5; +} + +message PbTableBucket { + required int64 table_id = 1; + optional int64 partition_id = 2; + required int32 bucket_id = 3; +} + +message PbAdjustIsrReqForTable { + required int64 table_id = 1; + repeated PbAdjustIsrReqForBucket buckets_req = 2; +} + + +message PbAdjustIsrReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required int32 leader_epoch = 3; + repeated int32 new_isr = 4 [packed = true]; + required int32 coordinator_epoch = 5; + required int32 bucket_epoch = 6; + repeated int32 standby_replicas = 7 [packed = true]; +} + +message PbAdjustIsrRespForTable { + required int64 table_id = 1; + repeated PbAdjustIsrRespForBucket buckets_resp = 2; +} + +message PbAdjustIsrRespForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 error_code = 3; + optional string error_message = 4; + optional int32 leader_id = 5; + optional int32 leader_epoch = 6; + repeated int32 isr = 7 [packed = true]; + optional int32 bucket_epoch = 8; + optional int32 coordinator_epoch = 9; + repeated int32 standby_replicas = 10 [packed = true]; +} + +message PbListOffsetsRespForBucket { + required int32 bucket_id = 1; + optional int32 error_code = 2; + optional string error_message = 3; + optional int64 offset = 4; +} + +message PbNotifyLeaderAndIsrReqForBucket { + // we need table path for tablet server to create data dir for the bucket, + // although tablet server don't need it after it has created data dir. + // For simplicity, we always pass table path. + required PbPhysicalTablePath physical_table_path = 1; + required PbTableBucket table_bucket = 2; + required int32 leader = 3; + required int32 leader_epoch = 4; + repeated int32 replicas = 5 [packed = true]; + repeated int32 isr = 6 [packed = true]; + required int32 bucket_epoch = 7; + repeated int32 standby_replicas = 8 [packed = true]; +} + +message PbNotifyLeaderAndIsrRespForBucket { + required PbTableBucket table_bucket = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbStopReplicaReqForBucket { + required PbTableBucket table_bucket = 1; + required int32 leader_epoch = 2; + // delete means remove local replica data (i.e., data stored on the current node). + required bool delete = 3; + // deleteRemote means remove remote replica data (e.g., data in object storage) and was introduced in v0.9. + // For backward compatibility, if a request does not include the deleteRemote flag, the system treats delete as + // deleteRemote (i.e., it falls back to remote deletion). This ensures older CoordinatorServer continues to function + // correctly with newer TabletServers. + optional bool deleteRemote = 4; +} + +message PbStopReplicaRespForBucket { + required PbTableBucket table_bucket = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbKvSnapshot { + required int32 bucket_id = 1; + // null if there is no snapshot for this bucket + optional int64 snapshot_id = 2; + // null if there is no snapshot for this bucket, then read from EARLIEST + optional int64 log_offset = 3; +} + +message PbLakeSnapshotForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int64 log_offset = 3; + optional string partition_name = 4; +} + +message PbRemotePathAndLocalFile { + required string remote_path = 1; + required string local_file_name = 2; +} + +message PbKeyValue { + required string key = 1; + required string value = 2; +} + +message PbRemoteLogFetchInfo { + required string remote_log_tablet_dir = 1; + optional string partition_name = 2; + repeated PbRemoteLogSegment remote_log_segments = 3; + optional int32 first_start_pos = 4; +} + +message PbRemoteLogSegment { + required string remote_log_segment_id = 1; + required int64 remote_log_start_offset = 2; + required int64 remote_log_end_offset = 3; + required int32 segment_size_in_bytes = 4; + optional int64 max_timestamp = 5; +} + +message PbPartitionInfo { + required int64 partition_id = 1; + required PbPartitionSpec partition_spec = 2; + optional string remote_data_dir = 3; +} + +message PbPartitionSpec { + repeated PbKeyValue partition_key_values = 1; +} + +message PbCreateAclRespInfo { + required PbAclInfo acl = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbAclInfo { + required string resource_name = 1; + required int32 resource_type = 2; + required string principal_name = 3; + required string principal_type = 4; + required string host = 5; + required int32 operation_type = 6; + required int32 permission_type = 7; +} + +message PbAclFilter { + optional string resource_name = 1; + required int32 resource_type = 2; + optional string principal_name = 3; + optional string principal_type = 4; + optional string host = 5; + required int32 operation_type = 6; + required int32 permission_type = 7; +} + +message PbDropAclsFilterResult { + repeated PbDropAclsMatchingAcl matching_acls = 1; + optional int32 error_code = 2; + optional string error_message = 3; + +} + +message PbDropAclsMatchingAcl { + required PbAclInfo acl = 1; + optional int32 error_code = 2; + optional string error_message = 3; +} + +message PbLakeTieringTableInfo { + required int64 table_id = 1; + required PbTablePath table_path = 2; + required int64 tiering_epoch = 3; +} + +message PbHeartbeatReqForTable { + required int64 table_id = 1; + // the coordinator epoch when the table is assigned to be tiering service + required int32 coordinator_epoch = 2; + // the tiering epoch when the table is assigned to be tiering service + required int64 tiering_epoch = 3; + // stats for the last tiering round; only set for finished_tables in LakeTieringHeartbeatRequest + optional PbLakeTieringStats lake_tiering_stats = 4; +} + +// Stats reported by the tiering service when a table finishes a tiering round. +message PbLakeTieringStats { + // cumulative total file size (bytes) of the lake table after tiering + optional int64 file_size = 1; + // cumulative total record count of the lake table after tiering + optional int64 record_count = 2; +} + +message PbHeartbeatRespForTable { + required int64 table_id = 1; + optional ErrorResponse error = 2; +} + +message PbAlterConfig { + required string config_key = 1; + optional string config_value = 2; + required int32 op_type = 3; // SET=0, DELETE=1, APPEND=2, SUBTRACT=3 +} + +message PbAddColumn { + required string column_name = 1; + required bytes data_type_json = 2; + optional string comment = 3; + required int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3 + optional string agg_function_type = 5; + repeated PbKeyValue agg_function_params = 6; +} + +message PbDropColumn { + required string column_name = 1; +} + +message PbRenameColumn { + required string old_column_name = 1; + required string new_column_name = 2; +} + +message PbModifyColumn { + required string column_name = 1; + optional bytes data_type_json = 2; + optional string comment = 3; + optional int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3 +} + +message PbDescribeConfig { + required string config_key = 1; + optional string config_value = 2; + required string config_source = 3; +} + +message PbRebalancePlanForTable { + required int64 table_id = 1; + repeated PbRebalancePlanForBucket buckets_plan = 2; +} + +message PbRebalanceProgressForTable { + required int64 table_id = 1; + repeated PbRebalanceProgressForBucket buckets_progress = 2; +} + +message PbRebalanceProgressForBucket { + required PbRebalancePlanForBucket rebalance_plan = 1; + required int32 rebalance_status = 2; +} + +message PbRebalancePlanForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int32 original_leader = 3; + optional int32 new_leader = 4; + repeated int32 original_replicas = 5 [packed = true]; + repeated int32 new_replicas = 6 [packed = true]; +} + +message PbLakeTableSnapshotMetadata { + required int64 table_id = 1; + required int64 snapshot_id = 2; + required string tiered_bucket_offsets_file_path = 3; + optional string readable_bucket_offsets_file_path = 4; + // Snapshot retention policy: The earliest snapshot ID to retain. + // 1. If set, the system will keep all snapshots in the range [earliest_snapshot_id_to_keep, current_snapshot_id]. + // 2. If not set, the system defaults to a "Single Snapshot Retention" policy, keeping only the snapshot specified in this request. + optional int64 earliest_snapshot_id_to_keep = 5; +} + +message PbLakeTableSnapshotInfo { + optional int64 table_id = 1; + required int64 snapshot_id = 2; + repeated PbLakeTableOffsetForBucket buckets_req = 3; + // add table path to reduce get table_path by table id + optional PbTablePath table_path = 4; +} + +message PbLakeTableOffsetForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + // Deprecated: log_start_offset is no longer used. Field number 3 is reserved for protocol compatibility. + // optional int64 log_start_offset = 3; + optional int64 log_end_offset = 4; + // Deprecated: partition_name is no longer used. Field number 5 is reserved for protocol compatibility. + // optional string partition_name = 5; + optional int64 max_timestamp = 6; +} + +message PbPrepareLakeTableRespForTable { + optional int32 error_code = 1; + optional string error_message = 2; + optional int64 table_id = 3; + optional string lake_table_offsets_path = 4; +} + +message PbTableOffsets { + required int64 table_id = 1; + required PbTablePath table_path = 2; + repeated PbBucketOffset bucket_offsets = 3; +} + +message PbBucketOffset { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + optional int64 log_end_offset = 4; +} + +// Helper message for table offsets in producer offsets (without table_path) +message PbProducerTableOffsets { + required int64 table_id = 1; + repeated PbBucketOffset bucket_offsets = 2; +} + +message PbDatabaseSummary { + required string database_name = 1; + required int64 created_time = 2; + required int32 table_count = 3; +} + +message PbKvSnapshotLeaseForTable { + required int64 table_id = 1; + repeated PbKvSnapshotLeaseForBucket bucket_snapshots = 2; +} + +message PbKvSnapshotLeaseForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; + required int64 snapshot_id = 3; +} + +message PbTableStatsReqForBucket { + optional int64 partition_id = 1; + required int32 bucket_id = 2; +} + +message PbTableStatsRespForBucket { + optional int32 error_code = 1; + optional string error_message = 2; + optional int64 partition_id = 3; + required int32 bucket_id = 4; + + // --- Table-level stats --- + // The number of rows in this bucket. + // For KV tables: the number of unique keys (live rows). + // For Log tables: total number of log records (highWatermark - logStartOffset). + // Absent if row count is not available (e.g., WAL changelog mode or legacy tables). + optional int64 row_count = 5; + + // --- data size stats (future) --- + // The data size in bytes of this bucket. + // For KV tables: the size of the KV store. + // For Log tables: the size of the log segments. + // Reserved for future use. + // optional int64 data_size_bytes = 6; + + // --- Column-level stats (future) --- + // Per-column statistics, keyed by column index. + // Only populated when target_columns is specified in the request. + // repeated PbColumnStats column_stats = 10; +} + +/** +* Definition of predicate system related message +*/ + +// PbPredicateType constants (using int32 instead of enum for proto3 compatibility) +// LEAF = 0, COMPOUND = 1 + +// Represents a predicate that can be serialized and transmitted across +// languages +message PbPredicate { + // See PbPredicateType constants: LEAF = 0, COMPOUND = 1 + required int32 type = 1; + optional PbLeafPredicate leaf = 2; + optional PbCompoundPredicate compound = 3; +} + +// Represents a leaf predicate that compares a field with literals +message PbLeafPredicate { + // The function to apply (see PbLeafFunction constants: EQUAL=0 ... NOT_IN=12) + required int32 function = 1; + // The schema field id of the referenced top-level field + required int32 field_id = 2; + // The literals to compare with + repeated PbLiteralValue literals = 3; +} + +// Represents a compound predicate that combines multiple predicates +message PbCompoundPredicate { + // The function to apply (see PbCompoundFunction constants: AND=0, OR=1) + required int32 function = 1; + // The child predicates + repeated PbPredicate children = 2; +} + +// PbLeafFunction constants (using int32 instead of enum for proto3 compatibility) +// EQUAL=0, NOT_EQUAL=1, LESS_THAN=2, LESS_OR_EQUAL=3, GREATER_THAN=4, +// GREATER_OR_EQUAL=5, IS_NULL=6, IS_NOT_NULL=7, STARTS_WITH=8, CONTAINS=9, +// END_WITH=10, IN=11, NOT_IN=12 + +// PbCompoundFunction constants (using int32 instead of enum for proto3 compatibility) +// AND=0, OR=1 + +// PbDataTypeRoot constants (using int32 instead of enum for proto3 compatibility) +// BOOLEAN=0, TINYINT=1, SMALLINT=2, INT=3, BIGINT=4, FLOAT=5, DOUBLE=6, +// CHAR=7, VARCHAR=8, DECIMAL=9, DATE=10, TIME_WITHOUT_TIME_ZONE=11, +// TIMESTAMP_WITHOUT_TIME_ZONE=12, TIMESTAMP_WITH_LOCAL_TIME_ZONE=13, +// BINARY=14, BYTES=15 + +// Represents a literal value +message PbLiteralValue { + // See PbDataTypeRoot constants: indicates which value field is populated + required int32 literal_type = 1; + required bool is_null = 2; + optional bool boolean_value = 3; + optional int32 int_value = 4; + optional int64 bigint_value = 5; + optional float float_value = 6; + optional double double_value = 7; + optional string string_value = 8; + optional bytes binary_value = 9; + optional int64 decimal_value = 10; // Serialized decimal (compact mode) + optional int64 timestamp_millis_value = 11; // Epoch millis + optional int32 timestamp_nano_of_millis_value = 12; // Nano of millis + optional bytes decimal_bytes = 13; // Serialized decimal (non-compact mode) +} diff --git a/fluss-rust/crates/fluss/regen.sh b/fluss-rust/crates/fluss/regen.sh new file mode 100755 index 00000000000..862de85cedd --- /dev/null +++ b/fluss-rust/crates/fluss/regen.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Regenerates src/proto/fluss.rs and the vendored proto/FlussApi.proto copy +# from the canonical fluss-rpc/src/main/proto/FlussApi.proto. +# Requires protoc on PATH (or the PROTOC env var pointing at a protoc binary). + +set -euo pipefail + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd "$SCRIPT_DIR" && cargo run --manifest-path gen/Cargo.toml diff --git a/fluss-rust/crates/fluss/src/lib.rs b/fluss-rust/crates/fluss/src/lib.rs index 6bfbe3cb48d..e9b0995af4c 100644 --- a/fluss-rust/crates/fluss/src/lib.rs +++ b/fluss-rust/crates/fluss/src/lib.rs @@ -153,5 +153,5 @@ pub(crate) mod proto { // caller yet, and the generated doc comments aren't clippy-clean. #![allow(dead_code)] #![allow(clippy::doc_lazy_continuation)] - include!(concat!(env!("OUT_DIR"), "/fluss.rs")); + include!("proto/fluss.rs"); } diff --git a/fluss-rust/crates/fluss/src/proto/fluss.rs b/fluss-rust/crates/fluss/src/proto/fluss.rs new file mode 100644 index 00000000000..6fe822a1af5 --- /dev/null +++ b/fluss-rust/crates/fluss/src/proto/fluss.rs @@ -0,0 +1,1935 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// This file was generated from the canonical FlussApi.proto by +// fluss-rust/crates/fluss/regen.sh, and should not be edited by hand. + +// This file is @generated by prost-build. +/// the message indicates a failed request of a server failure. +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ErrorResponse { + #[prost(int32, required, tag = "1")] + pub error_code: i32, + #[prost(string, optional, tag = "2")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ApiVersionsRequest { + #[prost(string, required, tag = "1")] + pub client_software_name: ::prost::alloc::string::String, + #[prost(string, required, tag = "2")] + pub client_software_version: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ApiVersionsResponse { + #[prost(message, repeated, tag = "1")] + pub api_versions: ::prost::alloc::vec::Vec, + #[prost(int32, optional, tag = "2")] + pub server_type: ::core::option::Option, +} +/// get schema request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetTableSchemaRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(int32, optional, tag = "2")] + pub schema_id: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetTableSchemaResponse { + #[prost(int32, required, tag = "1")] + pub schema_id: i32, + #[prost(bytes = "vec", required, tag = "2")] + pub schema_json: ::prost::alloc::vec::Vec, +} +/// create database request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CreateDatabaseRequest { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, + #[prost(bool, required, tag = "2")] + pub ignore_if_exists: bool, + #[prost(bytes = "vec", optional, tag = "3")] + pub database_json: ::core::option::Option<::prost::alloc::vec::Vec>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CreateDatabaseResponse {} +/// alter database request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AlterDatabaseRequest { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, + #[prost(bool, required, tag = "2")] + pub ignore_if_not_exists: bool, + #[prost(message, repeated, tag = "3")] + pub config_changes: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "4")] + pub comment: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AlterDatabaseResponse {} +/// get table request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetDatabaseInfoRequest { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetDatabaseInfoResponse { + #[prost(bytes = "vec", required, tag = "3")] + pub database_json: ::prost::alloc::vec::Vec, + #[prost(int64, required, tag = "4")] + pub created_time: i64, + #[prost(int64, required, tag = "5")] + pub modified_time: i64, +} +/// drop database request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DropDatabaseRequest { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, + #[prost(bool, required, tag = "2")] + pub ignore_if_not_exists: bool, + #[prost(bool, required, tag = "3")] + pub cascade: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DropDatabaseResponse {} +/// database exists request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DatabaseExistsRequest { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DatabaseExistsResponse { + #[prost(bool, required, tag = "1")] + pub exists: bool, +} +/// list databases request and response +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListDatabasesRequest { + #[prost(bool, optional, tag = "1")] + pub include_summary: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListDatabasesResponse { + #[prost(string, repeated, tag = "1")] + pub database_name: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(message, repeated, tag = "2")] + pub database_summary: ::prost::alloc::vec::Vec, +} +/// create table request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CreateTableRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(bytes = "vec", required, tag = "2")] + pub table_json: ::prost::alloc::vec::Vec, + #[prost(bool, required, tag = "3")] + pub ignore_if_exists: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CreateTableResponse {} +/// alter table request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AlterTableRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(bool, required, tag = "2")] + pub ignore_if_not_exists: bool, + #[prost(message, repeated, tag = "3")] + pub config_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub add_columns: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub drop_columns: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "6")] + pub rename_columns: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "7")] + pub modify_columns: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AlterTableResponse {} +/// get table request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetTableInfoRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetTableInfoResponse { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int32, required, tag = "2")] + pub schema_id: i32, + #[prost(bytes = "vec", required, tag = "3")] + pub table_json: ::prost::alloc::vec::Vec, + #[prost(int64, required, tag = "4")] + pub created_time: i64, + #[prost(int64, required, tag = "5")] + pub modified_time: i64, + #[prost(string, optional, tag = "6")] + pub remote_data_dir: ::core::option::Option<::prost::alloc::string::String>, +} +/// list tables request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListTablesRequest { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListTablesResponse { + #[prost(string, repeated, tag = "1")] + pub table_name: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +/// drop table request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DropTableRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(bool, required, tag = "2")] + pub ignore_if_not_exists: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DropTableResponse {} +/// table exists request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TableExistsRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct TableExistsResponse { + #[prost(bool, required, tag = "1")] + pub exists: bool, +} +/// metadata request and response, request send from client to each server. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MetadataRequest { + #[prost(message, repeated, tag = "1")] + pub table_path: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "2")] + pub partitions_path: ::prost::alloc::vec::Vec, + /// note: currently, we assume the partition ids must belong to the table_paths in the + /// metadata request + /// todo: we won't need the assumption after we introduce metadata cache in server + #[prost(int64, repeated, tag = "3")] + pub partitions_id: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MetadataResponse { + #[prost(message, optional, tag = "1")] + pub coordinator_server: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub tablet_servers: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub table_metadata: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub partition_metadata: ::prost::alloc::vec::Vec, +} +/// update metadata request and response, only send between server. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct UpdateMetadataRequest { + #[prost(message, optional, tag = "1")] + pub coordinator_server: ::core::option::Option, + #[prost(message, repeated, tag = "2")] + pub tablet_servers: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub table_metadata: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub partition_metadata: ::prost::alloc::vec::Vec, + #[prost(int32, optional, tag = "5")] + pub coordinator_epoch: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct UpdateMetadataResponse {} +/// produce log request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProduceLogRequest { + #[prost(int32, required, tag = "1")] + pub acks: i32, + #[prost(int64, required, tag = "2")] + pub table_id: i64, + #[prost(int32, required, tag = "3")] + pub timeout_ms: i32, + #[prost(message, repeated, tag = "4")] + pub buckets_req: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ProduceLogResponse { + #[prost(message, repeated, tag = "1")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +/// fetch log request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FetchLogRequest { + /// value -1 indicate the request from client. + #[prost(int32, required, tag = "1")] + pub follower_server_id: i32, + #[prost(int32, required, tag = "2")] + pub max_bytes: i32, + #[prost(message, repeated, tag = "3")] + pub tables_req: ::prost::alloc::vec::Vec, + #[prost(int32, optional, tag = "4")] + pub max_wait_ms: ::core::option::Option, + #[prost(int32, optional, tag = "5")] + pub min_bytes: ::core::option::Option, + #[prost(int32, optional, tag = "6")] + pub read_preference: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FetchLogResponse { + #[prost(message, repeated, tag = "1")] + pub tables_resp: ::prost::alloc::vec::Vec, +} +/// put kv request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PutKvRequest { + #[prost(int32, required, tag = "1")] + pub acks: i32, + #[prost(int64, required, tag = "2")] + pub table_id: i64, + #[prost(int32, required, tag = "3")] + pub timeout_ms: i32, + /// the indexes for the columns to write, + /// if empty, means write all columns + #[prost(int32, repeated, tag = "4")] + pub target_columns: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub buckets_req: ::prost::alloc::vec::Vec, + /// Aggregation mode for this request (see AGG_MODE_* constants above) + /// 0 = AGGREGATE (default), 1 = OVERWRITE, 2 = LOCAL_AGGREGATE (not yet supported) + #[prost(int32, optional, tag = "6")] + pub agg_mode: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PutKvResponse { + #[prost(message, repeated, tag = "1")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +/// lookup request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LookupRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_req: ::prost::alloc::vec::Vec, + #[prost(bool, optional, tag = "3")] + pub insert_if_not_exists: ::core::option::Option, + #[prost(int32, optional, tag = "4")] + pub acks: ::core::option::Option, + #[prost(int32, optional, tag = "5")] + pub timeout_ms: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LookupResponse { + #[prost(message, repeated, tag = "1")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +/// Prefix Lookup request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrefixLookupRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_req: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrefixLookupResponse { + #[prost(message, repeated, tag = "1")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +/// limit scan request and response +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct LimitScanRequest { + #[prost(int64, required, tag = "2")] + pub table_id: i64, + #[prost(int64, optional, tag = "3")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "4")] + pub bucket_id: i32, + #[prost(int32, required, tag = "5")] + pub limit: i32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct LimitScanResponse { + #[prost(int32, optional, tag = "1")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "2")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + /// flag to indicate the table type + #[prost(bool, optional, tag = "3")] + pub is_log_table: ::core::option::Option, + /// LogRecordBatch if is_log_table is true, otherwise KvRecordBatch + #[prost(bytes = "vec", optional, tag = "4")] + pub records: ::core::option::Option<::prost::alloc::vec::Vec>, +} +/// Full KV scan request and response. +/// A new scan is initiated with bucket_scan_req; subsequent batches use scanner_id. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbScanReqForBucket { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "3")] + pub bucket_id: i32, + /// If set, stops returning rows after this many records. + #[prost(int64, optional, tag = "4")] + pub limit: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScanKvRequest { + /// Mutually exclusive: either scanner_id (continuation) or bucket_scan_req (new scan). + #[prost(bytes = "vec", optional, tag = "1")] + pub scanner_id: ::core::option::Option<::prost::alloc::vec::Vec>, + #[prost(message, optional, tag = "2")] + pub bucket_scan_req: ::core::option::Option, + /// Monotonically increasing sequence number for in-order delivery validation. + #[prost(int32, optional, tag = "3")] + pub call_seq_id: ::core::option::Option, + /// Maximum number of bytes of record data to return in this batch. + #[prost(int32, optional, tag = "4")] + pub batch_size_bytes: ::core::option::Option, + /// If true, the server closes the scanner session immediately. + #[prost(bool, optional, tag = "5")] + pub close_scanner: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ScanKvResponse { + #[prost(int32, optional, tag = "1")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "2")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + /// Opaque server-assigned session identifier; echoed on every response. + #[prost(bytes = "vec", optional, tag = "3")] + pub scanner_id: ::core::option::Option<::prost::alloc::vec::Vec>, + /// False when the scan is complete and the scanner session has been closed. + #[prost(bool, optional, tag = "4")] + pub has_more_results: ::core::option::Option, + /// Serialised DefaultValueRecordBatch; absent when the batch is empty. + #[prost(bytes = "bytes", optional, tag = "5")] + pub records: ::core::option::Option<::prost::bytes::Bytes>, + /// Log high-watermark at the time the RocksDB snapshot was opened. + /// Only set on the first response (new scan initiation). + #[prost(int64, optional, tag = "6")] + pub log_offset: ::core::option::Option, +} +/// Get table statistics request and response. +/// Sent to TabletServer to get per-bucket statistics. +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetTableStatsRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_req: ::prost::alloc::vec::Vec, + /// Column indexes for column-level stats (e.g., ndv, sum). + /// If empty, column stats are not collected. + /// Reserved for future use. + #[prost(int32, repeated, tag = "3")] + pub target_columns: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetTableStatsResponse { + #[prost(message, repeated, tag = "1")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +/// notify bucket leader and isr request +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NotifyLeaderAndIsrRequest { + #[prost(int32, required, tag = "1")] + pub coordinator_epoch: i32, + #[prost(message, repeated, tag = "2")] + pub notify_buckets_leader_req: ::prost::alloc::vec::Vec< + PbNotifyLeaderAndIsrReqForBucket, + >, +} +/// response for notify bucket leader and isr request +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NotifyLeaderAndIsrResponse { + #[prost(message, repeated, tag = "1")] + pub notify_buckets_leader_resp: ::prost::alloc::vec::Vec< + PbNotifyLeaderAndIsrRespForBucket, + >, +} +/// stop bucket replica request +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StopReplicaRequest { + #[prost(int32, required, tag = "1")] + pub coordinator_epoch: i32, + #[prost(message, repeated, tag = "2")] + pub stop_replicas_req: ::prost::alloc::vec::Vec, +} +/// response for stop bucket replica request +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StopReplicaResponse { + #[prost(message, repeated, tag = "1")] + pub stop_replicas_resp: ::prost::alloc::vec::Vec, +} +/// adjust isr request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdjustIsrRequest { + #[prost(int32, required, tag = "1")] + pub server_id: i32, + #[prost(message, repeated, tag = "2")] + pub tables_req: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AdjustIsrResponse { + #[prost(message, repeated, tag = "1")] + pub tables_resp: ::prost::alloc::vec::Vec, +} +/// list offsets request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListOffsetsRequest { + /// value -1 indicate the request from client. + #[prost(int32, required, tag = "1")] + pub follower_server_id: i32, + /// value can be 0,1,2 (see ListOffsetsParam for more details) + #[prost(int32, required, tag = "2")] + pub offset_type: i32, + #[prost(int64, required, tag = "3")] + pub table_id: i64, + #[prost(int64, optional, tag = "4")] + pub partition_id: ::core::option::Option, + /// it is recommended to use packed for repeated numerics to get more efficient encoding + #[prost(int32, repeated, tag = "5")] + pub bucket_id: ::prost::alloc::vec::Vec, + #[prost(int64, optional, tag = "6")] + pub start_timestamp: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListOffsetsResponse { + #[prost(message, repeated, tag = "1")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +/// commit kv snapshot request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CommitKvSnapshotRequest { + #[prost(bytes = "vec", required, tag = "1")] + pub completed_snapshot: ::prost::alloc::vec::Vec, + #[prost(int32, required, tag = "2")] + pub coordinator_epoch: i32, + #[prost(int32, required, tag = "3")] + pub bucket_leader_epoch: i32, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CommitKvSnapshotResponse {} +/// notify the log offset about kv snapshot +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct NotifyKvSnapshotOffsetRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "3")] + pub bucket_id: i32, + #[prost(int32, required, tag = "4")] + pub coordinator_epoch: i32, + #[prost(int64, required, tag = "5")] + pub min_retain_offset: i64, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct NotifyKvSnapshotOffsetResponse {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetLatestKvSnapshotsRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(string, optional, tag = "2")] + pub partition_name: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetLatestKvSnapshotsResponse { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + /// null if it is a non-partitioned table, otherwise, it must be not null + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub latest_snapshots: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetKvSnapshotMetadataRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "3")] + pub bucket_id: i32, + #[prost(int64, required, tag = "4")] + pub snapshot_id: i64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetKvSnapshotMetadataResponse { + #[prost(int64, required, tag = "1")] + pub log_offset: i64, + #[prost(message, repeated, tag = "2")] + pub snapshot_files: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AcquireKvSnapshotLeaseRequest { + #[prost(string, required, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(int64, required, tag = "2")] + pub lease_duration_ms: i64, + #[prost(message, repeated, tag = "3")] + pub snapshots_to_lease: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AcquireKvSnapshotLeaseResponse { + #[prost(message, repeated, tag = "1")] + pub unavailable_snapshots: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReleaseKvSnapshotLeaseRequest { + #[prost(string, required, tag = "1")] + pub lease_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub buckets_to_release: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ReleaseKvSnapshotLeaseResponse {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DropKvSnapshotLeaseRequest { + #[prost(string, required, tag = "1")] + pub lease_id: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DropKvSnapshotLeaseResponse {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetLakeSnapshotRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + /// If not set, returns the latest snapshot + #[prost(int64, optional, tag = "2")] + pub snapshot_id: ::core::option::Option, + /// If true, returns the latest readable snapshot + #[prost(bool, optional, tag = "3")] + pub readable: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetLakeSnapshotResponse { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, required, tag = "2")] + pub snapshot_id: i64, + #[prost(message, repeated, tag = "3")] + pub bucket_snapshots: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetFileSystemSecurityTokenRequest {} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetFileSystemSecurityTokenResponse { + #[prost(string, required, tag = "1")] + pub schema: ::prost::alloc::string::String, + #[prost(bytes = "vec", required, tag = "2")] + pub token: ::prost::alloc::vec::Vec, + #[prost(int64, optional, tag = "3")] + pub expiration_time: ::core::option::Option, + #[prost(message, repeated, tag = "4")] + pub addition_info: ::prost::alloc::vec::Vec, +} +/// init writer request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct InitWriterRequest { + #[prost(message, repeated, tag = "1")] + pub table_path: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct InitWriterResponse { + #[prost(int64, required, tag = "1")] + pub writer_id: i64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListPartitionInfosRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(message, optional, tag = "2")] + pub partial_partition_spec: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListPartitionInfosResponse { + #[prost(message, repeated, tag = "1")] + pub partitions_info: ::prost::alloc::vec::Vec, +} +/// list remote log manifest entries (one per bucket of a table or partition) +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListRemoteLogManifestsRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + /// required if table is partitioned + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListRemoteLogManifestsResponse { + #[prost(message, repeated, tag = "1")] + pub manifests: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbRemoteLogManifestEntry { + #[prost(message, required, tag = "1")] + pub table_bucket: PbTableBucket, + #[prost(string, required, tag = "2")] + pub remote_log_manifest_path: ::prost::alloc::string::String, + #[prost(int64, required, tag = "3")] + pub remote_log_end_offset: i64, +} +/// list active KV snapshot dirs (retained_N + still-in-use) for a unit +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListKvSnapshotsRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListKvSnapshotsResponse { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + /// null if it is a non-partitioned table, otherwise, it must be not null + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + /// Active snapshots = retained_N ∪ still-in-use; multiple entries per bucket allowed. + #[prost(message, repeated, tag = "3")] + pub active_snapshots: ::prost::alloc::vec::Vec, +} +/// create partition request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreatePartitionRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(message, required, tag = "2")] + pub partition_spec: PbPartitionSpec, + #[prost(bool, required, tag = "3")] + pub ignore_if_not_exists: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CreatePartitionResponse {} +/// drop partition request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DropPartitionRequest { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(message, required, tag = "2")] + pub partition_spec: PbPartitionSpec, + #[prost(bool, required, tag = "3")] + pub ignore_if_not_exists: bool, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DropPartitionResponse {} +/// commit remote log manifest request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CommitRemoteLogManifestRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "3")] + pub bucket_id: i32, + #[prost(string, required, tag = "4")] + pub remote_log_manifest_path: ::prost::alloc::string::String, + #[prost(int64, required, tag = "5")] + pub remote_log_start_offset: i64, + #[prost(int64, required, tag = "6")] + pub remote_log_end_offset: i64, + #[prost(int32, required, tag = "7")] + pub coordinator_epoch: i32, + #[prost(int32, required, tag = "8")] + pub bucket_leader_epoch: i32, + #[prost(int64, optional, tag = "9")] + pub highest_copied_end_offset: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CommitRemoteLogManifestResponse { + #[prost(bool, required, tag = "1")] + pub commit_success: bool, +} +/// notify remote log offsets request and response +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct NotifyRemoteLogOffsetsRequest { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "3")] + pub bucket_id: i32, + #[prost(int32, required, tag = "4")] + pub coordinator_epoch: i32, + #[prost(int64, required, tag = "5")] + pub remote_start_offset: i64, + #[prost(int64, required, tag = "6")] + pub remote_end_offset: i64, + #[prost(int64, optional, tag = "7")] + pub highest_copied_end_offset: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct NotifyRemoteLogOffsetsResponse {} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrepareLakeTableSnapshotRequest { + #[prost(message, repeated, tag = "1")] + pub bucket_offsets: ::prost::alloc::vec::Vec, + /// If true, ignore previous table offsets and do not merge with existing lake table offsets. + /// If false (default), the system will merge the provided bucket offsets with the previous + /// lake table offsets to ensure continuity. This is useful when you want to replace all offsets + /// instead of merging with existing ones. + #[prost(bool, optional, tag = "2")] + pub ignore_previous_table_offsets: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PrepareLakeTableSnapshotResponse { + #[prost(message, repeated, tag = "1")] + pub prepare_lake_table_resp: ::prost::alloc::vec::Vec< + PbPrepareLakeTableRespForTable, + >, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommitLakeTableSnapshotRequest { + /// Deprecated: PbLakeTableSnapshotInfo is no longer used for committing lake table snapshots. + /// Currently, it is only used to allow the coordinator to notify tablet servers about the current + /// synchronized log end offsets, which are then reported to metrics. In the future, we plan to + /// have tiering directly report to metrics, and this field will be fully removed. + /// Still reserve it for protocol compatibility. + #[prost(message, repeated, tag = "1")] + pub tables_req: ::prost::alloc::vec::Vec, + /// The metadata for lake table snapshots to be committed. Each entry contains the table ID, + /// snapshot ID, and the file paths where the snapshot data (containing bucket log end offset + /// information) is stored. The tiered_snapshot_file_path points to the file storing tiered log + /// end offsets, while readable_snapshot_file_path (if present) points to the file storing readable + /// log end offsets. + #[prost(message, repeated, tag = "2")] + pub lake_table_snapshot_metadata: ::prost::alloc::vec::Vec< + PbLakeTableSnapshotMetadata, + >, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CommitLakeTableSnapshotResponse { + #[prost(message, repeated, tag = "1")] + pub table_resp: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbCommitLakeTableSnapshotRespForTable { + #[prost(int32, optional, tag = "1")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "2")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, required, tag = "3")] + pub table_id: i64, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NotifyLakeTableOffsetRequest { + #[prost(int32, required, tag = "1")] + pub coordinator_epoch: i32, + #[prost(message, repeated, tag = "2")] + pub notify_buckets_req: ::prost::alloc::vec::Vec< + PbNotifyLakeTableOffsetReqForBucket, + >, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbNotifyLakeTableOffsetReqForBucket { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "3")] + pub bucket_id: i32, + #[prost(int64, required, tag = "4")] + pub snapshot_id: i64, + #[prost(int64, optional, tag = "5")] + pub log_start_offset: ::core::option::Option, + #[prost(int64, optional, tag = "6")] + pub log_end_offset: ::core::option::Option, + #[prost(int64, optional, tag = "7")] + pub max_timestamp: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct NotifyLakeTableOffsetResponse {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AuthenticateRequest { + #[prost(string, required, tag = "1")] + pub protocol: ::prost::alloc::string::String, + #[prost(bytes = "vec", required, tag = "2")] + pub token: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AuthenticateResponse { + #[prost(bytes = "vec", optional, tag = "1")] + pub challenge: ::core::option::Option<::prost::alloc::vec::Vec>, +} +/// acl related requests and responses +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListAclsRequest { + #[prost(message, required, tag = "1")] + pub acl_filter: PbAclFilter, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListAclsResponse { + #[prost(message, repeated, tag = "1")] + pub acl: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateAclsRequest { + #[prost(message, repeated, tag = "1")] + pub acl: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct CreateAclsResponse { + #[prost(message, repeated, tag = "1")] + pub acl_res: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DropAclsRequest { + #[prost(message, repeated, tag = "1")] + pub acl_filter: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DropAclsResponse { + #[prost(message, repeated, tag = "1")] + pub filter_results: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LakeTieringHeartbeatRequest { + #[prost(message, repeated, tag = "1")] + pub tiering_tables: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "2")] + pub finished_tables: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "3")] + pub failed_tables: ::prost::alloc::vec::Vec, + /// whether to request a table + #[prost(bool, optional, tag = "4")] + pub request_table: ::core::option::Option, + /// Table IDs that were force finished due to reaching the maximum tiering duration. + /// When a table's tiering operation exceeds the max duration (data lake freshness), + /// it will be force finished to prevent it from blocking other tables' tiering operations. + /// These table IDs must also appear in finished_tables. The coordinator uses this field to + /// distinguish between normally finished tables and force finished tables. + #[prost(int64, repeated, packed = "false", tag = "5")] + pub force_finished_tables: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct LakeTieringHeartbeatResponse { + /// coordinator epoch + #[prost(int32, required, tag = "1")] + pub coordinator_epoch: i32, + /// the returned table to tier, empty when no table is needed by the lake tiering service + #[prost(message, optional, tag = "2")] + pub tiering_table: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub tiering_table_resp: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub finished_table_resp: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub failed_table_resp: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ControlledShutdownRequest { + #[prost(int32, required, tag = "1")] + pub tablet_server_id: i32, + #[prost(int32, required, tag = "2")] + pub tablet_server_epoch: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ControlledShutdownResponse { + #[prost(message, repeated, tag = "1")] + pub remaining_leader_buckets: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DescribeClusterConfigsRequest {} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DescribeClusterConfigsResponse { + #[prost(message, repeated, tag = "1")] + pub configs: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct AlterClusterConfigsRequest { + #[prost(message, repeated, tag = "1")] + pub alter_configs: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AlterClusterConfigsResponse {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AddServerTagRequest { + #[prost(int32, repeated, tag = "1")] + pub server_ids: ::prost::alloc::vec::Vec, + #[prost(int32, required, tag = "2")] + pub server_tag: i32, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct AddServerTagResponse {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RemoveServerTagRequest { + #[prost(int32, repeated, tag = "1")] + pub server_ids: ::prost::alloc::vec::Vec, + #[prost(int32, required, tag = "2")] + pub server_tag: i32, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RemoveServerTagResponse {} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RebalanceRequest { + #[prost(int32, repeated, tag = "1")] + pub goals: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RebalanceResponse { + #[prost(string, required, tag = "1")] + pub rebalance_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct ListRebalanceProgressRequest { + #[prost(string, optional, tag = "1")] + pub rebalance_id: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListRebalanceProgressResponse { + #[prost(string, optional, tag = "1")] + pub rebalance_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "2")] + pub rebalance_status: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub table_progress: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CancelRebalanceRequest { + #[prost(string, optional, tag = "1")] + pub rebalance_id: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct CancelRebalanceResponse {} +/// Register producer offsets request and response +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RegisterProducerOffsetsRequest { + #[prost(string, required, tag = "1")] + pub producer_id: ::prost::alloc::string::String, + #[prost(message, repeated, tag = "2")] + pub table_offsets: ::prost::alloc::vec::Vec, + #[prost(int64, optional, tag = "3")] + pub ttl_ms: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct RegisterProducerOffsetsResponse { + /// Result of registration: 0 = CREATED (new snapshot), 1 = ALREADY_EXISTS (snapshot existed) + #[prost(int32, optional, tag = "1")] + pub result: ::core::option::Option, +} +/// Get producer offsets request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetProducerOffsetsRequest { + #[prost(string, required, tag = "1")] + pub producer_id: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetProducerOffsetsResponse { + #[prost(string, optional, tag = "1")] + pub producer_id: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, optional, tag = "2")] + pub expiration_time: ::core::option::Option, + #[prost(message, repeated, tag = "3")] + pub table_offsets: ::prost::alloc::vec::Vec, +} +/// Delete producer offsets request and response +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DeleteProducerOffsetsRequest { + #[prost(string, required, tag = "1")] + pub producer_id: ::prost::alloc::string::String, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct DeleteProducerOffsetsResponse {} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetClusterHealthRequest {} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct GetClusterHealthResponse { + #[prost(int32, required, tag = "1")] + pub num_replicas: i32, + #[prost(int32, required, tag = "2")] + pub in_sync_replicas: i32, + #[prost(int32, required, tag = "3")] + pub num_leader_replicas: i32, + #[prost(int32, required, tag = "4")] + pub active_leader_replicas: i32, + /// PbClusterHealthStatus: GREEN=0, YELLOW=1, RED=2, UNKNOWN=3 + #[prost(int32, required, tag = "5")] + pub status: i32, +} +/// --------------- Inner classes ---------------- +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbApiVersion { + #[prost(int32, required, tag = "1")] + pub api_key: i32, + #[prost(int32, required, tag = "2")] + pub min_version: i32, + #[prost(int32, required, tag = "3")] + pub max_version: i32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbTablePath { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, + #[prost(string, required, tag = "2")] + pub table_name: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbPhysicalTablePath { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, + #[prost(string, required, tag = "2")] + pub table_name: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub partition_name: ::core::option::Option<::prost::alloc::string::String>, +} +/// For MetadataResponse, host and port are still used for all versions. +/// For UpdateMetadataRequest, +/// * versions <= 0.6: host and port are used. +/// * versions >= 0.7: listeners is used to replace host and port. +/// For MetadataResponse and UpdateMetadataRequest: Fluss versions >= 0.7: we add rack for each tabletServer +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbServerNode { + #[prost(int32, required, tag = "1")] + pub node_id: i32, + #[prost(string, required, tag = "2")] + pub host: ::prost::alloc::string::String, + #[prost(int32, required, tag = "3")] + pub port: i32, + #[prost(string, optional, tag = "4")] + pub listeners: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "5")] + pub rack: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbTableMetadata { + #[prost(message, required, tag = "1")] + pub table_path: PbTablePath, + #[prost(int64, required, tag = "2")] + pub table_id: i64, + #[prost(int32, required, tag = "3")] + pub schema_id: i32, + /// TODO Starting from version 0.9, fluss-client no longer get table info from metadata response. However, for backward + /// compatibility, we will still retain the field in the request. Once metadataV2 is introduced, table info will be + /// officially omitted from the new metadata response. + #[prost(bytes = "vec", required, tag = "4")] + pub table_json: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "5")] + pub bucket_metadata: ::prost::alloc::vec::Vec, + #[prost(int64, required, tag = "6")] + pub created_time: i64, + #[prost(int64, required, tag = "7")] + pub modified_time: i64, + #[prost(string, optional, tag = "8")] + pub remote_data_dir: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbPartitionMetadata { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + /// the partition name and id for the partition + #[prost(string, required, tag = "2")] + pub partition_name: ::prost::alloc::string::String, + #[prost(int64, required, tag = "3")] + pub partition_id: i64, + #[prost(message, repeated, tag = "4")] + pub bucket_metadata: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbBucketMetadata { + #[prost(int32, required, tag = "1")] + pub bucket_id: i32, + /// optional as some time the leader may not elected yet + #[prost(int32, optional, tag = "2")] + pub leader_id: ::core::option::Option, + #[prost(int32, repeated, tag = "3")] + pub replica_id: ::prost::alloc::vec::Vec, + /// TODO: Add isr here. + #[prost(int32, optional, tag = "4")] + pub leader_epoch: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbProduceLogReqForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(bytes = "bytes", required, tag = "3")] + pub records: ::prost::bytes::Bytes, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbProduceLogRespForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "3")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, optional, tag = "5")] + pub base_offset: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbFetchLogReqForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(bool, required, tag = "2")] + pub projection_pushdown_enabled: bool, + #[prost(int32, repeated, tag = "3")] + pub projected_fields: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag = "4")] + pub buckets_req: ::prost::alloc::vec::Vec, + /// filter_predicate and filter_schema_id MUST be set together. + /// filter_predicate is the serialized predicate; filter_schema_id identifies + /// the schema against which the predicate was compiled. + #[prost(message, optional, tag = "5")] + pub filter_predicate: ::core::option::Option, + #[prost(int32, optional, tag = "6")] + pub filter_schema_id: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbFetchLogReqForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + /// TODO leader epoch + #[prost(int64, required, tag = "3")] + pub fetch_offset: i64, + #[prost(int32, required, tag = "4")] + pub max_fetch_bytes: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbFetchLogRespForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbFetchLogRespForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "3")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, optional, tag = "5")] + pub high_watermark: ::core::option::Option, + /// TODO now we don't introduce log start offset, but remain it in protobuf + #[prost(int64, optional, tag = "6")] + pub log_start_offset: ::core::option::Option, + #[prost(message, optional, tag = "7")] + pub remote_log_fetch_info: ::core::option::Option, + #[prost(bytes = "vec", optional, tag = "8")] + pub records: ::core::option::Option<::prost::alloc::vec::Vec>, + /// The offset the server scanned up to while filtering out batches, which may accompany a + /// non-empty records field when only trailing batches were filtered. The client should start + /// its next fetch from the later of this offset and the end of the records it received. + #[prost(int64, optional, tag = "9")] + pub filtered_end_offset: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbPutKvReqForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(bytes = "bytes", required, tag = "3")] + pub records: ::prost::bytes::Bytes, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbPutKvRespForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "3")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + /// the log end offset (LEO) of the changelog after this write + /// this is the offset of the next record to be written, used for exactly-once semantics + #[prost(int64, optional, tag = "5")] + pub log_end_offset: ::core::option::Option, + /// backpressure signal from RocksDB write stall detection: + /// 0 = normal (no pressure), (0,1) = DELAYED zone (normalized L0 ratio) + /// introduced in v1.0 + #[prost(float, optional, tag = "6")] + pub pressure: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLookupReqForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(bytes = "bytes", repeated, tag = "3")] + pub keys: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>, + /// The original partition name for historical lookup. It is unset for normal lookup. + #[prost(string, optional, tag = "4")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbLookupRespForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "3")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, repeated, tag = "5")] + pub values: ::prost::alloc::vec::Vec, + /// The original partition name for historical lookup. It is unset for normal lookup. + #[prost(string, optional, tag = "6")] + pub original_partition_name: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbValue { + /// optional, if empty, means no value + #[prost(bytes = "vec", optional, tag = "1")] + pub values: ::core::option::Option<::prost::alloc::vec::Vec>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbValueList { + #[prost(bytes = "vec", repeated, tag = "1")] + pub values: ::prost::alloc::vec::Vec<::prost::alloc::vec::Vec>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbPrefixLookupReqForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(bytes = "bytes", repeated, tag = "3")] + pub keys: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbPrefixLookupRespForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "3")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, repeated, tag = "5")] + pub value_lists: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbTableBucket { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, optional, tag = "2")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "3")] + pub bucket_id: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbAdjustIsrReqForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_req: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbAdjustIsrReqForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, required, tag = "3")] + pub leader_epoch: i32, + #[prost(int32, repeated, tag = "4")] + pub new_isr: ::prost::alloc::vec::Vec, + #[prost(int32, required, tag = "5")] + pub coordinator_epoch: i32, + #[prost(int32, required, tag = "6")] + pub bucket_epoch: i32, + #[prost(int32, repeated, tag = "7")] + pub standby_replicas: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbAdjustIsrRespForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_resp: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbAdjustIsrRespForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "3")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, optional, tag = "5")] + pub leader_id: ::core::option::Option, + #[prost(int32, optional, tag = "6")] + pub leader_epoch: ::core::option::Option, + #[prost(int32, repeated, tag = "7")] + pub isr: ::prost::alloc::vec::Vec, + #[prost(int32, optional, tag = "8")] + pub bucket_epoch: ::core::option::Option, + #[prost(int32, optional, tag = "9")] + pub coordinator_epoch: ::core::option::Option, + #[prost(int32, repeated, tag = "10")] + pub standby_replicas: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbListOffsetsRespForBucket { + #[prost(int32, required, tag = "1")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "2")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "3")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, optional, tag = "4")] + pub offset: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbNotifyLeaderAndIsrReqForBucket { + /// we need table path for tablet server to create data dir for the bucket, + /// although tablet server don't need it after it has created data dir. + /// For simplicity, we always pass table path. + #[prost(message, required, tag = "1")] + pub physical_table_path: PbPhysicalTablePath, + #[prost(message, required, tag = "2")] + pub table_bucket: PbTableBucket, + #[prost(int32, required, tag = "3")] + pub leader: i32, + #[prost(int32, required, tag = "4")] + pub leader_epoch: i32, + #[prost(int32, repeated, tag = "5")] + pub replicas: ::prost::alloc::vec::Vec, + #[prost(int32, repeated, tag = "6")] + pub isr: ::prost::alloc::vec::Vec, + #[prost(int32, required, tag = "7")] + pub bucket_epoch: i32, + #[prost(int32, repeated, tag = "8")] + pub standby_replicas: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbNotifyLeaderAndIsrRespForBucket { + #[prost(message, required, tag = "1")] + pub table_bucket: PbTableBucket, + #[prost(int32, optional, tag = "2")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "3")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbStopReplicaReqForBucket { + #[prost(message, required, tag = "1")] + pub table_bucket: PbTableBucket, + #[prost(int32, required, tag = "2")] + pub leader_epoch: i32, + /// delete means remove local replica data (i.e., data stored on the current node). + #[prost(bool, required, tag = "3")] + pub delete: bool, + /// deleteRemote means remove remote replica data (e.g., data in object storage) and was introduced in v0.9. + /// For backward compatibility, if a request does not include the deleteRemote flag, the system treats delete as + /// deleteRemote (i.e., it falls back to remote deletion). This ensures older CoordinatorServer continues to function + /// correctly with newer TabletServers. + #[prost(bool, optional, tag = "4")] + pub delete_remote: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbStopReplicaRespForBucket { + #[prost(message, required, tag = "1")] + pub table_bucket: PbTableBucket, + #[prost(int32, optional, tag = "2")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "3")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbKvSnapshot { + #[prost(int32, required, tag = "1")] + pub bucket_id: i32, + /// null if there is no snapshot for this bucket + #[prost(int64, optional, tag = "2")] + pub snapshot_id: ::core::option::Option, + /// null if there is no snapshot for this bucket, then read from EARLIEST + #[prost(int64, optional, tag = "3")] + pub log_offset: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLakeSnapshotForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int64, optional, tag = "3")] + pub log_offset: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub partition_name: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbRemotePathAndLocalFile { + #[prost(string, required, tag = "1")] + pub remote_path: ::prost::alloc::string::String, + #[prost(string, required, tag = "2")] + pub local_file_name: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbKeyValue { + #[prost(string, required, tag = "1")] + pub key: ::prost::alloc::string::String, + #[prost(string, required, tag = "2")] + pub value: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbRemoteLogFetchInfo { + #[prost(string, required, tag = "1")] + pub remote_log_tablet_dir: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub partition_name: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, repeated, tag = "3")] + pub remote_log_segments: ::prost::alloc::vec::Vec, + #[prost(int32, optional, tag = "4")] + pub first_start_pos: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbRemoteLogSegment { + #[prost(string, required, tag = "1")] + pub remote_log_segment_id: ::prost::alloc::string::String, + #[prost(int64, required, tag = "2")] + pub remote_log_start_offset: i64, + #[prost(int64, required, tag = "3")] + pub remote_log_end_offset: i64, + #[prost(int32, required, tag = "4")] + pub segment_size_in_bytes: i32, + #[prost(int64, optional, tag = "5")] + pub max_timestamp: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbPartitionInfo { + #[prost(int64, required, tag = "1")] + pub partition_id: i64, + #[prost(message, required, tag = "2")] + pub partition_spec: PbPartitionSpec, + #[prost(string, optional, tag = "3")] + pub remote_data_dir: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbPartitionSpec { + #[prost(message, repeated, tag = "1")] + pub partition_key_values: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbCreateAclRespInfo { + #[prost(message, required, tag = "1")] + pub acl: PbAclInfo, + #[prost(int32, optional, tag = "2")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "3")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbAclInfo { + #[prost(string, required, tag = "1")] + pub resource_name: ::prost::alloc::string::String, + #[prost(int32, required, tag = "2")] + pub resource_type: i32, + #[prost(string, required, tag = "3")] + pub principal_name: ::prost::alloc::string::String, + #[prost(string, required, tag = "4")] + pub principal_type: ::prost::alloc::string::String, + #[prost(string, required, tag = "5")] + pub host: ::prost::alloc::string::String, + #[prost(int32, required, tag = "6")] + pub operation_type: i32, + #[prost(int32, required, tag = "7")] + pub permission_type: i32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbAclFilter { + #[prost(string, optional, tag = "1")] + pub resource_name: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, required, tag = "2")] + pub resource_type: i32, + #[prost(string, optional, tag = "3")] + pub principal_name: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "4")] + pub principal_type: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, optional, tag = "5")] + pub host: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int32, required, tag = "6")] + pub operation_type: i32, + #[prost(int32, required, tag = "7")] + pub permission_type: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbDropAclsFilterResult { + #[prost(message, repeated, tag = "1")] + pub matching_acls: ::prost::alloc::vec::Vec, + #[prost(int32, optional, tag = "2")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "3")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbDropAclsMatchingAcl { + #[prost(message, required, tag = "1")] + pub acl: PbAclInfo, + #[prost(int32, optional, tag = "2")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "3")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLakeTieringTableInfo { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, required, tag = "2")] + pub table_path: PbTablePath, + #[prost(int64, required, tag = "3")] + pub tiering_epoch: i64, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbHeartbeatReqForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + /// the coordinator epoch when the table is assigned to be tiering service + #[prost(int32, required, tag = "2")] + pub coordinator_epoch: i32, + /// the tiering epoch when the table is assigned to be tiering service + #[prost(int64, required, tag = "3")] + pub tiering_epoch: i64, + /// stats for the last tiering round; only set for finished_tables in LakeTieringHeartbeatRequest + #[prost(message, optional, tag = "4")] + pub lake_tiering_stats: ::core::option::Option, +} +/// Stats reported by the tiering service when a table finishes a tiering round. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLakeTieringStats { + /// cumulative total file size (bytes) of the lake table after tiering + #[prost(int64, optional, tag = "1")] + pub file_size: ::core::option::Option, + /// cumulative total record count of the lake table after tiering + #[prost(int64, optional, tag = "2")] + pub record_count: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbHeartbeatRespForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, optional, tag = "2")] + pub error: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbAlterConfig { + #[prost(string, required, tag = "1")] + pub config_key: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub config_value: ::core::option::Option<::prost::alloc::string::String>, + /// SET=0, DELETE=1, APPEND=2, SUBTRACT=3 + #[prost(int32, required, tag = "3")] + pub op_type: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbAddColumn { + #[prost(string, required, tag = "1")] + pub column_name: ::prost::alloc::string::String, + #[prost(bytes = "vec", required, tag = "2")] + pub data_type_json: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "3")] + pub comment: ::core::option::Option<::prost::alloc::string::String>, + /// LAST=0,FIRST=1,AFTER=3 + #[prost(int32, required, tag = "4")] + pub column_position_type: i32, + #[prost(string, optional, tag = "5")] + pub agg_function_type: ::core::option::Option<::prost::alloc::string::String>, + #[prost(message, repeated, tag = "6")] + pub agg_function_params: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbDropColumn { + #[prost(string, required, tag = "1")] + pub column_name: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbRenameColumn { + #[prost(string, required, tag = "1")] + pub old_column_name: ::prost::alloc::string::String, + #[prost(string, required, tag = "2")] + pub new_column_name: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbModifyColumn { + #[prost(string, required, tag = "1")] + pub column_name: ::prost::alloc::string::String, + #[prost(bytes = "vec", optional, tag = "2")] + pub data_type_json: ::core::option::Option<::prost::alloc::vec::Vec>, + #[prost(string, optional, tag = "3")] + pub comment: ::core::option::Option<::prost::alloc::string::String>, + /// LAST=0,FIRST=1,AFTER=3 + #[prost(int32, optional, tag = "4")] + pub column_position_type: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbDescribeConfig { + #[prost(string, required, tag = "1")] + pub config_key: ::prost::alloc::string::String, + #[prost(string, optional, tag = "2")] + pub config_value: ::core::option::Option<::prost::alloc::string::String>, + #[prost(string, required, tag = "3")] + pub config_source: ::prost::alloc::string::String, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbRebalancePlanForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_plan: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbRebalanceProgressForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub buckets_progress: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbRebalanceProgressForBucket { + #[prost(message, required, tag = "1")] + pub rebalance_plan: PbRebalancePlanForBucket, + #[prost(int32, required, tag = "2")] + pub rebalance_status: i32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbRebalancePlanForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int32, optional, tag = "3")] + pub original_leader: ::core::option::Option, + #[prost(int32, optional, tag = "4")] + pub new_leader: ::core::option::Option, + #[prost(int32, repeated, tag = "5")] + pub original_replicas: ::prost::alloc::vec::Vec, + #[prost(int32, repeated, tag = "6")] + pub new_replicas: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLakeTableSnapshotMetadata { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(int64, required, tag = "2")] + pub snapshot_id: i64, + #[prost(string, required, tag = "3")] + pub tiered_bucket_offsets_file_path: ::prost::alloc::string::String, + #[prost(string, optional, tag = "4")] + pub readable_bucket_offsets_file_path: ::core::option::Option< + ::prost::alloc::string::String, + >, + /// Snapshot retention policy: The earliest snapshot ID to retain. + /// 1. If set, the system will keep all snapshots in the range \[earliest_snapshot_id_to_keep, current_snapshot_id\]. + /// 2. If not set, the system defaults to a "Single Snapshot Retention" policy, keeping only the snapshot specified in this request. + #[prost(int64, optional, tag = "5")] + pub earliest_snapshot_id_to_keep: ::core::option::Option, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbLakeTableSnapshotInfo { + #[prost(int64, optional, tag = "1")] + pub table_id: ::core::option::Option, + #[prost(int64, required, tag = "2")] + pub snapshot_id: i64, + #[prost(message, repeated, tag = "3")] + pub buckets_req: ::prost::alloc::vec::Vec, + /// add table path to reduce get table_path by table id + #[prost(message, optional, tag = "4")] + pub table_path: ::core::option::Option, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbLakeTableOffsetForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + /// Deprecated: log_start_offset is no longer used. Field number 3 is reserved for protocol compatibility. + /// optional int64 log_start_offset = 3; + #[prost(int64, optional, tag = "4")] + pub log_end_offset: ::core::option::Option, + /// Deprecated: partition_name is no longer used. Field number 5 is reserved for protocol compatibility. + /// optional string partition_name = 5; + #[prost(int64, optional, tag = "6")] + pub max_timestamp: ::core::option::Option, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbPrepareLakeTableRespForTable { + #[prost(int32, optional, tag = "1")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "2")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, optional, tag = "3")] + pub table_id: ::core::option::Option, + #[prost(string, optional, tag = "4")] + pub lake_table_offsets_path: ::core::option::Option<::prost::alloc::string::String>, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbTableOffsets { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, required, tag = "2")] + pub table_path: PbTablePath, + #[prost(message, repeated, tag = "3")] + pub bucket_offsets: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbBucketOffset { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int64, optional, tag = "4")] + pub log_end_offset: ::core::option::Option, +} +/// Helper message for table offsets in producer offsets (without table_path) +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbProducerTableOffsets { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub bucket_offsets: ::prost::alloc::vec::Vec, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbDatabaseSummary { + #[prost(string, required, tag = "1")] + pub database_name: ::prost::alloc::string::String, + #[prost(int64, required, tag = "2")] + pub created_time: i64, + #[prost(int32, required, tag = "3")] + pub table_count: i32, +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbKvSnapshotLeaseForTable { + #[prost(int64, required, tag = "1")] + pub table_id: i64, + #[prost(message, repeated, tag = "2")] + pub bucket_snapshots: ::prost::alloc::vec::Vec, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbKvSnapshotLeaseForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, + #[prost(int64, required, tag = "3")] + pub snapshot_id: i64, +} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbTableStatsReqForBucket { + #[prost(int64, optional, tag = "1")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "2")] + pub bucket_id: i32, +} +#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PbTableStatsRespForBucket { + #[prost(int32, optional, tag = "1")] + pub error_code: ::core::option::Option, + #[prost(string, optional, tag = "2")] + pub error_message: ::core::option::Option<::prost::alloc::string::String>, + #[prost(int64, optional, tag = "3")] + pub partition_id: ::core::option::Option, + #[prost(int32, required, tag = "4")] + pub bucket_id: i32, + /// --- Table-level stats --- + /// The number of rows in this bucket. + /// For KV tables: the number of unique keys (live rows). + /// For Log tables: total number of log records (highWatermark - logStartOffset). + /// Absent if row count is not available (e.g., WAL changelog mode or legacy tables). + #[prost(int64, optional, tag = "5")] + pub row_count: ::core::option::Option, +} +/// Represents a predicate that can be serialized and transmitted across +/// languages +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbPredicate { + /// See PbPredicateType constants: LEAF = 0, COMPOUND = 1 + #[prost(int32, required, tag = "1")] + pub r#type: i32, + #[prost(message, optional, tag = "2")] + pub leaf: ::core::option::Option, + #[prost(message, optional, tag = "3")] + pub compound: ::core::option::Option, +} +/// Represents a leaf predicate that compares a field with literals +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbLeafPredicate { + /// The function to apply (see PbLeafFunction constants: EQUAL=0 ... NOT_IN=12) + #[prost(int32, required, tag = "1")] + pub function: i32, + /// The schema field id of the referenced top-level field + #[prost(int32, required, tag = "2")] + pub field_id: i32, + /// The literals to compare with + #[prost(message, repeated, tag = "3")] + pub literals: ::prost::alloc::vec::Vec, +} +/// Represents a compound predicate that combines multiple predicates +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbCompoundPredicate { + /// The function to apply (see PbCompoundFunction constants: AND=0, OR=1) + #[prost(int32, required, tag = "1")] + pub function: i32, + /// The child predicates + #[prost(message, repeated, tag = "2")] + pub children: ::prost::alloc::vec::Vec, +} +/// Represents a literal value +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PbLiteralValue { + /// See PbDataTypeRoot constants: indicates which value field is populated + #[prost(int32, required, tag = "1")] + pub literal_type: i32, + #[prost(bool, required, tag = "2")] + pub is_null: bool, + #[prost(bool, optional, tag = "3")] + pub boolean_value: ::core::option::Option, + #[prost(int32, optional, tag = "4")] + pub int_value: ::core::option::Option, + #[prost(int64, optional, tag = "5")] + pub bigint_value: ::core::option::Option, + #[prost(float, optional, tag = "6")] + pub float_value: ::core::option::Option, + #[prost(double, optional, tag = "7")] + pub double_value: ::core::option::Option, + #[prost(string, optional, tag = "8")] + pub string_value: ::core::option::Option<::prost::alloc::string::String>, + #[prost(bytes = "vec", optional, tag = "9")] + pub binary_value: ::core::option::Option<::prost::alloc::vec::Vec>, + /// Serialized decimal (compact mode) + #[prost(int64, optional, tag = "10")] + pub decimal_value: ::core::option::Option, + /// Epoch millis + #[prost(int64, optional, tag = "11")] + pub timestamp_millis_value: ::core::option::Option, + /// Nano of millis + #[prost(int32, optional, tag = "12")] + pub timestamp_nano_of_millis_value: ::core::option::Option, + /// Serialized decimal (non-compact mode) + #[prost(bytes = "vec", optional, tag = "13")] + pub decimal_bytes: ::core::option::Option<::prost::alloc::vec::Vec>, +} diff --git a/fluss-rust/crates/fluss/tests/vendored_proto.rs b/fluss-rust/crates/fluss/tests/vendored_proto.rs new file mode 100644 index 00000000000..3c045e93ba9 --- /dev/null +++ b/fluss-rust/crates/fluss/tests/vendored_proto.rs @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Guards the vendored `proto/FlussApi.proto` copy that ships with the crate. +//! Both it and `src/proto/fluss.rs` come from the same `regen.sh` run, so this +//! check only needs to compare the copy against the canonical proto — and it +//! runs without protoc. + +use std::path::Path; + +#[test] +fn the_vendored_proto_matches_the_canonical_one() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let canonical = dir.join("../../../fluss-rpc/src/main/proto/FlussApi.proto"); + if !canonical.exists() { + return; // released crate: only the vendored copy ships + } + assert!( + std::fs::read(dir.join("proto/FlussApi.proto")).ok() == std::fs::read(&canonical).ok(), + "FlussApi.proto changed; rerun fluss-rust/crates/fluss/regen.sh and commit the result" + ); +} diff --git a/fluss-rust/docs/cpp-bazel-usage.md b/fluss-rust/docs/cpp-bazel-usage.md index 61d861edccb..3766fb54915 100644 --- a/fluss-rust/docs/cpp-bazel-usage.md +++ b/fluss-rust/docs/cpp-bazel-usage.md @@ -25,7 +25,6 @@ Current simplification scope: Version baseline references currently used by examples: -- `protobuf/protoc`: `3.25.5` - `arrow-cpp`: `19.0.1` ## Common Consumer `BUILD.bazel` @@ -44,10 +43,8 @@ cc_binary( ## Mode 1: `system` (Recommended in preinstalled environments) -Use this mode when your environment already provides: - -- `protoc` -- Arrow C++ (headers + shared libraries) +Use this mode when your environment already provides Arrow C++ +(headers + shared libraries). ### Consumer `MODULE.bazel` (pattern) @@ -60,7 +57,6 @@ bazel_dep(name = "fluss-cpp", version = "") fluss_cpp = use_extension("@fluss-cpp//bindings/cpp/bazel/cpp:deps.bzl", "cpp_sdk") fluss_cpp.config( mode = "system", - protobuf_version = "3.25.5", arrow_cpp_version = "19.0.1", # Adjust Arrow paths for your environment system_arrow_prefix = "/usr", @@ -71,16 +67,14 @@ fluss_cpp.config( use_repo(fluss_cpp, "apache_arrow_cpp") ``` -### Build and run (consumer workspace pattern) +### Build and run (consumer workspace pattern, system mode) Run from your consumer workspace root (the directory containing `MODULE.bazel` and your top-level `BUILD.bazel`). ```bash -PROTOC_BIN="$(command -v protoc)" CARGO_BIN="$(command -v cargo)" bazel run \ - --action_env=PROTOC="$PROTOC_BIN" \ --action_env=CARGO="$CARGO_BIN" \ --action_env=PATH="$(dirname "$CARGO_BIN"):$PATH" \ //:fluss_reader @@ -92,10 +86,8 @@ bazel run \ ```bash cd bindings/cpp/examples/bazel-consumer/system -PROTOC_BIN="$(command -v protoc)" CARGO_BIN="$(command -v cargo)" bazel run \ - --action_env=PROTOC="$PROTOC_BIN" \ --action_env=CARGO="$CARGO_BIN" \ --action_env=PATH="$(dirname "$CARGO_BIN"):$PATH" \ //:consumer_system @@ -117,7 +109,6 @@ bazel_dep(name = "fluss-cpp", version = "") fluss_cpp = use_extension("@fluss-cpp//bindings/cpp/bazel/cpp:deps.bzl", "cpp_sdk") fluss_cpp.config( mode = "build", - protobuf_version = "3.25.5", arrow_cpp_version = "19.0.1", ) use_repo(fluss_cpp, "apache_arrow_cpp") @@ -125,23 +116,12 @@ use_repo(fluss_cpp, "apache_arrow_cpp") Notes: -- `build` mode in the core Bazel integration still uses `PROTOC` (env / PATH). -- To auto-download a pinned `protoc` for `build` mode, use - `bindings/cpp/scripts/ensure_protoc.sh` and pass the result via `--action_env=PROTOC=...`. -- `ensure_protoc.sh` auto-detects host OS/arch (`linux`/`osx`, `x86_64`/`aarch_64`). - Some environments may require `ep_cmake_ar/ranlib/nm` overrides. -### Build and run (consumer workspace pattern, with auto-downloaded `protoc`) - -Run from the `fluss-rust` repository root, or adjust the script path if you -copied it elsewhere. - -```bash -PROTOC_BIN="$(bash bindings/cpp/scripts/ensure_protoc.sh --print-path)" -``` +### Build and run (consumer workspace pattern, build mode) ```bash -bazel run --action_env=PROTOC="$PROTOC_BIN" //:fluss_reader +bazel run //:fluss_reader ``` If `cargo` is not on Bazel action `PATH`, also pass: @@ -149,7 +129,6 @@ If `cargo` is not on Bazel action `PATH`, also pass: ```bash CARGO_BIN="$(command -v cargo)" bazel run \ - --action_env=PROTOC="$PROTOC_BIN" \ --action_env=CARGO="$CARGO_BIN" \ --action_env=PATH="$(dirname "$CARGO_BIN"):$PATH" \ //:fluss_reader @@ -161,10 +140,8 @@ bazel run \ ```bash cd bindings/cpp/examples/bazel-consumer/build -PROTOC_BIN="$(bash ../../../scripts/ensure_protoc.sh --print-path)" CARGO_BIN="$(command -v cargo)" bazel run \ - --action_env=PROTOC="$PROTOC_BIN" \ --action_env=CARGO="$CARGO_BIN" \ --action_env=PATH="$(dirname "$CARGO_BIN"):$PATH" \ //:consumer_build @@ -206,7 +183,6 @@ unset all_proxy ALL_PROXY ```bash cd bindings/cpp/examples/bazel-consumer/build -PROTOC_BIN="$(bash ../../../scripts/ensure_protoc.sh --print-path)" CARGO_BIN="$(command -v cargo)" bazel --ignore_all_rc_files run \ --registry=https://bcr.bazel.build \ @@ -221,7 +197,6 @@ bazel --ignore_all_rc_files run \ --action_env=HTTPS_PROXY="${HTTPS_PROXY:-}" \ --action_env=all_proxy= \ --action_env=ALL_PROXY= \ - --action_env=PROTOC="$PROTOC_BIN" \ --action_env=CARGO="$CARGO_BIN" \ --action_env=PATH="$(dirname "$CARGO_BIN"):$PATH" \ --strategy=CcCmakeMakeRule=local \ @@ -247,7 +222,6 @@ sed -i \ -e 's|system_arrow_runtime_glob = "lib/x86_64-linux-gnu/libarrow.so\\*"|system_arrow_runtime_glob = "lib/libarrow.so*"|' \ "$tmp_dir/MODULE.bazel" cd "$tmp_dir" -PROTOC_BIN="$(command -v protoc)" CARGO_BIN="$(command -v cargo)" bazel --ignore_all_rc_files run \ --registry=https://bcr.bazel.build \ @@ -262,7 +236,6 @@ bazel --ignore_all_rc_files run \ --action_env=HTTPS_PROXY="${HTTPS_PROXY:-}" \ --action_env=all_proxy= \ --action_env=ALL_PROXY= \ - --action_env=PROTOC="$PROTOC_BIN" \ --action_env=CARGO="$CARGO_BIN" \ --action_env=PATH="$(dirname "$CARGO_BIN"):$PATH" \ //:consumer_system @@ -273,7 +246,7 @@ On macOS (BSD `sed`), replace `sed -i` with `sed -i ''` in the patch step above. ## Upgrade Procedure 1. Update `bazel_dep(name = "fluss-cpp", version = "...")` -2. Update mode version settings if needed (`protobuf_version`, `arrow_cpp_version`) +2. Update mode version settings if needed (`arrow_cpp_version`) 3. Run `bazel mod tidy` 4. Commit `MODULE.bazel` and `MODULE.bazel.lock` 5. Run build + tests diff --git a/fluss-rust/docs/cpp-cmake-usage.md b/fluss-rust/docs/cpp-cmake-usage.md index 3002d1c4b4f..6f3c0ed04b6 100644 --- a/fluss-rust/docs/cpp-cmake-usage.md +++ b/fluss-rust/docs/cpp-cmake-usage.md @@ -12,18 +12,15 @@ Current tested baselines: -- `protoc`: `3.25.5` - `arrow-cpp`: `19.0.1` Notes: -- CMake currently warns (does not fail) when local `protoc`/Arrow versions differ from the baselines. -- `protoc` is required because Rust `prost-build` runs during the C++ build. +- CMake currently warns (does not fail) when the local Arrow version differs from the baseline. ## Common Prerequisites - Rust toolchain (`cargo` in `PATH`, or set `CARGO=/path/to/cargo`) -- `protoc` in `PATH` (required for `system` mode; `build` mode can auto-download via `bindings/cpp/scripts/ensure_protoc.sh`) - C++17 compiler - CMake 3.22+ @@ -56,14 +53,7 @@ cmake --build /tmp/fluss-cpp-cmake-system --target fluss_cpp -j Use this mode when Arrow C++ is not preinstalled and CMake should fetch/build it. -### Configure (with auto-downloaded `protoc`) - -```bash -PROTOC_BIN="$(bash bindings/cpp/scripts/ensure_protoc.sh --print-path)" -export PATH="$(dirname "$PROTOC_BIN"):$PATH" -``` - -Then configure: +### Configure ```bash cmake -S bindings/cpp -B /tmp/fluss-cpp-cmake-build \ @@ -74,7 +64,6 @@ Optional overrides: - `-DFLUSS_CPP_ARROW_VERSION=19.0.1` - `-DFLUSS_CPP_ARROW_SOURCE_URL=...` (internal mirror or pinned archive) -- `-DFLUSS_CPP_PROTOBUF_VERSION=3.25.5` (baseline warning only) If your environment needs a proxy for CMake/FetchContent downloads, export standard proxy vars before configure/build: @@ -98,8 +87,6 @@ This mode is slower on first build because it compiles Arrow C++ from source. ### Validate `system` mode ```bash -PROTOC_BIN="$(bash bindings/cpp/scripts/ensure_protoc.sh --print-path)" -export PATH="$(dirname "$PROTOC_BIN"):$PATH" cmake -S bindings/cpp -B /tmp/fluss-cpp-cmake-system \ -DFLUSS_CPP_DEP_MODE=system \ -DFLUSS_CPP_ARROW_SYSTEM_ROOT=/tmp/fluss-system-arrow-19.0.1 @@ -109,8 +96,6 @@ cmake --build /tmp/fluss-cpp-cmake-system --target fluss_cpp -j ### Validate `build` mode ```bash -PROTOC_BIN="$(bash bindings/cpp/scripts/ensure_protoc.sh --print-path)" -export PATH="$(dirname "$PROTOC_BIN"):$PATH" cmake -S bindings/cpp -B /tmp/fluss-cpp-cmake-build \ -DFLUSS_CPP_DEP_MODE=build cmake --build /tmp/fluss-cpp-cmake-build --target fluss_cpp -j @@ -120,9 +105,6 @@ cmake --build /tmp/fluss-cpp-cmake-build --target fluss_cpp -j - `cargo not found` - Install Rust toolchain or set `CARGO=/path/to/cargo`. -- `protoc not found` - - Install `protoc` and ensure it is in `PATH`. - - For `build` mode, use `bindings/cpp/scripts/ensure_protoc.sh` and prepend the returned path to `PATH`. - `arrow/c/bridge.h` not found (build mode) - Reconfigure after updating to the latest `bindings/cpp/CMakeLists.txt`; build mode now adds Arrow source/build include dirs explicitly. - Long first build in `build` mode diff --git a/fluss-rust/scripts/vendor-proto.sh b/fluss-rust/scripts/vendor-proto.sh deleted file mode 100755 index 64b328c894b..00000000000 --- a/fluss-rust/scripts/vendor-proto.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to you under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Vendor the canonical FlussApi.proto into the fluss-rs crate so `cargo publish` -# produces a self-contained crate. build.rs prefers crates/fluss/proto/FlussApi.proto -# when present, otherwise reads it from ../../../fluss-rpc in the monorepo. -# -# Usage: -# scripts/vendor-proto.sh # copy canonical proto into the crate -# scripts/vendor-proto.sh --clean # remove the vendored copy - -set -euo pipefail - -REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -CANONICAL="${REPO_ROOT}/../fluss-rpc/src/main/proto/FlussApi.proto" -DEST_DIR="${REPO_ROOT}/crates/fluss/proto" -DEST="${DEST_DIR}/FlussApi.proto" - -if [ "${1:-}" = "--clean" ]; then - rm -rf "$DEST_DIR" - echo "Removed vendored proto: ${DEST_DIR}" - exit 0 -fi - -if [ ! -f "$CANONICAL" ]; then - echo "Canonical proto not found: ${CANONICAL}" >&2 - echo "Run from the consolidated repo (fluss-rpc must be a sibling of fluss-rust)." >&2 - exit 1 -fi - -mkdir -p "$DEST_DIR" -cp "$CANONICAL" "$DEST" -echo "Vendored ${CANONICAL} -> ${DEST}" diff --git a/fluss-rust/website/docs/developer-guide/contributing.md b/fluss-rust/website/docs/developer-guide/contributing.md index c58ad5bf82a..638ba42dfce 100644 --- a/fluss-rust/website/docs/developer-guide/contributing.md +++ b/fluss-rust/website/docs/developer-guide/contributing.md @@ -5,22 +5,18 @@ Welcome to the development guide for `fluss-rust`! This project builds the Fluss ## Prerequisites - Rust 1.85+ (see [rust-toolchain.toml](https://github.com/apache/fluss/blob/main/fluss-rust/rust-toolchain.toml)) -- Protobuf compiler (`protoc`) -Install using your preferred package/version manager: +Install using your preferred package/version manager, for example: ```bash # Using mise -mise install protobuf mise install rust - -# Using Homebrew (macOS) -brew install protobuf - -# Using apt (Ubuntu/Debian) -sudo apt-get install protobuf-compiler ``` +The Protobuf compiler (`protoc`) is only needed when regenerating the protobuf code after +changing the canonical `FlussApi.proto` (run `crates/fluss/regen.sh` and commit the result); +regular builds use the checked-in `crates/fluss/src/proto/fluss.rs`. + ## IDE Setup We recommend [RustRover](https://www.jetbrains.com/rust/) IDE. diff --git a/fluss-rust/website/docs/release/verifying-a-release-candidate.md b/fluss-rust/website/docs/release/verifying-a-release-candidate.md index f97c1a0828d..86911d23ebb 100644 --- a/fluss-rust/website/docs/release/verifying-a-release-candidate.md +++ b/fluss-rust/website/docs/release/verifying-a-release-candidate.md @@ -62,7 +62,7 @@ fluss-rust-0.1.0.tgz: OK ## Verifying build -Extract the source release archive and verify that it builds (and optionally that tests pass). You need **Rust** (see [rust-toolchain.toml](https://github.com/apache/fluss/blob/main/fluss-rust/rust-toolchain.toml) for the expected version) and, for full builds, **protobuf** and **Python 3.9+** for bindings. +Extract the source release archive and verify that it builds (and optionally that tests pass). You need **Rust** (see [rust-toolchain.toml](https://github.com/apache/fluss/blob/main/fluss-rust/rust-toolchain.toml) for the expected version) and, for full builds, **Python 3.9+** for bindings. ```bash tar -xzf fluss-rust-${RELEASE_VERSION}.tgz diff --git a/fluss-rust/website/docs/user-guide/rust/installation.md b/fluss-rust/website/docs/user-guide/rust/installation.md index e4993689e35..6be98a279ba 100644 --- a/fluss-rust/website/docs/user-guide/rust/installation.md +++ b/fluss-rust/website/docs/user-guide/rust/installation.md @@ -52,23 +52,13 @@ fluss = { path = "/path/to/fluss/fluss-rust/crates/fluss", package = "fluss-rs" ## Building from Source -**Prerequisites:** Rust 1.85+, Protobuf compiler (`protoc`) +**Prerequisites:** Rust 1.85+ ```bash git clone https://github.com/apache/fluss.git cd fluss/fluss-rust ``` -Install `protoc`: - -```bash -# macOS -brew install protobuf - -# Ubuntu/Debian -sudo apt-get install protobuf-compiler -``` - Build: ```bash diff --git a/website/community/dev/building.md b/website/community/dev/building.md index a891c84b6b1..f8c5856f8fc 100644 --- a/website/community/dev/building.md +++ b/website/community/dev/building.md @@ -56,13 +56,9 @@ mvn clean install -DskipTests -T 1C ## Building the Rust client (fluss-rust) -The Rust client, language bindings, and examples live under `fluss-rust/` and build with Cargo. You need **Rust** (the toolchain pinned in `fluss-rust/rust-toolchain.toml`, currently 1.85+) and **protoc**, the Protobuf compiler — `build.rs` compiles the canonical `fluss-rpc/src/main/proto/FlussApi.proto`. +The Rust client, language bindings, and examples live under `fluss-rust/` and build with Cargo. You need **Rust** (the toolchain pinned in `fluss-rust/rust-toolchain.toml`, currently 1.85+). The code generated from the canonical `fluss-rpc/src/main/proto/FlussApi.proto` is checked in, so **protoc** is only needed when the proto changes — run `fluss-rust/crates/fluss/regen.sh` and commit the result. ```bash -# protoc (pick one) -brew install protobuf # macOS -sudo apt-get install protobuf-compiler # Debian/Ubuntu - cd fluss-rust cargo build --workspace --all-targets # build everything cargo test --workspace # unit tests diff --git a/website/community/dev/ide-setup.md b/website/community/dev/ide-setup.md index 2914dd8467e..84d743c1e56 100644 --- a/website/community/dev/ide-setup.md +++ b/website/community/dev/ide-setup.md @@ -227,6 +227,6 @@ run configuration, or create a test that calls the `main()` method of the exampl For the Rust client under `fluss-rust/`, we recommend [RustRover](https://www.jetbrains.com/rust/). 1. Open RustRover, choose **Open**, and select the `fluss-rust/` directory as the project root so Cargo resolves the workspace. -2. RustRover uses the toolchain pinned in `fluss-rust/rust-toolchain.toml` (install it with `rustup` if prompted). You also need **protoc** on your `PATH` — see [Building the Rust client](/community/dev/building). +2. RustRover uses the toolchain pinned in `fluss-rust/rust-toolchain.toml` (install it with `rustup` if prompted). 3. Enable **Rustfmt** and **Clippy** under Settings → Rust so local formatting and lints match CI (`cargo fmt --all -- --check`, `cargo clippy --all-targets -- -D warnings`). 4. Apply the Apache license header to new files — the boilerplate is in `fluss-rust/copyright.txt`; configure it under Settings → Editor → Copyright. diff --git a/website/community/how-to-release/creating-a-fluss-release.mdx b/website/community/how-to-release/creating-a-fluss-release.mdx index 054663d1b2f..747d1e16b62 100644 --- a/website/community/how-to-release/creating-a-fluss-release.mdx +++ b/website/community/how-to-release/creating-a-fluss-release.mdx @@ -521,7 +521,7 @@ fluss-rust/scripts/bump-version.sh ${RELEASE_VERSION} Pushing the **RC tag** (`v${RELEASE_VERSION}-rc${RC_NUM}`) triggers the client workflows: - `python-release.yml` builds the wheels + sdist and publishes them to **TestPyPI**. -- `rust-release.yml` runs `cargo publish --dry-run` to validate the crate. The canonical proto is vendored into the crate automatically (`fluss-rust/scripts/vendor-proto.sh`), because `build.rs` reads it from the in-repo `fluss-rpc`. +- `rust-release.yml` runs `cargo publish --dry-run` to validate the crate. The generated protobuf code and the proto it came from are checked in under `fluss-rust/crates/fluss/`, so the crate publishes standalone with no extra preparation. The final publish to **crates.io** and **PyPI** happens when the release tag is pushed (see the **Git tag** step under [Finalize the release](#finalize-the-release)). Publishing uses the CI secrets `CARGO_REGISTRY_TOKEN`, `PYPI_API_TOKEN`, and `TEST_PYPI_API_TOKEN`. diff --git a/website/community/how-to-release/pre-rc-checklist.md b/website/community/how-to-release/pre-rc-checklist.md index 658bc4a7787..a83150a617d 100644 --- a/website/community/how-to-release/pre-rc-checklist.md +++ b/website/community/how-to-release/pre-rc-checklist.md @@ -16,7 +16,7 @@ Run through this before cutting a release candidate. It catches the problems tha ## Build and publish dry-runs -- [ ] `cargo publish -p fluss-rs --dry-run` succeeds (run after `fluss-rust/scripts/vendor-proto.sh` so the proto is vendored) +- [ ] `cargo publish -p fluss-rs --dry-run` succeeds - [ ] Python wheels + sdist install from **TestPyPI** (the RC tag publishes there): ```bash diff --git a/website/docs/apis/rust/installation.md b/website/docs/apis/rust/installation.md index e4993689e35..6be98a279ba 100644 --- a/website/docs/apis/rust/installation.md +++ b/website/docs/apis/rust/installation.md @@ -52,23 +52,13 @@ fluss = { path = "/path/to/fluss/fluss-rust/crates/fluss", package = "fluss-rs" ## Building from Source -**Prerequisites:** Rust 1.85+, Protobuf compiler (`protoc`) +**Prerequisites:** Rust 1.85+ ```bash git clone https://github.com/apache/fluss.git cd fluss/fluss-rust ``` -Install `protoc`: - -```bash -# macOS -brew install protobuf - -# Ubuntu/Debian -sudo apt-get install protobuf-compiler -``` - Build: ```bash From 1ee8db4b30a6935bd8d060f6448dc5fc9b5b86d0 Mon Sep 17 00:00:00 2001 From: Anton Borisov Date: Mon, 24 Aug 2026 14:06:56 +0200 Subject: [PATCH 2/2] [rust] Fix bazel config attribute and tarball regeneration --- fluss-rust/MODULE.bazel | 1 - fluss-rust/crates/fluss/gen/src/main.rs | 31 ++++++++++++++----------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/fluss-rust/MODULE.bazel b/fluss-rust/MODULE.bazel index f0e60250737..06a48d4e12f 100644 --- a/fluss-rust/MODULE.bazel +++ b/fluss-rust/MODULE.bazel @@ -57,7 +57,6 @@ cpp_sdk = use_extension("//bindings/cpp/bazel/cpp:deps.bzl", "cpp_sdk") cpp_sdk.config( mode = "build", arrow_cpp_version = "19.0.1", - protobuf_version = "3.25.5", ep_cmake_ranlib = "/usr/bin/ranlib", ep_cmake_ar = "/usr/bin/ar", ep_cmake_nm = "/usr/bin/nm", diff --git a/fluss-rust/crates/fluss/gen/src/main.rs b/fluss-rust/crates/fluss/gen/src/main.rs index 18b6d0e2b88..62a8d6bc1fe 100644 --- a/fluss-rust/crates/fluss/gen/src/main.rs +++ b/fluss-rust/crates/fluss/gen/src/main.rs @@ -47,22 +47,25 @@ fn main() -> Result<(), Box> { .parent() .ok_or("the gen crate must live inside crates/fluss")?; - // The canonical proto lives in fluss-rpc and is shared with the Java server. - // Regeneration always reads it and refreshes the copy vendored under proto/, - // so the published crate ships the schema next to the generated code. - let include_dir = fluss_dir.join("../../../fluss-rpc/src/main/proto"); + // The canonical proto in fluss-rpc is the source of truth whenever it is reachable, and + // regeneration refreshes the copy vendored under proto/. A source release carries only that + // copy, so fall back to it there. + let vendored_dir = fluss_dir.join("proto"); + let canonical_dir = fluss_dir.join("../../../fluss-rpc/src/main/proto"); + let include_dir = if canonical_dir.join("FlussApi.proto").exists() { + canonical_dir + } else if vendored_dir.join("FlussApi.proto").exists() { + vendored_dir.clone() + } else { + return Err("no FlussApi.proto found in fluss-rpc or proto/".into()); + }; let proto = include_dir.join("FlussApi.proto"); - if !proto.exists() { - return Err(format!( - "canonical proto not found at {}; regeneration must run from the Fluss monorepo", - proto.display() - ) - .into()); - } - let vendored_dir = fluss_dir.join("proto"); - fs::create_dir_all(&vendored_dir)?; - fs::copy(&proto, vendored_dir.join("FlussApi.proto"))?; + let vendored = vendored_dir.join("FlussApi.proto"); + if proto != vendored { + fs::create_dir_all(&vendored_dir)?; + fs::copy(&proto, &vendored)?; + } let mut config = prost_build::Config::new(); config.bytes([