From 7cab1d92d68ff478f877296311aad42ebe6937ea Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 1/8] feat: add the Comet Native Executors wire protocol A Spark driver submits a query to a Comet Native Executor cluster as a QueryRequest: a stage graph of Comet native blocks, connected by the exchanges between them, with each stage's Spark output schema and, for stages that read files, each partition's plan. A Fragment carries a stage from the scheduler to its executors. The proto crate also exports a fingerprint of the .proto files, so that a cluster can refuse a query from a client that speaks another plan format. See #7288. --- native/proto/build.rs | 26 ++++++ native/proto/src/lib.rs | 10 +++ native/proto/src/proto/native_executors.proto | 82 +++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 native/proto/src/proto/native_executors.proto diff --git a/native/proto/build.rs b/native/proto/build.rs index 634888e29b7..59c1dd9c166 100644 --- a/native/proto/build.rs +++ b/native/proto/build.rs @@ -34,8 +34,34 @@ fn main() -> Result<()> { "src/proto/partitioning.proto", "src/proto/operator.proto", "src/proto/config.proto", + "src/proto/native_executors.proto", ], &["src/proto"], )?; + + println!( + "cargo:rustc-env=COMET_PROTO_FINGERPRINT={:016x}", + proto_fingerprint("src/proto")? + ); Ok(()) } + +/// A hash of every .proto file, so two builds can tell whether they speak the same plan format. +/// FNV-1a rather than the standard library's hasher, whose output can change between Rust +/// releases: a driver and a native executor cluster may be built with different compilers. +fn proto_fingerprint(dir: &str) -> Result { + let mut paths: Vec<_> = fs::read_dir(dir)? + .map(|entry| entry.map(|e| e.path())) + .collect::>()?; + paths.retain(|path| path.extension().is_some_and(|ext| ext == "proto")); + paths.sort(); + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for path in paths { + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + for byte in name.bytes().chain(fs::read(&path)?) { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + } + Ok(hash) +} diff --git a/native/proto/src/lib.rs b/native/proto/src/lib.rs index b1d6a173bfb..c0ff3200922 100644 --- a/native/proto/src/lib.rs +++ b/native/proto/src/lib.rs @@ -51,6 +51,16 @@ pub mod spark_config { include!(concat!("generated", "/spark.spark_config.rs")); } +// Include generated modules from .proto files. +#[allow(missing_docs)] +pub mod spark_native_executors { + include!(concat!("generated", "/spark.spark_native_executors.rs")); +} + +/// A hash of the .proto files this crate was built from. A Comet Native Executor cluster +/// requires a driver's to match its own. +pub const PROTO_FINGERPRINT: &str = env!("COMET_PROTO_FINGERPRINT"); + #[cfg(test)] mod tests { use super::spark_operator::{ diff --git a/native/proto/src/proto/native_executors.proto b/native/proto/src/proto/native_executors.proto new file mode 100644 index 00000000000..6c0ba50afb4 --- /dev/null +++ b/native/proto/src/proto/native_executors.proto @@ -0,0 +1,82 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + + +// Messages a Spark driver exchanges with a Comet Native Executor cluster. See issue #7288. + +syntax = "proto3"; + +package spark.spark_native_executors; + +import "operator.proto"; +import "partitioning.proto"; + +option java_package = "org.apache.comet.serde"; + +// A query submitted by a Spark driver to a Comet Native Executor cluster, as the command of +// an Arrow Flight GetFlightInfo request. +message QueryRequest { + // The Comet version and proto fingerprint of the client's build. The scheduler requires both + // to match its own. + string comet_version = 1; + string proto_fingerprint = 2; + // Spark and Comet configs that native execution reads, as a Spark task would pass them. + map conf = 3; + // Stages in topological order: every stage's producers come before it. The last stage is + // the root, and its output is the query result. + repeated Stage stages = 4; +} + +message Stage { + // The stage's native block. Its input leaves (Scan and ShuffleScan) read the stage's + // inputs, matched in the order a depth-first walk of the block meets them. + spark.spark_operator.Operator plan = 1; + repeated StageInput inputs = 2; + // For a stage that reads files: the block with each partition's files injected, one + // serialized Operator per partition. Empty for a stage whose partitions all run `plan`. + repeated bytes partition_plans = 3; + // The number of partitions of a stage that reads no exchange: the length of partition_plans + // for a stage that reads files, or otherwise the block's own, such as a range's number of + // slices. A stage that reads exchanges has as many partitions as they deliver. + uint32 num_partitions = 4; + // The block's output as Spark declares it. Each partition's output is cast to it before it + // leaves the stage, as a Spark task's shuffle writer does, because a hash exchange places + // rows by the types of their values too. + repeated spark.spark_operator.SparkStructField output_schema = 5; +} + +// How a scheduler sends a stage to its executors, inside the task plans Ballista ships. The +// scheduler and the executors come from one build, so this needs no version check of its own. +message Fragment { + // The block every partition runs. Empty for a stage that reads files, where each partition + // runs its own copy, which the partition descriptor carries. + bytes block = 1; + repeated spark.spark_operator.SparkStructField output_schema = 2; + // Whether every partition reads all of each input, one entry per input. + repeated bool broadcast = 3; + map conf = 4; +} + +message StageInput { + // Index into QueryRequest.stages of the stage producing this input. + uint32 producer = 1; + // How the producer's output is partitioned for this stage. Unset for a broadcast. + spark.spark_partitioning.Partitioning partitioning = 2; + // Every partition of this stage reads the producer's whole output. + bool broadcast = 3; +} From 9cbe505187fda1754835c425218e044ac922aea3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 2/8] feat: run Comet native blocks as DataFusion operators outside Spark CometFragmentExec runs a stage's native block for each of its partitions, with the block's input leaves fed by other DataFusion plans, such as a distributed engine's shuffle readers, instead of by the JVM. PhysicalPlanner::with_native_inputs is the seam: an input leaf (Scan or ShuffleScan) takes the next native input, cast to the types Spark declared for it, and an expression that calls into the JVM fails to plan. Each partition's output is cast to the stage's declared Spark schema before it leaves the stage, as a Spark task's shuffle writer casts its input, since a hash exchange places rows by the types of their values too. The block runs in a Comet session over the host's memory pool and disk manager. A partition descriptor child, an in-memory source with one row per partition, carries each partition's index in its stage and, for a stage that reads files, the partition's own copy of the block. An engine that splits a stage between tasks slices it with the stage's other leaves. block_schema plans a block with empty inputs, so that a scheduler can check a block before running it. See #7288. --- native/core/src/execution/jni_api.rs | 24 +- native/core/src/execution/mod.rs | 1 + .../execution/native_executors/fragment.rs | 440 ++++++++++++++++++ .../src/execution/native_executors/mod.rs | 33 ++ native/core/src/execution/operators/scan.rs | 3 +- native/core/src/execution/planner.rs | 133 +++++- native/core/src/execution/planner/shuffle.rs | 2 +- native/core/src/execution/spark_config.rs | 3 + 8 files changed, 617 insertions(+), 22 deletions(-) create mode 100644 native/core/src/execution/native_executors/fragment.rs create mode 100644 native/core/src/execution/native_executors/mod.rs diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 494e7acdff9..00af7f9aac9 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -1067,11 +1067,27 @@ fn prepare_datafusion_session_context( spark_plan: &Operator, ) -> CometResult { let paths = local_dirs.into_iter().map(PathBuf::from).collect(); - let mut rt_config = RuntimeEnvBuilder::new() - .with_disk_manager_builder(plan_disk_manager(paths, max_temp_directory_size)) - .with_object_store_registry(Arc::new(CometObjectStoreRegistry::default())); - rt_config = rt_config.with_memory_pool(memory_pool); + let rt_config = comet_runtime_env(memory_pool) + .with_disk_manager_builder(plan_disk_manager(paths, max_temp_directory_size)); + comet_session_context(rt_config, batch_size, task_cpus, spark_config, spark_plan) +} + +/// The runtime of a Comet session that reserves memory from `memory_pool`. It has no disk +/// manager of its own yet. +pub(crate) fn comet_runtime_env(memory_pool: Arc) -> RuntimeEnvBuilder { + RuntimeEnvBuilder::new() + .with_object_store_registry(Arc::new(CometObjectStoreRegistry::default())) + .with_memory_pool(memory_pool) +} +/// Configures a DataFusion session for `spark_plan` over the runtime `rt_config` builds. +pub(crate) fn comet_session_context( + rt_config: RuntimeEnvBuilder, + batch_size: usize, + task_cpus: usize, + spark_config: &HashMap, + spark_plan: &Operator, +) -> CometResult { let mut session_config = SessionConfig::new() .with_target_partitions(task_cpus) // This DataFusion context is within the scope of an executing Spark Task. We want to set diff --git a/native/core/src/execution/mod.rs b/native/core/src/execution/mod.rs index 39ee2627d53..5088bfd3e46 100644 --- a/native/core/src/execution/mod.rs +++ b/native/core/src/execution/mod.rs @@ -24,6 +24,7 @@ pub mod expressions; pub mod jni_api; pub(crate) mod merge_as_partial; pub(crate) mod metrics; +pub mod native_executors; pub mod operators; pub(crate) mod planner; #[cfg(feature = "python-udf")] diff --git a/native/core/src/execution/native_executors/fragment.rs b/native/core/src/execution/native_executors/fragment.rs new file mode 100644 index 00000000000..4e4aec790e0 --- /dev/null +++ b/native/core/src/execution/native_executors/fragment.rs @@ -0,0 +1,440 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use arrow::array::{Array, AsArray, BinaryArray, RecordBatch, RecordBatchOptions, UInt32Array}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef, UInt32Type}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::common::{internal_datafusion_err, DataFusionError, Result}; +use datafusion::datasource::memory::MemorySourceConfig; +use datafusion::execution::disk_manager::DiskManager; +use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; +use datafusion::execution::TaskContext; +use datafusion::physical_expr::{Distribution, EquivalenceProperties, PhysicalExpr}; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, + InputDistributionRequirements, Partitioning, PlanProperties, SendableRecordBatchStream, +}; +use datafusion::prelude::SessionContext; +use datafusion_comet_proto::spark_native_executors::Fragment; +use datafusion_comet_proto::spark_operator::{Operator, SparkStructField}; +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; + +use crate::execution::jni_api::{comet_runtime_env, comet_session_context}; +use crate::execution::planner::{ + align_shuffle_writer_input, convert_spark_types_to_arrow_schema, PhysicalPlanner, +}; +use crate::execution::serde::deserialize_op; +use crate::execution::spark_config::{ + SparkConfig, COMET_BATCH_SIZE, COMET_STRICT_TIMESTAMP_LABELS, +}; + +const DEFAULT_BATCH_SIZE: usize = 8192; + +fn descriptor_schema() -> SchemaRef { + Arc::new(Schema::new(vec![ + Field::new("partition", DataType::UInt32, false), + Field::new("plan", DataType::Binary, true), + ])) +} + +/// The partition descriptor of a fragment with `num_partitions` partitions. +/// +/// Partition `i` is a single row holding `i`, the partition's index in its stage, and +/// `plans[i]`, the partition's own copy of the block, when `plans` is not empty. A stage that +/// reads files has one, naming the partition's files. A distributed engine that splits a stage's +/// partitions between tasks slices the descriptor along with the stage's other leaves, so a task +/// still learns each partition's index in the stage: Spark seeds `rand`, +/// `monotonically_increasing_id` and similar expressions with it. +pub fn partition_descriptor( + num_partitions: usize, + plans: &[Vec], +) -> Result> { + let schema = descriptor_schema(); + let partitions = (0..num_partitions) + .map(|index| { + let plan = plans.get(index).map(Vec::as_slice); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(UInt32Array::from(vec![index as u32])), + Arc::new(BinaryArray::from(vec![plan])), + ], + )?; + Ok(vec![batch]) + }) + .collect::>>()?; + Ok(MemorySourceConfig::try_new_exec(&partitions, schema, None)?) +} + +/// Runs a Comet native block for each partition of a stage. +/// +/// The first child is the partition descriptor (see [`partition_descriptor`]), and each other +/// child is an input of the block, matched to the block's input leaves in the order Comet's +/// planner meets them. Partition `p` of the fragment reads partition `p` of each partitioned +/// input, and the single partition of each broadcast input. +#[derive(Debug)] +pub struct CometFragmentExec { + fragment: Arc, + /// The partition descriptor, then the inputs. + children: Vec>, + props: Arc, +} + +impl CometFragmentExec { + pub fn try_new(fragment: Arc, children: Vec>) -> Result { + if fragment + .output_schema + .iter() + .any(|field| field.data_type.is_none()) + { + return Err(internal_datafusion_err!( + "CometFragmentExec's output schema has a field without a type" + )); + } + let Some((descriptor, inputs)) = children.split_first() else { + return Err(internal_datafusion_err!( + "CometFragmentExec needs a partition descriptor" + )); + }; + if inputs.len() != fragment.broadcast.len() { + return Err(internal_datafusion_err!( + "CometFragmentExec has {} inputs but {} broadcast flags", + inputs.len(), + fragment.broadcast.len() + )); + } + let partitions = descriptor.output_partitioning().partition_count(); + for (index, (input, broadcast)) in inputs.iter().zip(&fragment.broadcast).enumerate() { + let expected = if *broadcast { 1 } else { partitions }; + let actual = input.output_partitioning().partition_count(); + if actual != expected { + return Err(internal_datafusion_err!( + "CometFragmentExec input {index} has {actual} partitions, expected {expected}" + )); + } + } + let props = Arc::new(PlanProperties::new( + EquivalenceProperties::new(fragment_schema(&fragment.output_schema)), + Partitioning::UnknownPartitioning(partitions), + EmissionType::Incremental, + Boundedness::Bounded, + )); + Ok(Self { + fragment, + children, + props, + }) + } + + pub fn fragment(&self) -> &Arc { + &self.fragment + } +} + +/// The schema of a fragment's output: the types Spark declared for it, every column nullable. +/// The cast to those types keeps a column nullable wherever the block's output is, even where +/// Spark knows it has no nulls, such as a join key a filter has checked, so a fragment can't +/// promise more. Nothing between stages relies on a column having no nulls. +fn fragment_schema(output_schema: &[SparkStructField]) -> SchemaRef { + let declared = convert_spark_types_to_arrow_schema(output_schema); + let fields: Vec = declared + .fields() + .iter() + .map(|field| field.as_ref().clone().with_nullable(true)) + .collect(); + Arc::new(Schema::new(fields)) +} + +impl DisplayAs for CometFragmentExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "CometFragmentExec: block={} bytes, broadcast={:?}", + self.fragment.block.len(), + self.fragment.broadcast + ) + } +} + +impl ExecutionPlan for CometFragmentExec { + fn name(&self) -> &str { + "CometFragmentExec" + } + + fn properties(&self) -> &Arc { + &self.props + } + + fn children(&self) -> Vec<&Arc> { + self.children.iter().collect() + } + + /// The descriptor's and the partitioned inputs' partitions line up with this plan's, while a + /// broadcast input is read whole by every partition, so a distributed engine must not split + /// it between tasks. + fn input_distribution_requirements(&self) -> InputDistributionRequirements { + let inputs = self.fragment.broadcast.iter().map(|broadcast| { + if *broadcast { + Distribution::SinglePartition + } else { + Distribution::UnspecifiedDistribution + } + }); + InputDistributionRequirements::new( + std::iter::once(Distribution::UnspecifiedDistribution) + .chain(inputs) + .collect(), + ) + } + + /// Repartitioning an input for parallelism would break its alignment with the descriptor. + fn benefits_from_input_partitioning(&self) -> Vec { + vec![false; self.children.len()] + } + + /// The block's expressions are in its serialized plan, so there are none to visit here. + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(Self::try_new( + Arc::clone(&self.fragment), + children, + )?)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let descriptor = Arc::clone(&self.children[0]); + let inputs = self.children[1..] + .iter() + .zip(&self.fragment.broadcast) + .map(|(input, broadcast)| { + let input_partition = if *broadcast { 0 } else { partition }; + fragment_input(Arc::clone(input), input_partition, Arc::clone(&context)) + }) + .collect::>>()?; + let fragment = Arc::clone(&self.fragment); + let schema = self.schema(); + let stream = futures::stream::once(async move { + let (index, own_block) = read_descriptor(&descriptor, partition, &context).await?; + // Planning can block on Comet's own runtime, which panics on a runtime thread. + tokio::task::spawn_blocking(move || { + let block = own_block.as_deref().unwrap_or(&fragment.block); + run_block(block, index, inputs, &fragment, schema, &context) + }) + .await + .map_err(|e| DataFusionError::External(Box::new(e)))? + }) + .try_flatten(); + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream, + ))) + } +} + +/// Reads partition `partition` of a fragment's descriptor: the partition's index in its stage, +/// and the partition's own copy of the block, if it has one. +async fn read_descriptor( + descriptor: &Arc, + partition: usize, + context: &Arc, +) -> Result<(u32, Option>)> { + let batches: Vec = descriptor + .execute(partition, Arc::clone(context))? + .try_collect() + .await?; + let batch = match batches.as_slice() { + [batch] if batch.num_rows() == 1 => batch, + _ => { + return Err(internal_datafusion_err!( + "A fragment's partition descriptor must have one row per partition" + )) + } + }; + let index = batch.column(0).as_primitive::().value(0); + let plans = batch.column(1).as_binary::(); + let plan = (!plans.is_null(0)).then(|| plans.value(0).to_vec()); + Ok((index, plan)) +} + +/// Plans `block` for the partition with index `index` in its stage, feeding its input leaves from +/// `inputs`, and starts it. +/// +/// Its output is cast to the fragment's output schema, the types Spark declared for it, before it +/// leaves the stage, as a Spark task's shuffle writer casts its input: a hash exchange places a +/// row by the types of its key values as well as by the values. A wrong timezone label on a +/// timestamp fails instead in strict test runs, as it does there. Each batch then carries +/// `schema`, the fragment's, which a distributed engine checks it against. +fn run_block( + block: &[u8], + index: u32, + inputs: Vec>, + fragment: &Fragment, + schema: SchemaRef, + context: &TaskContext, +) -> Result>> { + let spark_plan = deserialize_op(block)?; + let session = fragment_session( + &spark_plan, + &fragment.conf, + Arc::clone(context.memory_pool()), + Some(Arc::clone(&context.runtime_env().disk_manager)), + )?; + let planner = + PhysicalPlanner::new(Arc::clone(&session), index as i32).with_native_inputs(inputs); + let root = plan_block(&spark_plan, planner)?; + let root = align_shuffle_writer_input( + root, + &fragment.output_schema, + fragment.conf.get_bool(COMET_STRICT_TIMESTAMP_LABELS), + )?; + let stream = root.execute(0, session.task_ctx())?; + // The session holds state that the block's operators may use while they run. + Ok(stream + .map(move |batch| { + let _session = &session; + let batch = batch?; + // The cast made the names and types the declared ones, so this changes only + // nullability and metadata. + let options = RecordBatchOptions::new().with_row_count(Some(batch.num_rows())); + Ok(RecordBatch::try_new_with_options( + Arc::clone(&schema), + batch.columns().to_vec(), + &options, + )?) + }) + .boxed()) +} + +/// The Comet session a block runs in. It reserves memory from `memory_pool` and spills through +/// `disk_manager`, the host's, so that spill files go where the host keeps them. +fn fragment_session( + spark_plan: &Operator, + conf: &HashMap, + memory_pool: Arc, + disk_manager: Option>, +) -> Result> { + let mut runtime = comet_runtime_env(memory_pool); + runtime.disk_manager = disk_manager; + let session = comet_session_context( + runtime, + conf.get_usize(COMET_BATCH_SIZE, DEFAULT_BATCH_SIZE), + 1, + conf, + spark_plan, + ) + .map_err(|e| DataFusionError::External(Box::new(e)))?; + Ok(Arc::new(session)) +} + +fn plan_block(spark_plan: &Operator, planner: PhysicalPlanner) -> Result> { + let planner = planner.with_sql_text_pool(spark_plan); + let (_, _, root) = planner.create_plan(spark_plan, &mut vec![], 1)?; + Ok(Arc::clone(&root.native_plan)) +} + +/// The output schema of `block`, found by planning it without its data: each input leaf is empty, +/// and each file scan builds no object store for its files, since nothing reads them, so this +/// uses no storage credentials. +/// +/// Fails where running the block would fail at planning, such as for an operator or expression +/// this build doesn't support, or one that calls into the JVM. A problem with the files or their +/// store shows only when the block runs. Planning can block on Comet's own runtime, so don't call +/// this on an async runtime's thread. +pub fn block_schema(block: &[u8], conf: &HashMap) -> Result { + let spark_plan = deserialize_op(block)?; + let session = fragment_session( + &spark_plan, + conf, + Arc::new(UnboundedMemoryPool::default()), + None, + )?; + let planner = PhysicalPlanner::new(session, 0).without_data(); + Ok(plan_block(&spark_plan, planner)?.schema()) +} + +/// One partition of a host plan, read with the host's task context: what a block's input leaf +/// reads. Comet's planner sees it as a single-partition leaf. +fn fragment_input( + input: Arc, + partition: usize, + context: Arc, +) -> Result> { + let schema = input.schema(); + let stream = FragmentInput { + input, + partition, + context, + schema: Arc::clone(&schema), + }; + Ok(Arc::new(StreamingTableExec::try_new( + schema, + vec![Arc::new(stream)], + None, + vec![], + false, + None, + )?)) +} + +#[derive(Debug)] +struct FragmentInput { + input: Arc, + partition: usize, + context: Arc, + schema: SchemaRef, +} + +impl PartitionStream for FragmentInput { + fn schema(&self) -> &SchemaRef { + &self.schema + } + + /// Ignores the block's task context: the input belongs to the host's plan. + fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { + match self + .input + .execute(self.partition, Arc::clone(&self.context)) + { + Ok(stream) => stream, + Err(e) => Box::pin(RecordBatchStreamAdapter::new( + Arc::clone(&self.schema), + futures::stream::once(async { Err(e) }), + )), + } + } +} diff --git a/native/core/src/execution/native_executors/mod.rs b/native/core/src/execution/native_executors/mod.rs new file mode 100644 index 00000000000..fc0b9d6b2f4 --- /dev/null +++ b/native/core/src/execution/native_executors/mod.rs @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Running Comet plans outside a Spark executor, on Comet Native Executors. See +//! . +//! +//! A Spark driver splits a query into stages at its exchanges. Each stage is one native block: +//! the serialized `Operator` tree a Spark task would run, whose input leaves (`Scan` and +//! `ShuffleScan`) read the stage's inputs. [`CometFragmentExec`] runs such a block for each +//! partition of a stage, reading its inputs from other DataFusion plans, such as a distributed +//! engine's shuffle readers, instead of from the JVM. + +mod fragment; + +pub use fragment::{block_schema, partition_descriptor, CometFragmentExec}; + +/// The version of this Comet build. A Comet Native Executor cluster requires a driver's to match +/// its own, together with [`datafusion_comet_proto::PROTO_FINGERPRINT`]. +pub const COMET_VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/native/core/src/execution/operators/scan.rs b/native/core/src/execution/operators/scan.rs index b3ec14f2e96..64b9ce813f5 100644 --- a/native/core/src/execution/operators/scan.rs +++ b/native/core/src/execution/operators/scan.rs @@ -192,7 +192,8 @@ fn import_column(col: &ArrayRef) -> Result { Ok(decode_string_arrays(col)?) } -fn schema_from_data_types(data_types: &[DataType]) -> SchemaRef { +/// The schema of an input leaf whose columns have `data_types`, without dictionary types. +pub(crate) fn schema_from_data_types(data_types: &[DataType]) -> SchemaRef { let fields = data_types .iter() .enumerate() diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 899d8ff3320..d1015136186 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -28,6 +28,8 @@ mod shuffle; mod window; mod write; +pub(crate) use shuffle::align_shuffle_writer_input; + // Glue that wires the optional Delta integration into core's plan-tree builder. // Compiled only under `--features contrib-delta`; default builds carry zero Delta // surface. The bulk of the Delta logic lives in the `comet-contrib-delta` crate -- @@ -49,10 +51,11 @@ use crate::execution::operators::TopKReaderFilterExec; #[cfg(feature = "python-udf")] use crate::execution::operators::{ArrowPythonUdfExec, ArrowPythonUdfSpec}; use crate::execution::{ - operators::{ExecutionError, ScanExec, ShuffleScanExec}, + operators::{schema_from_data_types, ExecutionError, ScanExec, ShuffleScanExec}, planner::expression_registry::ExpressionRegistry, planner::operator_registry::OperatorRegistry, serde::{to_arrow_datatype, to_arrow_field}, + shuffle::SchemaAlignExec, }; use crate::jvm_bridge::{jni_call, JVMClasses, OutputStreamSink, ShufflePartitionPusher}; use arrow::compute::CastOptions; @@ -127,7 +130,10 @@ use jni::objects::{Global, JObject}; use log::warn; use num::{BigInt, ToPrimitive}; use object_store::path::Path; -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, VecDeque}, + sync::{Arc, Mutex}, +}; pub type PlanCreationResult = Result<(Vec, Vec, Arc), ExecutionError>; @@ -207,6 +213,18 @@ pub struct PhysicalPlanner { /// Whether a boundary that would cast a timestamp's timezone label to the declared one fails /// instead. Test-only; see `spark.comet.testing.strictTimestampLabels.enabled`. strict_timestamp_labels: bool, + /// What feeds the block's input leaves (`Scan` and `ShuffleScan`) in place of the JVM. Set + /// when Comet runs outside a Spark executor, which also means there is no JVM to call; see + /// [`crate::execution::native_executors`]. + native_inputs: Option>, +} + +/// What feeds a block's input leaves when Comet runs outside a Spark executor. +enum NativeInputs { + /// The plans left to feed the leaves, in the order the planner meets them. + Plans(Mutex>>), + /// Nothing: the block is planned without its data. See [`PhysicalPlanner::without_data`]. + Empty, } impl Default for PhysicalPlanner { @@ -229,6 +247,7 @@ impl PhysicalPlanner { output_stream: None, float_operands: FloatOperands::Normalize, strict_timestamp_labels: false, + native_inputs: None, } } @@ -312,6 +331,92 @@ impl PhysicalPlanner { self } + /// Feeds the block's input leaves from `inputs` instead of the JVM, one per leaf in the order + /// the planner meets them. Each input's output is cast to the types Spark declared for its + /// leaf. Expressions that call into the JVM fail to plan, since there is no JVM. + pub fn with_native_inputs(mut self, inputs: Vec>) -> Self { + self.native_inputs = Some(Arc::new(NativeInputs::Plans(Mutex::new(inputs.into())))); + self + } + + /// Like [`Self::with_native_inputs`], but plans a block without its data, to check it and + /// learn its schema: each input leaf is empty, and each file scan is planned with its files + /// but builds no object store for them, since nothing reads them. Planning then uses no + /// storage credentials. + pub fn without_data(mut self) -> Self { + self.native_inputs = Some(Arc::new(NativeInputs::Empty)); + self + } + + /// Whether the planner plans the block without its data; see [`Self::without_data`]. + fn plans_without_data(&self) -> bool { + matches!(self.native_inputs.as_deref(), Some(NativeInputs::Empty)) + } + + /// Registers the object store a scan reads `file` through, configured by `options`, and + /// returns its URL and backend. When planning without data, the scan gets the local file + /// system instead, which is always registered, and no store is built. + fn scan_store( + &self, + file: &str, + options: &HashMap, + ) -> Result<(ObjectStoreUrl, ObjectStoreBackend), ExecutionError> { + if self.plans_without_data() { + return Ok(( + ObjectStoreUrl::local_filesystem(), + ObjectStoreBackend::Local, + )); + } + let (url, _, backend) = prepare_object_store_with_configs( + self.session_ctx.runtime_env(), + file.to_string(), + options, + )?; + Ok((url, backend)) + } + + /// Plans `spark_plan` as the next native input when it is an input leaf and the planner has + /// native inputs. `None` otherwise. + fn native_input_leaf( + &self, + spark_plan: &Operator, + ) -> Result>, ExecutionError> { + let Some(native_inputs) = &self.native_inputs else { + return Ok(None); + }; + let fields = match spark_plan.op_struct.as_ref() { + Some(OpStruct::Scan(scan)) => &scan.fields, + Some(OpStruct::ShuffleScan(scan)) => &scan.fields, + _ => return Ok(None), + }; + let data_types = fields.iter().map(to_arrow_datatype).collect_vec(); + let schema = schema_from_data_types(&data_types); + let input = match native_inputs.as_ref() { + NativeInputs::Plans(plans) => { + let input = plans.lock().unwrap().pop_front().ok_or_else(|| { + GeneralError("The plan has more input leaves than native inputs".to_string()) + })?; + SchemaAlignExec::try_new_or_passthrough(input, &schema)? + } + NativeInputs::Empty => Arc::new(EmptyExec::new(schema)), + }; + Ok(Some(Arc::new(SparkPlan::new( + spark_plan.plan_id, + input, + vec![], + )))) + } + + /// Fails to plan `what`, which calls into the JVM, when the plan runs without one. + fn require_jvm(&self, what: &str) -> Result<(), ExecutionError> { + if self.native_inputs.is_some() { + return Err(GeneralError(format!( + "{what} calls into the JVM, so it cannot run on Comet Native Executors" + ))); + } + Ok(()) + } + pub(crate) fn strict_timestamp_labels(&self) -> bool { self.strict_timestamp_labels } @@ -498,7 +603,7 @@ impl PhysicalPlanner { partition_files: &SparkFilePartition, ) -> Result<(ObjectStoreUrl, ObjectStoreBackend, Vec), ExecutionError> { let one_file = match partition_files.partitioned_file.first() { - Some(f) => f.file_path.clone(), + Some(f) => f.file_path.as_str(), None => { // Empty partition: no store to resolve; the URL and backend are unused because // the file group is empty and build_parquet_scan_plan returns EmptyExec. @@ -514,11 +619,8 @@ impl PhysicalPlanner { .iter() .map(|(k, v)| (k.clone(), v.clone())) .collect(); - let (object_store_url, _, object_store_backend) = prepare_object_store_with_configs( - self.session_ctx.runtime_env(), - one_file, - &object_store_options, - )?; + let (object_store_url, object_store_backend) = + self.scan_store(one_file, &object_store_options)?; let files = self.get_partitioned_files(partition_files, &object_store_options)?; Ok((object_store_url, object_store_backend, files)) } @@ -963,6 +1065,7 @@ impl PhysicalPlanner { Ok(Arc::new(CastExpr::new(child, data_type, None))) } ExprStruct::Subquery(expr) => { + self.require_jvm("A scalar subquery")?; let id = expr.id; let data_type = to_arrow_datatype(expr.datatype.as_ref().unwrap()); Ok(Arc::new(Subquery::new(self.exec_context_id, id, data_type))) @@ -1129,6 +1232,7 @@ impl PhysicalPlanner { ))) } ExprStruct::JvmScalarUdf(udf) => { + self.require_jvm(&format!("The JVM UDF {}", udf.class_name))?; let args = udf .args .iter() @@ -1634,6 +1738,10 @@ impl PhysicalPlanner { inputs: &mut Vec>>>, partition_count: usize, ) -> PlanCreationResult { + if let Some(input) = self.native_input_leaf(spark_plan)? { + return Ok((vec![], vec![], input)); + } + // Try to use the modular registry first - this automatically handles any registered operator types if OperatorRegistry::global().can_handle(spark_plan) { return OperatorRegistry::global().create_plan( @@ -1886,14 +1994,7 @@ impl PhysicalPlanner { // Build the store from this partition's files; other partitions may read // from a different one. let object_store_url = match partition.partitioned_file.first() { - Some(file) => { - prepare_object_store_with_configs( - self.session_ctx.runtime_env(), - file.file_path.clone(), - &object_store_options, - )? - .0 - } + Some(file) => self.scan_store(&file.file_path, &object_store_options)?.0, None => ObjectStoreUrl::local_filesystem(), }; let files = self.get_partitioned_files(partition, &object_store_options)?; diff --git a/native/core/src/execution/planner/shuffle.rs b/native/core/src/execution/planner/shuffle.rs index f5fb1f869ac..9d3b073879d 100644 --- a/native/core/src/execution/planner/shuffle.rs +++ b/native/core/src/execution/planner/shuffle.rs @@ -292,7 +292,7 @@ impl OperatorBuilder for ShuffleScanBuilder { /// /// With `strict_timestamp_labels`, a timestamp labelled with another timezone than declared fails /// planning instead of being cast. The child is native, so the label is wrong inside its plan too. -fn align_shuffle_writer_input( +pub(crate) fn align_shuffle_writer_input( child: Arc, expected_proto: &[spark_operator::SparkStructField], strict_timestamp_labels: bool, diff --git a/native/core/src/execution/spark_config.rs b/native/core/src/execution/spark_config.rs index 511b5733388..302719f4a98 100644 --- a/native/core/src/execution/spark_config.rs +++ b/native/core/src/execution/spark_config.rs @@ -32,6 +32,9 @@ pub(crate) const COMET_EXEC_SPILL_COMPRESSION_CODEC: &str = "spark.comet.exec.spill.compression.codec"; pub(crate) const COMET_NATIVE_UDF_ENABLED: &str = "spark.comet.nativeUdf.enabled"; pub(crate) const COMET_NATIVE_UDF_ALLOWED_PATHS: &str = "spark.comet.nativeUdf.allowedPaths"; +/// A Spark task passes the batch size to native code apart from the other configs, so only a host +/// other than a Spark executor reads it from them. +pub(crate) const COMET_BATCH_SIZE: &str = "spark.comet.batchSize"; pub(crate) const COMET_STRICT_TIMESTAMP_LABELS: &str = "spark.comet.testing.strictTimestampLabels.enabled"; pub(crate) const SPARK_EXECUTOR_CORES: &str = "spark.executor.cores"; From 59e1043a03fefe77267395f821d0b667d0ba2657 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 3/8] feat: add comet-scheduler and comet-executor built on DataFusion Ballista A new Cargo workspace, native/native-executors, builds the two processes of a Comet Native Executor cluster on the Ballista 55 crates. It is a workspace of its own, so the main Comet build never depends on Ballista. comet-scheduler wraps Ballista's scheduler in an Arrow Flight service. GetFlightInfo takes a QueryRequest, checks that the client runs the same Comet build, plans each block with Comet's planner, turns the stage graph into a Ballista physical plan, runs it, and returns one endpoint per result partition with an opaque ticket for the executor's DoGet. Hash exchanges become RepartitionExec(Hash), and single-partition and broadcast exchanges become CoalescePartitionsExec, which Ballista's planner turns into stage boundaries. A stage that reads exchanges has as many partitions as they deliver. Ballista can't plan a range shuffle for a physical plan yet (apache/datafusion-ballista#2680), so a range exchange is read as one partition, and the stage that sorts it runs in a single task. comet-executor is Ballista's executor with Comet's codec, which carries each CometFragmentExec's Fragment through Ballista's stage and task plans. It needs no JVM. Both binaries take their settings from the environment, with Ballista's defaults otherwise. See #7288. --- dev/release/rat_exclude_files.txt | 1 + .../contributor-guide/release_process.md | 4 +- native/Cargo.toml | 3 +- native/native-executors/Cargo.lock | 7468 +++++++++++++++++ native/native-executors/Cargo.toml | 61 + native/native-executors/README.md | 87 + native/native-executors/src/assembly.rs | 190 + .../src/bin/comet-executor.rs | 67 + .../src/bin/comet-scheduler.rs | 81 + native/native-executors/src/codec.rs | 75 + native/native-executors/src/frontend.rs | 259 + native/native-executors/src/lib.rs | 53 + native/native-executors/tests/assembly.rs | 253 + native/native-executors/tests/common/mod.rs | 164 + 14 files changed, 8764 insertions(+), 2 deletions(-) create mode 100644 native/native-executors/Cargo.lock create mode 100644 native/native-executors/Cargo.toml create mode 100644 native/native-executors/README.md create mode 100644 native/native-executors/src/assembly.rs create mode 100644 native/native-executors/src/bin/comet-executor.rs create mode 100644 native/native-executors/src/bin/comet-scheduler.rs create mode 100644 native/native-executors/src/codec.rs create mode 100644 native/native-executors/src/frontend.rs create mode 100644 native/native-executors/src/lib.rs create mode 100644 native/native-executors/tests/assembly.rs create mode 100644 native/native-executors/tests/common/mod.rs diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 3347c2a8ac8..2fd9a988405 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -6,6 +6,7 @@ .claude/skills CLAUDE.md native/Cargo.lock +native/native-executors/Cargo.lock dev/copyright/scala-header.txt dev/diffs/iceberg/*.diff dev/release/requirements.txt diff --git a/docs/source/contributor-guide/release_process.md b/docs/source/contributor-guide/release_process.md index c4e23606fee..2d98def89b5 100644 --- a/docs/source/contributor-guide/release_process.md +++ b/docs/source/contributor-guide/release_process.md @@ -202,7 +202,9 @@ Create a PR against the main branch to prepare for developing the next release: - Update the Rust crate version to `0.14.0` in `native/Cargo.toml` and in each `contrib/*/native/Cargo.toml`. The contrib crates sit outside the `native/` workspace, so they do not inherit its version. Then run - `cargo update --workspace` in `native/` and in each contrib crate that has its own `Cargo.lock`. + `cargo update --workspace` in `native/`, in `native/native-executors/` (a workspace with its own `Cargo.lock`, + which records the versions of the `native/` crates it uses) and in each contrib crate that has its own + `Cargo.lock`. - Update the Maven version to `0.14.0-SNAPSHOT` in the same set of files listed above (the `pom.xml` files, the Spark test diffs under `dev/diffs`, and the Iceberg test diffs under `dev/diffs/iceberg`). diff --git a/native/Cargo.toml b/native/Cargo.toml index c858f9e715b..902cf832905 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -20,7 +20,8 @@ default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuff members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs"] # Crates under ../contrib are intentionally NOT workspace members. Core pulls them in as # optional path dependencies when their corresponding contrib feature is enabled. -exclude = ["../contrib"] +# native-executors is its own workspace, so that this one never depends on Ballista. +exclude = ["../contrib", "native-executors"] resolver = "2" [workspace.package] diff --git a/native/native-executors/Cargo.lock b/native/native-executors/Cargo.lock new file mode 100644 index 00000000000..4d21ac8a3e5 --- /dev/null +++ b/native/native-executors/Cargo.lock @@ -0,0 +1,7468 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +dependencies = [ + "crypto-common 0.1.7", + "generic-array", +] + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + +[[package]] +name = "aes-gcm" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ghash", + "subtle", +] + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "apache-avro" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312c1ea69e5fe9966e0029fb95aca8790100b85aff4f0d3b00a9337c74069a9c" +dependencies = [ + "bigdecimal", + "bon", + "crc32fast", + "digest 0.11.3", + "log", + "miniz_oxide", + "num-bigint 0.4.8", + "ouroboros", + "quad-rand", + "rand 0.10.3", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum 0.28.0", + "thiserror", + "uuid", + "zstd 0.13.3", +] + +[[package]] +name = "ar_archive_writer" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" +dependencies = [ + "object", +] + +[[package]] +name = "arc-swap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e" +dependencies = [ + "rustversion", +] + +[[package]] +name = "array-init" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d62b7694a562cdf5a74227903507c56ab2cc8bdd1f781ed5cb4cf9c9f810bfc" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "arrow" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c25317ebddb8f2e6f006759336c95e5ded847a446f3524c27853bfe7d77a69b" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.17.1", + "libc", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92" +dependencies = [ + "bytes", + "half", + "num-bigint 0.5.1", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.23.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace83d5935421b950fea3a0a3f9972c0837c297ef98ade57127a5661b681112c" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-flight" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f998a8cc9ffdc37731b66c0414dccf4f4f11a03d9136d383ef10f95181cd7d" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", + "base64 0.23.1", + "bytes", + "futures", + "once_cell", + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + +[[package]] +name = "arrow-ipc" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex", + "zstd 0.13.3", +] + +[[package]] +name = "arrow-json" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "727b21ebf8894dcd9504d6ab66bffdac338391e1f85cde594526a0210299ea40" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "chrono", + "half", + "indexmap 2.14.2", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" +dependencies = [ + "bitflags 2.13.2", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "59.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + +[[package]] +name = "as-any" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" + +[[package]] +name = "async-compression" +version = "0.4.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "asyncband" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" +dependencies = [ + "hashbrown 0.17.1", + "slab", +] + +[[package]] +name = "asyncband" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5161fb43cb8f231f06983cb2b8f9ee9cee14be0b1356d8383398cc0f312800d1" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-config" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939e256cfe0ed68e96d32f194339d88210d5c10685cff0787400158f852c6819" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http", + "sha1 0.10.7", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "576e8da35751dc85409ec1cd15999f692c277a1fbe9336780fe6f15e49dd637d" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-lc-rs" +version = "1.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faac5829c2b74c28f830747e7818ccfb684261b5f48a1118b1e2a13d36dfab13" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1622d8446a2d4b2ce0c7eefc73dd43a99779028d5ee5c2dd8073a658ba8a2bc" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "aws-runtime" +version = "1.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a0cc2144962cc5e358467b16e3d82981b1d424a7c3d34602be0036103814a1a" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "bytes-utils", + "fastrand", + "http", + "http-body", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.117.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5316813498fa9c403877007934ae448aa44b18136f58051f8fa3c5b79b44bdc6" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.119.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31db35e5662787b786daba15f431e4393c7ea512b70fa112bd048d1f3ecf877a" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.122.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10491638d9e560db90c484dbc77d73b8ad202780cdc9b9e92180603acb07ada" +dependencies = [ + "arc-swap", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d3d48355cb86b15ccbb38b8bc6b87500c824ee4ed8cbc10a961b435242b5e16" +dependencies = [ + "aws-credential-types", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac 0.13.0", + "http", + "percent-encoding", + "sha2 0.11.1", + "time", + "tracing", +] + +[[package]] +name = "aws-smithy-async" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de681a9df7ff79823b759ee17061076ec049cd61a8fa28ece4293d89c9f6964e" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-http" +version = "0.64.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba8960956e0ee0ca3f3161e62913a61bc65a7ef638ca3ef1d4c7159f8443e398" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "tracing", +] + +[[package]] +name = "aws-smithy-http-client" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeca69f65362f7375891008ea525fd440ad6db1836458ac188c87b126804114e" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.65.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f6835fb38bc6fc8c91e584b0366eb4ff595e5c485134dbf4f0bdeaa18856666" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-observability" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e86338c869539a581bf161247762a6e87f92c5c075060057b5ed6d06632ed0c" +dependencies = [ + "aws-smithy-runtime-api", +] + +[[package]] +name = "aws-smithy-query" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165f6265342a313ad6f7a1ac17a0cdba30c522039d3f41d8d6871ce8a2b34ecd" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "aws-smithy-xml", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cef0c41e8ca40f74f68ed8ee4d1ddbde12bb847b95ef9b3c025681ddfae58a" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "bytes", + "fastrand", + "http", + "http-body", + "http-body-util", + "pin-project-lite", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06788004dea79018413f381891da6dcca23196acd79c4d51b767fbdacc76eae4" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api-macros", + "aws-smithy-types", + "bytes", + "http", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-runtime-api-macros" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "221eaa237ddf1ca79b60d1372aad77e47f9c0ea5b3ce5099da8c61d027dc77b3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "aws-smithy-schema" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "http", +] + +[[package]] +name = "aws-smithy-types" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55752025c39c0e094962fd73f1c57014b191f678612e517f219468e575af9822" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "http", + "http-body", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "ryu", + "serde", + "time", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6581618a2fd006159016d1c36881d94d61b4a017f4a8d631310875e18252413" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba0d970d3947bb064d2ae1d823efa186c1dbde88f88afa8f6549946e5d8deee" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-schema", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + +[[package]] +name = "ballista-core" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a090511fa9b3661dbf17444ecbbf5efcd11793e140ddd7fe2fb85f2de88f561" +dependencies = [ + "arrow-flight", + "async-trait", + "chrono", + "datafusion", + "datafusion-proto", + "datafusion-proto-common", + "futures", + "log", + "parking_lot", + "pastey", + "prost", + "rand 0.10.3", + "rustc_version", + "serde", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tonic-prost", + "tonic-prost-build", + "url", + "uuid", +] + +[[package]] +name = "ballista-executor" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403cdac6a4e822e6025348f4f7e43765ba0ca176ca1f225888a9eeb90b9c66ce" +dependencies = [ + "arrow", + "arrow-flight", + "async-trait", + "ballista-core", + "bytesize", + "dashmap", + "datafusion", + "datafusion-proto", + "futures", + "libc", + "log", + "lru", + "memory-stats", + "parking_lot", + "serde", + "sysinfo", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "uuid", +] + +[[package]] +name = "ballista-scheduler" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "845bb1c2c763ae2b50ecfd9debdfe8e092d13a4fefe84c20e9262bb191a1f1e7" +dependencies = [ + "arrow-flight", + "async-trait", + "axum", + "ballista-core", + "dashmap", + "datafusion", + "datafusion-proto", + "ferroid", + "futures", + "http", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "prost", + "rand 0.10.3", + "serde", + "tokio", + "tokio-stream", + "tonic", + "tower-http 0.7.1", + "url-escape", + "uuid", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bigdecimal" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fa3f3d8cbf4dffcfe4991de61d012bef509a409ecbe9dd41049bfe32b4d4653" +dependencies = [ + "autocfg", + "libm", + "num-bigint 0.4.8", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bimap" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bnum" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f781dba93de3a5ef6dc5b17c9958b208f6f3f021623b360fb605ea51ce443f10" +dependencies = [ + "serde", + "serde-big-array", +] + +[[package]] +name = "bon" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05442cd7005532d0cfedec676b5b9333d2b4c7c3afabe2b1a9ce2742fbe2ee6" +dependencies = [ + "bon-macros", +] + +[[package]] +name = "bon-macros" +version = "3.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93833e97ac7c46ba97045afcf247238c27057538fcf2f06344662ddd5f567be6" +dependencies = [ + "darling 0.24.1", + "ident_case", + "prettyplease 0.3.0", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "brotli" +version = "8.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "bytesize" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7354288c522e7e980fafd2075d63d1285794c3a6a16cdd492f189ea406e5f18b" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common 0.1.7", + "inout", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "comet-native-executors" +version = "0.0.0" +dependencies = [ + "arrow-flight", + "ballista-core", + "ballista-executor", + "ballista-scheduler", + "datafusion", + "datafusion-comet", + "datafusion-comet-proto", + "datafusion-proto", + "env_logger", + "futures", + "log", + "prost", + "prost-types", + "tokio", + "tonic", +] + +[[package]] +name = "comet-udf-sdk" +version = "1.3.0" +dependencies = [ + "arrow", +] + +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "compression-codecs" +version = "0.4.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd 0.14.3", + "zstd-safe 8.2.0", +] + +[[package]] +name = "compression-core" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a" + +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + +[[package]] +name = "ctr" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +dependencies = [ + "cipher", +] + +[[package]] +name = "ctutils" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" +dependencies = [ + "cmov", +] + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" +dependencies = [ + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", +] + +[[package]] +name = "darling_core" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 3.0.9", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "darling_macro" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" +dependencies = [ + "darling_core 0.24.1", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + +[[package]] +name = "datafusion" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bzip2", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "flate2", + "futures", + "indexmap 2.14.2", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "parquet", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-catalog" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "percent-encoding", +] + +[[package]] +name = "datafusion-comet" +version = "1.3.0" +dependencies = [ + "arrow", + "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-sts", + "aws-smithy-runtime-api", + "base64 0.23.1", + "bytes", + "chrono-tz", + "comet-udf-sdk", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-jni-bridge", + "datafusion-comet-operators", + "datafusion-comet-proto", + "datafusion-comet-shuffle", + "datafusion-comet-spark-expr", + "datafusion-datasource", + "datafusion-physical-expr-adapter", + "datafusion-spark", + "futures", + "hdrs", + "iceberg", + "iceberg-storage-opendal", + "itertools 0.15.0", + "jni", + "libloading", + "log", + "log4rs", + "num", + "object_store", + "once_cell", + "opendal", + "parking_lot", + "parquet", + "paste", + "procfs", + "prost", + "rand 0.10.3", + "reqsign-core", + "reqwest 0.12.28", + "serde", + "serde_json", + "tempfile", + "tokio", + "typetag", + "url", + "uuid", +] + +[[package]] +name = "datafusion-comet-common" +version = "1.3.0" +dependencies = [ + "arrow", + "datafusion", + "serde", + "serde_json", + "simdutf8", + "thiserror", +] + +[[package]] +name = "datafusion-comet-jni-bridge" +version = "1.3.0" +dependencies = [ + "arrow", + "datafusion", + "datafusion-comet-common", + "jni", + "lazy_static", + "once_cell", + "parquet", + "paste", + "prost", + "regex", + "serde_json", + "thiserror", +] + +[[package]] +name = "datafusion-comet-operators" +version = "1.3.0" +dependencies = [ + "arrow", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-spark-expr", + "futures", + "parking_lot", +] + +[[package]] +name = "datafusion-comet-proto" +version = "1.3.0" +dependencies = [ + "prost", + "prost-build", +] + +[[package]] +name = "datafusion-comet-shuffle" +version = "1.3.0" +dependencies = [ + "arrow", + "arrow-data", + "arrow-select", + "async-trait", + "crc32c", + "crc32fast", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-jni-bridge", + "datafusion-comet-spark-expr", + "futures", + "itertools 0.15.0", + "jni", + "log", + "lz4_flex", + "simd-adler32", + "snap", + "tokio", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-comet-spark-expr" +version = "1.3.0" +dependencies = [ + "arrow", + "base64 0.23.1", + "chrono", + "datafusion", + "datafusion-comet-common", + "datafusion-comet-jni-bridge", + "datafusion-functions", + "datafusion-spark", + "datasketches", + "futures", + "jni", + "memchr", + "num", + "paste", + "rand 0.10.3", + "regex", + "ryu", + "serde", + "serde_json", + "smallvec", + "twox-hash", + "uuid", +] + +[[package]] +name = "datafusion-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.1", + "hex", + "indexmap 2.14.2", + "itertools 0.15.0", + "libc", + "log", + "num-traits", + "object_store", + "parquet", + "recursive", + "sqlparser", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" +dependencies = [ + "arrow", + "async-compression", + "async-trait", + "bytes", + "bzip2", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "flate2", + "futures", + "glob", + "itertools 0.15.0", + "liblzma", + "log", + "object_store", + "parking_lot", + "rand 0.9.5", + "tokio", + "tokio-util", + "url", + "zstd 0.13.3", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "itertools 0.15.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-session", + "futures", + "object_store", + "tokio", + "tokio-stream", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-models", + "datafusion-pruning", + "datafusion-session", + "futures", + "itertools 0.15.0", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" + +[[package]] +name = "datafusion-execution" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "bytes", + "dashmap", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", + "futures", + "log", + "object_store", + "parking_lot", + "parquet", + "pin-project-lite", + "rand 0.9.5", + "tempfile", + "tokio", + "tokio-util", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", + "indexmap 2.14.2", + "itertools 0.15.0", + "recursive", + "serde", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap 2.14.2", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-functions" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" +dependencies = [ + "arrow", + "arrow-buffer", + "base64 0.23.1", + "blake2", + "blake3", + "chrono", + "chrono-tz", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hex", + "itertools 0.15.0", + "log", + "md-5 0.11.0", + "memchr", + "num-traits", + "rand 0.9.5", + "regex", + "sha2 0.11.1", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.17.1", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "hashbrown 0.17.1", + "itertools 0.15.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-table" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-functions-window" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" +dependencies = [ + "datafusion-doc", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "datafusion-optimizer" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap 2.14.2", + "itertools 0.15.0", + "log", + "recursive", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" +dependencies = [ + "arrow", + "arrow-schema", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "datafusion-proto-models", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", + "parking_lot", + "petgraph", + "recursive", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.15.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr-common", + "datafusion-proto-models", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", + "parking_lot", + "pin-project", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "datafusion-session", + "itertools 0.15.0", + "recursive", +] + +[[package]] +name = "datafusion-physical-plan" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" +dependencies = [ + "arrow", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", + "futures", + "half", + "hashbrown 0.17.1", + "indexmap 2.14.2", + "itertools 0.15.0", + "log", + "num-traits", + "parking_lot", + "pin-project-lite", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "datafusion-proto" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481881a3422072554c8c26a177a4f0a67f4d8c0e2a394a79e551e6bd9a5cec4b" +dependencies = [ + "arrow", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-datasource-parquet", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-table", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-proto-common", + "datafusion-proto-models", + "object_store", + "prost", +] + +[[package]] +name = "datafusion-proto-common" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a95763d5fa64a88b658ba4ec7314d9c538e2aea749dc88d5ce2afaafbb4c69b8" +dependencies = [ + "arrow", + "datafusion-common", + "prost", +] + +[[package]] +name = "datafusion-proto-models" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c4982d05dee1e0b90b7f3b39845dca7eee5635c550db09158d385b8181bdc6a" +dependencies = [ + "datafusion-common", + "datafusion-proto-common", + "prost", +] + +[[package]] +name = "datafusion-pruning" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "log", +] + +[[package]] +name = "datafusion-session" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" +dependencies = [ + "arrow-schema", + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-spark" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3d6f76463984c4ed8587c52ed4f21f142fd96fa55d8f128f673f2db4c46942" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "crc32fast", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-functions-nested", + "datafusion-session", + "log", + "num-traits", + "percent-encoding", + "rand 0.9.5", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.1", + "twox-hash", + "url", +] + +[[package]] +name = "datafusion-sql" +version = "55.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", + "indexmap 2.14.2", + "log", + "recursive", + "regex", + "sqlparser", + "stacker", +] + +[[package]] +name = "datasketches" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c4cf71a36b46dcfc00e5014c0c20ccad2b1b6a008304d7d57d2749b2d41b3d" + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid 0.9.6", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", + "unicode-xid", +] + +[[package]] +name = "destructure_traitobject" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c877555693c14d2f84191cfd3ad8582790fc52b5e2274b40b59cf5f5cea25c7" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "dlv-list" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "442039f5147480ba31067cb00ada1adae6892028e40e45fc5de7b7df6dcc1b5f" +dependencies = [ + "const-random", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "erased-serde" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2add8a07dd6a8d93ff627029c51de145e12686fbc36ecb298ac22e74cf02dec" +dependencies = [ + "serde", + "serde_core", + "typeid", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fastnum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020d1b59a944bc239d79903fbac2eda2365138b44890c27979562f6592059dcd" +dependencies = [ + "bnum", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "ferroid" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee93edf3c501f0035bbeffeccfed0b79e14c311f12195ec0e661e114a0f60da4" +dependencies = [ + "futures", + "portable-atomic", + "rand 0.10.3", + "tokio", + "web-time", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.13.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core 0.10.1", + "wasm-bindgen", +] + +[[package]] +name = "ghash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +dependencies = [ + "opaque-debug", + "polyval", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "h2" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap 2.14.2", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] + +[[package]] +name = "hdfs-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18138c3dacdf0e95f834369f996b21d2b6fa99e71787ef45e2ece09b37bd1dd1" +dependencies = [ + "cc", + "java-locator", +] + +[[package]] +name = "hdrs" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1ac22889be3d758d8168395e2ab70374afb4b1959caa9586ab42e6dd9f454d" +dependencies = [ + "errno", + "hdfs-sys", + "libc", + "log", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "iceberg" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +dependencies = [ + "aes-gcm", + "anyhow", + "apache-avro", + "array-init", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-ord", + "arrow-schema", + "arrow-select", + "arrow-string", + "as-any", + "async-trait", + "backon", + "base64 0.22.1", + "bimap", + "bytes", + "chrono", + "crc32fast", + "derive_builder", + "fastnum", + "flate2", + "fnv", + "form_urlencoded", + "futures", + "iceberg-property-macro", + "itertools 0.13.0", + "moka", + "murmur3", + "once_cell", + "ordered-float 4.6.0", + "parquet", + "rand 0.9.5", + "reqwest 0.12.28", + "roaring", + "serde", + "serde_bytes", + "serde_derive", + "serde_json", + "serde_repr", + "serde_with", + "strum 0.27.2", + "tokio", + "tracing", + "typed-builder", + "typetag", + "url", + "uuid", + "zeroize", + "zstd 0.13.3", +] + +[[package]] +name = "iceberg-property-macro" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "iceberg-storage-opendal" +version = "0.10.1" +source = "git+https://github.com/apache/iceberg-rust?rev=1f3bc34058e3f3e45336bc5428f37b92649a608b#1f3bc34058e3f3e45336bc5428f37b92649a608b" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "cfg-if", + "futures", + "iceberg", + "iceberg-property-macro", + "opendal", + "reqsign-aws-v4", + "reqsign-core", + "serde", + "typetag", + "url", +] + +[[package]] +name = "icu_collections" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" + +[[package]] +name = "icu_properties" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" + +[[package]] +name = "icu_provider" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", + "serde", + "serde_core", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] + +[[package]] +name = "inventory" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6928282826c822ad91bf1c9a1cb90a30ba1c26770749929b4656cd6be829cd7c" +dependencies = [ + "rustversion", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "java-locator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09c46c1fe465c59b1474e665e85e1256c3893dd00927b8d55f63b09044c1e64f" +dependencies = [ + "glob", +] + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "js-sys", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +dependencies = [ + "spin 0.9.9", +] + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "liblzma" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa980ea6744074a5b5e0ac6b164aca94fc970121869b3e78ae12a2b0097dbea7" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e3581f367a78d7b7e7ae948d023310556f0cfc13156c2e4e00e25616492b9" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +dependencies = [ + "serde_core", +] + +[[package]] +name = "log-mdc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a94d21414c1f4a51209ad204c1776a3d0765002c76c6abcb602a6f09f1e881c7" + +[[package]] +name = "log4rs" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e947bb896e702c711fccc2bf02ab2abb6072910693818d1d6b07ee2b9dfd86c" +dependencies = [ + "anyhow", + "arc-swap", + "chrono", + "derive_more", + "fnv", + "humantime", + "libc", + "log", + "log-mdc", + "mock_instant", + "parking_lot", + "rand 0.9.5", + "serde", + "serde-value", + "serde_json", + "serde_yaml", + "thiserror", + "thread-id", + "typemap-ors", + "unicode-segmentation", + "winapi", +] + +[[package]] +name = "lru" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "lz4_flex" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memory-stats" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c73f5c649995a115e1a0220b35e4df0a1294500477f97a91d0660fb5abeb574a" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "mock_instant" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb517913cfcfb9eeda59f36020269075a152701a01606c612f547e4890be399" + +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmur3" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252111cf132ba0929b6f8e030cac2a24b507f3a4d6db6fb2896f27b354c714b" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint 0.4.8", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.8", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint 0.4.8", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "object" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e5a6c098c7a3b6547378093f5cc30bc54fd361ce711e05293a5cc589562739b" +dependencies = [ + "memchr", +] + +[[package]] +name = "object_store" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "chrono", + "form_urlencoded", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body-util", + "httparse", + "humantime", + "hyper", + "itertools 0.14.0", + "md-5 0.10.6", + "parking_lot", + "percent-encoding", + "quick-xml 0.39.4", + "rand 0.10.3", + "reqwest 0.12.28", + "ring", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "opaque-debug" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" + +[[package]] +name = "opendal" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33dbff14cc9bb085224256d6a81289d2f3202e85b06f408d42534b42162a4231" +dependencies = [ + "ctor", + "opendal-core", + "opendal-http-transport-reqwest", + "opendal-layer-concurrent-limit", + "opendal-layer-logging", + "opendal-layer-retry", + "opendal-layer-timeout", + "opendal-service-azdls", + "opendal-service-fs", + "opendal-service-gcs", + "opendal-service-oss", + "opendal-service-s3", +] + +[[package]] +name = "opendal-core" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48dbcef97d3eb7591db2c18d5cae95c836bcce07359b98d98dd6f4e861eb77b7" +dependencies = [ + "anyhow", + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "futures", + "http", + "jiff", + "log", + "md-5 0.11.0", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "serde", + "serde_json", + "tokio", + "url", + "uuid", + "web-time", +] + +[[package]] +name = "opendal-http-transport-reqwest" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85663452ea32bbc17e8f79ab29788c846d116ec7de31451be9c787e462dcb36c" +dependencies = [ + "bytes", + "futures", + "http", + "http-body", + "opendal-core", + "reqwest 0.13.5", +] + +[[package]] +name = "opendal-layer-concurrent-limit" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03f9e144b5228d741c3763ade8711d9b72e5fb6d998e779f2d7a09da0b5a3eba" +dependencies = [ + "asyncband 0.6.7", + "futures", + "http", + "opendal-core", +] + +[[package]] +name = "opendal-layer-logging" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2de17c61cd32e9d8d7d8efb91795e714dbccbafbc3c4e219e1542f4d6324161" +dependencies = [ + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-retry" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94db301964a25366090484d61e6da16d5979cc8faf02c3e12210dc74fafed38" +dependencies = [ + "backon", + "log", + "opendal-core", +] + +[[package]] +name = "opendal-layer-timeout" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08956ddda07465449bfd48825f4f0f25e0351278ac974eaa659895d9d74f2c80" +dependencies = [ + "opendal-core", + "tokio", +] + +[[package]] +name = "opendal-service-azdls" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d564484a8f7d091827e825cfc91ed45bd48e64d262451ee041fe843db81bd8a" +dependencies = [ + "asyncband 0.6.7", + "base64 0.23.1", + "bytes", + "http", + "log", + "opendal-core", + "opendal-service-azure-common", + "quick-xml 0.41.0", + "reqsign-azure-storage", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "serde_json", +] + +[[package]] +name = "opendal-service-azure-common" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcc1bfdac4f54d9018c462dd32ad9e2f68fdf584f825c811550c8ffc87894cd" +dependencies = [ + "http", + "opendal-core", +] + +[[package]] +name = "opendal-service-fs" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7ef1e1c45f3f89282a59073897e0d685e51385fed0aea771714789525cff996" +dependencies = [ + "bytes", + "log", + "opendal-core", + "serde", + "tokio", + "xattr", +] + +[[package]] +name = "opendal-service-gcs" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da1f2a8c975fd22fea01f0409bcb8774f1ac02ad79863a3490098203121555d3" +dependencies = [ + "async-trait", + "bytes", + "http", + "log", + "opendal-core", + "percent-encoding", + "quick-xml 0.41.0", + "reqsign-core", + "reqsign-file-read-tokio", + "reqsign-google", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "opendal-service-oss" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e3ce7a2ceb925e0f28f545b169eb57d04cec6116aae94b8584d2bdbe7d456c6" +dependencies = [ + "bytes", + "http", + "log", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aliyun-oss", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", +] + +[[package]] +name = "opendal-service-s3" +version = "0.58.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64335f9f24ccb62ac36f1d976342b48611a75ba61979813a4f78a4ebd94de42" +dependencies = [ + "base64 0.23.1", + "bytes", + "crc-fast", + "http", + "log", + "md-5 0.11.0", + "opendal-core", + "quick-xml 0.41.0", + "reqsign-aws-v4", + "reqsign-core", + "reqsign-file-read-tokio", + "serde", + "url", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "ordered-float" +version = "2.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "ordered-multimap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49203cdcae0030493bad186b28da2fa25645fa276a51b6fec8010d281e02ef79" +dependencies = [ + "dlv-list", + "hashbrown 0.14.5", +] + +[[package]] +name = "ouroboros" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0f050db9c44b97a94723127e6be766ac5c340c48f2c4bb3ffa11713744be59" +dependencies = [ + "aliasable", + "ouroboros_macro", + "static_assertions", +] + +[[package]] +name = "ouroboros_macro" +version = "0.18.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c7028bdd3d43083f6d8d4d5187680d0d3560d54df4cc9d752005268b41e64d0" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "proc-macro2-diagnostics", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "parquet" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3d2ef0fb136e9341570f219ecb168902e76079c6d81ad555f0a4840b1e08a3" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "arrow-select", + "base64 0.23.1", + "brotli", + "bytes", + "chrono", + "flate2", + "futures", + "half", + "hashbrown 0.17.1", + "lz4_flex", + "num-bigint 0.5.1", + "num-integer", + "num-traits", + "object_store", + "parquet-variant", + "parquet-variant-compute", + "parquet-variant-json", + "ring", + "seq-macro", + "simdutf8", + "snap", + "tokio", + "twox-hash", + "zstd 0.13.3", +] + +[[package]] +name = "parquet-variant" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "587889e0dddd33f9eb536b20a489a20b2f556c4997ef1e09e49ae6df378d2ff0" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.2", + "num-traits", + "simdutf8", + "uuid", +] + +[[package]] +name = "parquet-variant-compute" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea15140dcafc6d1b4c17f1a12fcb9590ec8b82095cbd70925a5a8aa4c905e04a" +dependencies = [ + "arrow", + "arrow-schema", + "chrono", + "half", + "indexmap 2.14.2", + "parquet-variant", + "parquet-variant-json", + "serde_json", + "uuid", +] + +[[package]] +name = "parquet-variant-json" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf54eea151d1c8369a0b491fb727229149e4a09c9518e00aaaab1f816c25c8e" +dependencies = [ + "arrow-schema", + "base64 0.23.1", + "chrono", + "parquet-variant", + "serde_json", + "uuid", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", +] + +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap 2.14.2", + "serde", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2 0.10.9", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "polyval" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "opaque-debug", + "universal-hash", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.119", +] + +[[package]] +name = "prettyplease" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bfe0f4c752e450fc2faf62654f1c134747922825d5b04ca717b8874f41a40c0" +dependencies = [ + "proc-macro2", + "syn 3.0.9", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proc-macro2-diagnostics" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "version_check", + "yansi", +] + +[[package]] +name = "procfs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" +dependencies = [ + "bitflags 2.13.2", + "chrono", + "flate2", + "procfs-core", + "rustix", +] + +[[package]] +name = "procfs-core" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" +dependencies = [ + "bitflags 2.13.2", + "chrono", + "hex", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" +dependencies = [ + "heck 0.5.0", + "itertools 0.14.0", + "log", + "multimap", + "petgraph", + "prettyplease 0.2.37", + "prost", + "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", + "regex", + "syn 2.0.119", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "prost-types" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f94967dc7688f3054c7fac87473ffae4cc4c3904800e2d9f5b857246d8963b0a" +dependencies = [ + "prost", +] + +[[package]] +name = "psm" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dcd034599e63b970727f70d79e02d62390a4a84f7c6b827c27c46d5ac3fa622" +dependencies = [ + "ar_archive_writer", + "cc", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb" +dependencies = [ + "bitflags 2.13.2", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84bbb29c624230c4bd1047bbdb2aa47e41c860e9665ce62ba9504eebe91bf867" +dependencies = [ + "pulldown-cmark", +] + +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.39.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.41.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quick-xml" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand 0.10.3", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.2", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqsign-aliyun-oss" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cf3d65d37ac910073d921c1cc58c3e4e6c963795c6062eee196e6d19054dde" +dependencies = [ + "anyhow", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", +] + +[[package]] +name = "reqsign-aws-core" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38c902d8ec745bd9eadd1abbcbe227f4c1fc2926b2ec8929f89234af3114500" +dependencies = [ + "bytes", + "form_urlencoded", + "hex", + "http", + "log", + "percent-encoding", + "quick-xml 0.42.0", + "reqsign-core", + "rust-ini", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-aws-v4" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "906bfd028f96bfe73fdb892ce26f3b3f99b4f30feb9bd024be58bd6657f68b2b" +dependencies = [ + "bytes", + "http", + "log", + "quick-xml 0.42.0", + "reqsign-aws-core", + "reqsign-core", + "serde", +] + +[[package]] +name = "reqsign-azure-storage" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7260e6c1d89422ba62d624a32c491b561386fb364ac4bf0959c740e34b251a2e" +dependencies = [ + "anyhow", + "asyncband 0.7.3", + "base64 0.23.1", + "bytes", + "form_urlencoded", + "http", + "log", + "pem", + "percent-encoding", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", +] + +[[package]] +name = "reqsign-core" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fed85c436b6b920e4b36310f699a77771a91e38ae483df48447bd7d162671c" +dependencies = [ + "anyhow", + "asyncband 0.7.3", + "base64 0.23.1", + "bytes", + "hex", + "hmac 0.13.0", + "http", + "jiff", + "log", + "percent-encoding", + "rsa", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.1", + "windows-sys 0.61.2", +] + +[[package]] +name = "reqsign-file-read-tokio" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "586a8e198057773f9616901fd723a4bc186812b9765bc0f20368661d6169d906" +dependencies = [ + "anyhow", + "reqsign-core", + "tokio", +] + +[[package]] +name = "reqsign-google" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719e55931cc81b8dcd69f95f0a2def55617cb76cb3aa2f8c439e10682e46fa10" +dependencies = [ + "bytes", + "form_urlencoded", + "http", + "log", + "percent-encoding", + "reqsign-aws-v4", + "reqsign-core", + "rsa", + "serde", + "serde_json", + "tokio", +] + +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.4.2", + "web-sys", +] + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64 0.23.1", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http 0.6.11", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams 0.5.0", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "roaring" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bd8a37d17a58532776dcdf6041ce64929adca78e8489d5cacbafe99229d3e1" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid 0.9.6", + "digest 0.10.7", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2 0.10.9", + "signature", + "spki", + "subtle", + "zeroize", +] + +[[package]] +name = "rust-ini" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" +dependencies = [ + "cfg-if", + "ordered-multimap", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a073f5dc7469f984c52ad2752b63b0807745133b6de880b7b64c1ac4c48aec4" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2 0.10.9", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.13.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde-big-array" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11fc7cc2c76d73e0f27ee52abbd64eec84d46f370c88371120433196934e4b7f" +dependencies = [ + "serde", +] + +[[package]] +name = "serde-value" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c" +dependencies = [ + "ordered-float 2.10.1", + "serde", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "foldhash 0.2.0", + "indexmap 2.14.2", + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9adc193c780ef8f159aee8b61e2d5801aaa555e6eb0947fe45530ec506296f" +dependencies = [ + "base64 0.23.1", + "bs58", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.14.2", + "jiff", + "schemars 0.9.0", + "schemars 1.2.3", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e17bbc68e28663bbbb90df47e058aa7eda4fb445b89fe70457bb94fbccf6e49" +dependencies = [ + "darling 0.24.1", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap 2.14.2", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest 0.10.7", + "rand_core 0.6.4", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" + +[[package]] +name = "snap" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "199905e6153d6405f9728fe44daace35f8f837bbf830bb6e85fbd5828709a886" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" + +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", + "recursive", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "stacker" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.61.2", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", +] + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros 0.28.0", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "thread-id" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2010d27add3f3240c1fef7959f46c814487b216baee662af53be645ba7831c07" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + +[[package]] +name = "tinystr" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.54.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-util" +version = "0.7.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tonic" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1fa9b1a51188f3266a823208078315811269d06d272c3a61be0b9a194194ff" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c07bd4e9f99e2a3ed8bebad596ae02a94e53745b68acc6900e22c3475c8bc44" +dependencies = [ + "prettyplease 0.2.37", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tonic-prost" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63c616ffaa648f7c5dd5657e97d9a285fcb44ff3b70da761d436f5a13ff7be6" +dependencies = [ + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05d8957fac6b3542999796cb4449d45da790751962f974a457e5e00c43d78c" +dependencies = [ + "prettyplease 0.2.37", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.119", + "tempfile", + "tonic-build", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 2.14.2", + "pin-project-lite", + "slab", + "sync_wrapper", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-http" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a05a66a4fdd61cbbe0a1d755ffe0ca6aba159dd4820936a0ff8a8278245b9c" +dependencies = [ + "bitflags 2.13.2", + "bytes", + "http", + "percent-encoding", + "pin-project-lite", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" +dependencies = [ + "rand 0.10.3", +] + +[[package]] +name = "typed-builder" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + +[[package]] +name = "typemap-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68c24b707f02dd18f1e4ccceb9d49f2058c2fb86384ef9972592904d7a28867" +dependencies = [ + "unsafe-any-ors", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "typetag" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c90e86058a30d42a1a928dfb4b49bb33c98c3a2b4909492e6b0881cd94798ec2" +dependencies = [ + "erased-serde", + "inventory", + "once_cell", + "serde", + "typetag-impl", +] + +[[package]] +name = "typetag-impl" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f153acc4e99a5f2a5aefa09fb078be54e26271b2813f6041200b224c098d8328" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "unicase" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "universal-hash" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +dependencies = [ + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "unsafe-any-ors" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a303d30665362d9680d7d91d78b23f5f899504d4f08b3c4cf08d055d87c0ad" +dependencies = [ + "destructure_traitobject", +] + +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "serde_core", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" +dependencies = [ + "js-sys", + "tokio", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.9", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "xmlparser" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "yoke" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe 7.3.0", +] + +[[package]] +name = "zstd" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +dependencies = [ + "zstd-safe 8.2.0", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.2+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/native/native-executors/Cargo.toml b/native/native-executors/Cargo.toml new file mode 100644 index 00000000000..117cd7e3df4 --- /dev/null +++ b/native/native-executors/Cargo.toml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Comet Native Executors: the comet-scheduler and comet-executor binaries, built on Apache +# DataFusion Ballista. See https://github.com/apache/datafusion-comet/issues/7288. +# +# This is its own workspace, with its own lockfile, so that the main Comet build never depends +# on Ballista. It builds Comet's native crates by path together with the Ballista release on the +# same DataFusion major version. +[workspace] + +[package] +name = "comet-native-executors" +homepage = "https://datafusion.apache.org/comet" +repository = "https://github.com/apache/datafusion-comet" +authors = ["Apache DataFusion "] +description = "Apache DataFusion Comet: native executors and scheduler built on Apache DataFusion Ballista" +license = "Apache-2.0" +edition = "2021" +publish = false + +[dependencies] +arrow-flight = "59.2.0" +ballista-core = "55.0.0" +ballista-executor = { version = "55.0.0", default-features = false, features = ["arrow-ipc-optimizations"] } +ballista-scheduler = { version = "55.0.0", default-features = false } +datafusion = { version = "55.2.0", default-features = false } +# Comet's core crate, under the name of its library. Without the default hdfs-opendal feature, +# which links libjvm through libhdfs. +comet = { package = "datafusion-comet", path = "../core", default-features = false } +datafusion-comet-proto = { path = "../proto" } +datafusion-proto = "55.2.0" +env_logger = "0.11" +futures = "0.3" +log = "0.4" +prost = "0.14.3" +prost-types = "0.14.3" +tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync"] } +tonic = "0.14" + +[[bin]] +name = "comet-scheduler" +path = "src/bin/comet-scheduler.rs" + +[[bin]] +name = "comet-executor" +path = "src/bin/comet-executor.rs" diff --git a/native/native-executors/README.md b/native/native-executors/README.md new file mode 100644 index 00000000000..4224ab1de7a --- /dev/null +++ b/native/native-executors/README.md @@ -0,0 +1,87 @@ + + +# Comet Native Executors + +**Experimental.** Comet Native Executors run whole Spark queries without a JVM in the data plane. A +Spark driver plans each query with Catalyst and Comet, then submits it to a cluster of one +`comet-scheduler` and any number of `comet-executor` processes, built on +[Apache DataFusion Ballista](https://github.com/apache/datafusion-ballista). The design is in +[issue #7288](https://github.com/apache/datafusion-comet/issues/7288). + +This directory holds the cluster. The Spark driver side is in progress in +[#7289](https://github.com/apache/datafusion-comet/pull/7289). It is its own Cargo workspace, so the +main Comet build never depends on Ballista. + +## The scheduler's Flight service + +A client runs a query through the scheduler's Arrow Flight service: + +1. `GetFlightInfo`, with a `QueryRequest` (`native/proto/src/proto/native_executors.proto`) as the + command. The request is a stage graph of Comet native blocks connected by the exchanges between + them, stamped with the client's Comet version and plan format. The scheduler refuses a request + from another Comet build and plans each block with Comet's planner to check it. That builds no + object store for the block's files, so the scheduler uses no storage credentials. It then + builds a Ballista physical plan (`src/assembly.rs`), runs it on the Ballista scheduler it + wraps, and answers with one endpoint per result partition. +2. `DoGet` with each endpoint's ticket, at the endpoint's location: the executor that holds the + partition. A query's results stay there for `COMET_SCHEDULER_RESULT_RETENTION_SECONDS` after + the query ends (five minutes by default), whether or not a client has read them, and each + endpoint's expiration time says until when. + +A stage that several other stages read runs once for each of them, since Ballista's planner +doesn't share a stage between consumers yet. + +Each stage runs on the executors as a `CometFragmentExec` +(`native/core/src/execution/native_executors`), which plans the stage's block with Comet's planner +for each partition. Ballista moves the data between stages: + +| Exchange | Ballista plan | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | +| Hash | `RepartitionExec(Hash)`: Ballista's hash shuffle, with DataFusion's hash | +| Single partition, broadcast | `CoalescePartitionsExec` | +| Range | `CoalescePartitionsExec`, so a sort runs in one task ([ballista#2680](https://github.com/apache/datafusion-ballista/issues/2680)) | + +## Building and running + +```shell +CARGO_TARGET_DIR=native/target cargo build --manifest-path native/native-executors/Cargo.toml +native/target/debug/comet-scheduler & +native/target/debug/comet-executor & +``` + +Sharing `native/target` with the main workspace saves building the common dependencies twice. Both +binaries read their settings from environment variables, listed at the top of +`src/bin/comet-scheduler.rs` and `src/bin/comet-executor.rs`. + +## Testing + +```shell +CARGO_TARGET_DIR=native/target cargo test --manifest-path native/native-executors/Cargo.toml +``` + +`tests/assembly.rs` runs stage graphs on DataFusion alone. `tests/cluster.rs` starts the two +binaries and runs queries through the scheduler's Flight service, as a driver would. + +## Keeping it building + +This workspace builds Comet's native crates by path against the Ballista release on the same +DataFusion major version, and CI builds and tests it in the `linux-test-native-executors` job. When +Comet moves to a DataFusion major that Ballista hasn't released yet, the upgrade can turn that job +off until Ballista catches up. diff --git a/native/native-executors/src/assembly.rs b/native/native-executors/src/assembly.rs new file mode 100644 index 00000000000..8464d97fbc9 --- /dev/null +++ b/native/native-executors/src/assembly.rs @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Turns a driver's stage graph into a Ballista physical plan. + +use std::collections::HashMap; +use std::sync::Arc; + +use comet::execution::native_executors::{ + block_schema, partition_descriptor, CometFragmentExec, COMET_VERSION, +}; +use datafusion::common::{plan_datafusion_err, plan_err, Result}; +use datafusion::physical_expr::expressions::Column; +use datafusion::physical_expr::PhysicalExpr; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::ExecutionPlanProperties; +use datafusion::physical_plan::{ExecutionPlan, Partitioning}; +use datafusion_comet_proto::spark_expression::expr::ExprStruct; +use datafusion_comet_proto::spark_native_executors::{Fragment, QueryRequest, Stage, StageInput}; +use datafusion_comet_proto::spark_partitioning::partitioning::PartitioningStruct; +use datafusion_comet_proto::PROTO_FINGERPRINT; +use prost::Message; + +/// Checks that `request` comes from a driver running this Comet build. The plan format is not +/// yet stable across versions, and an executor would ignore any field it doesn't know. +pub fn check_version(request: &QueryRequest) -> Result<()> { + if request.comet_version != COMET_VERSION || request.proto_fingerprint != PROTO_FINGERPRINT { + return plan_err!( + "This Comet Native Executor cluster runs Comet {COMET_VERSION} with plan format \ + {PROTO_FINGERPRINT}, but the query comes from Comet {} with plan format {}. The \ + driver and the cluster must run the same Comet build.", + request.comet_version, + request.proto_fingerprint + ); + } + Ok(()) +} + +/// Builds the physical plan of `request`: a [`CometFragmentExec`] per stage, with a node that +/// Ballista's planner turns into a stage boundary on each edge. The root is the last stage. +/// +/// Each stage's block is planned with Comet's planner to check it before anything runs. That can +/// block, so don't call this on an async runtime's thread. A stage that several stages read runs +/// once for each of them, since Ballista's planner doesn't share a stage between consumers yet. +pub fn build_plan(request: QueryRequest) -> Result> { + let mut stages: Vec> = Vec::with_capacity(request.stages.len()); + for (index, stage) in request.stages.into_iter().enumerate() { + let fragment = build_stage(stage, &stages, &request.conf) + .map_err(|e| e.context(format!("Stage {index} of the query")))?; + stages.push(fragment); + } + stages + .pop() + .ok_or_else(|| plan_datafusion_err!("The query has no stages")) +} + +fn build_stage( + stage: Stage, + producers: &[Arc], + conf: &HashMap, +) -> Result> { + let block = stage + .plan + .as_ref() + .ok_or_else(|| plan_datafusion_err!("The stage has no plan"))? + .encode_to_vec(); + let mut inputs = Vec::with_capacity(stage.inputs.len()); + for input in &stage.inputs { + let producer = producers.get(input.producer as usize).ok_or_else(|| { + plan_datafusion_err!( + "The stage reads stage {}, which doesn't come before it", + input.producer + ) + })?; + inputs.push(exchange(input, Arc::clone(producer))?); + } + // A stage that reads exchanges has as many partitions as they deliver, which + // `CometFragmentExec` checks they agree on. + let num_partitions = stage + .inputs + .iter() + .zip(&inputs) + .find(|(input, _)| !input.broadcast) + .map(|(_, exchange)| exchange.output_partitioning().partition_count()) + .unwrap_or(stage.num_partitions as usize); + if num_partitions == 0 { + return plan_err!("The stage has no partitions"); + } + if !stage.partition_plans.is_empty() && stage.partition_plans.len() != num_partitions { + return plan_err!( + "The stage has {num_partitions} partitions but {} partition plans", + stage.partition_plans.len() + ); + } + // A stage that reads files carries its scans' schemas and options only in each partition's + // copy of the block, so check one of those. Checking reads none of its files. + let planned = block_schema(stage.partition_plans.first().unwrap_or(&block), conf)?; + if planned.fields().len() != stage.output_schema.len() { + return plan_err!( + "The stage's block produces {} columns but the stage declares {}", + planned.fields().len(), + stage.output_schema.len() + ); + } + + let mut children = vec![partition_descriptor( + num_partitions, + &stage.partition_plans, + )?]; + children.extend(inputs); + let fragment = Fragment { + // A stage that reads files runs each partition's own copy of the block instead. + block: if stage.partition_plans.is_empty() { + block + } else { + vec![] + }, + broadcast: stage.inputs.iter().map(|input| input.broadcast).collect(), + output_schema: stage.output_schema, + conf: conf.clone(), + }; + Ok(Arc::new(CometFragmentExec::try_new( + Arc::new(fragment), + children, + )?)) +} + +/// The exchange that carries `producer`'s output to a stage, as a node Ballista's planner turns +/// into a stage boundary. +/// +/// A hash exchange partitions with DataFusion's hash function, not Spark's Murmur3. That is +/// consistent within a query, which is all joins, aggregates and windows need, but a plan that +/// relies on Spark's own placement of rows, such as a join with a bucketed table, would be wrong. +fn exchange( + input: &StageInput, + producer: Arc, +) -> Result> { + if input.broadcast { + return Ok(Arc::new(CoalescePartitionsExec::new(producer))); + } + let partitioning = input + .partitioning + .as_ref() + .and_then(|p| p.partitioning_struct.as_ref()); + match partitioning { + Some(PartitioningStruct::HashPartition(hash)) => { + let schema = producer.schema(); + let keys = hash + .hash_expression + .iter() + .map(|expr| match expr.expr_struct.as_ref() { + Some(ExprStruct::Bound(bound)) + if (bound.index as usize) < schema.fields().len() => + { + let index = bound.index as usize; + Ok(Arc::new(Column::new(schema.field(index).name(), index)) + as Arc) + } + _ => plan_err!("Hash partitioning keys must be columns of the stage's input"), + }) + .collect::>>()?; + Ok(Arc::new(RepartitionExec::try_new( + producer, + Partitioning::Hash(keys, hash.num_partitions as usize), + )?)) + } + // Ballista can't plan a range shuffle for a physical plan yet + // (https://github.com/apache/datafusion-ballista/issues/2680), so the consuming stage + // reads a range exchange as one partition too. That sorts correctly, in a single task. + Some(PartitioningStruct::SinglePartition(_) | PartitioningStruct::RangePartition(_)) => { + Ok(Arc::new(CoalescePartitionsExec::new(producer))) + } + other => plan_err!("Comet Native Executors don't support this exchange yet: {other:?}"), + } +} diff --git a/native/native-executors/src/bin/comet-executor.rs b/native/native-executors/src/bin/comet-executor.rs new file mode 100644 index 00000000000..f1b7cec6a99 --- /dev/null +++ b/native/native-executors/src/bin/comet-executor.rs @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A Comet executor: Ballista's executor, able to run the stages of Comet queries with Comet's +//! planner, operators and expressions. It needs no JVM. +//! +//! Configured through environment variables, with Ballista's defaults otherwise: +//! - `COMET_EXECUTOR_BIND_HOST` (default `127.0.0.1`) +//! - `COMET_EXECUTOR_EXTERNAL_HOST`: the host that drivers and other executors reach this +//! executor at (default: the address the scheduler sees it connect from) +//! - `COMET_EXECUTOR_PORT`: the Flight port that serves shuffle and result data (default `50051`) +//! - `COMET_EXECUTOR_GRPC_PORT` (default `50052`) +//! - `COMET_SCHEDULER_HOST` (default `localhost`) and `COMET_SCHEDULER_PORT` (default `50050`) +//! - `COMET_EXECUTOR_TASKS`: concurrent tasks (default: the number of CPUs) +//! - `COMET_EXECUTOR_WORK_DIR`: where shuffle and spill files go (default: a temporary directory) + +use std::sync::Arc; + +use ballista_executor::executor_process::{start_executor_process, ExecutorProcessConfig}; +use comet_native_executors::codec::CometPhysicalCodec; +use comet_native_executors::{env_var, init_logging}; + +#[tokio::main] +async fn main() -> Result<(), Box> { + init_logging(); + let mut config = ExecutorProcessConfig { + override_physical_codec: Some(Arc::new(CometPhysicalCodec::default())), + ..Default::default() + }; + if let Some(host) = env_var("COMET_EXECUTOR_BIND_HOST")? { + config.bind_host = host; + } + config.external_host = env_var("COMET_EXECUTOR_EXTERNAL_HOST")?; + if let Some(port) = env_var("COMET_EXECUTOR_PORT")? { + config.port = port; + } + if let Some(port) = env_var("COMET_EXECUTOR_GRPC_PORT")? { + config.grpc_port = port; + } + if let Some(host) = env_var("COMET_SCHEDULER_HOST")? { + config.scheduler_host = host; + } + if let Some(port) = env_var("COMET_SCHEDULER_PORT")? { + config.scheduler_port = port; + } + if let Some(tasks) = env_var("COMET_EXECUTOR_TASKS")? { + config.vcores = tasks; + } + config.work_dir = env_var("COMET_EXECUTOR_WORK_DIR")?; + + start_executor_process(Arc::new(config)).await?; + Ok(()) +} diff --git a/native/native-executors/src/bin/comet-scheduler.rs b/native/native-executors/src/bin/comet-scheduler.rs new file mode 100644 index 00000000000..d1506f335ac --- /dev/null +++ b/native/native-executors/src/bin/comet-scheduler.rs @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! A Comet scheduler: Ballista's scheduler, which Comet executors register with, wrapped in the +//! Arrow Flight service that Spark drivers submit queries to. +//! +//! Configured through environment variables, with Ballista's defaults otherwise: +//! - `COMET_SCHEDULER_BIND_HOST` (default `127.0.0.1`) +//! - `COMET_SCHEDULER_PORT`: Ballista's gRPC port, for executors (default `50050`) +//! - `COMET_SCHEDULER_FLIGHT_PORT`: the Flight port, for drivers (default `50060`) +//! - `COMET_SCHEDULER_RESULT_RETENTION_SECONDS`: how long a query's results stay readable after +//! the query ends, whether or not a driver has read them (default `300`). `0` keeps them. + +use std::net::SocketAddr; +use std::sync::Arc; + +use arrow_flight::flight_service_server::FlightServiceServer; +use ballista_scheduler::cluster::BallistaCluster; +use ballista_scheduler::config::SchedulerConfig; +use ballista_scheduler::scheduler_process::{create_scheduler, start_grpc_service}; +use comet_native_executors::codec::CometPhysicalCodec; +use comet_native_executors::frontend::CometFlightService; +use comet_native_executors::{env_var, init_logging}; +use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; +use log::info; + +#[tokio::main] +async fn main() -> Result<(), Box> { + init_logging(); + let mut config = SchedulerConfig { + override_physical_codec: Some(Arc::new(CometPhysicalCodec::default())), + ..Default::default() + }; + if let Some(host) = env_var("COMET_SCHEDULER_BIND_HOST")? { + config.bind_host = host; + } + if let Some(port) = env_var("COMET_SCHEDULER_PORT")? { + config.bind_port = port; + } + if let Some(seconds) = env_var("COMET_SCHEDULER_RESULT_RETENTION_SECONDS")? { + config.finished_job_data_clean_up_interval_seconds = seconds; + } + let flight_port: u16 = env_var("COMET_SCHEDULER_FLIGHT_PORT")?.unwrap_or(50060); + let grpc_addr: SocketAddr = format!("{}:{}", config.bind_host, config.bind_port).parse()?; + let flight_addr = SocketAddr::new(grpc_addr.ip(), flight_port); + let config = Arc::new(config); + config.validate()?; + + let cluster = BallistaCluster::new_from_config(&config).await?; + let scheduler = + create_scheduler::(cluster, Arc::clone(&config)).await?; + + info!("Comet scheduler: executors connect to {grpc_addr}, drivers to {flight_addr}"); + let grpc = tokio::spawn(start_grpc_service(grpc_addr, scheduler.clone())); + let flight = tonic::transport::Server::builder() + .add_service( + FlightServiceServer::new(CometFlightService::new(scheduler)) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX), + ) + .serve(flight_addr); + tokio::select! { + result = grpc => result??, + result = flight => result?, + } + Ok(()) +} diff --git a/native/native-executors/src/codec.rs b/native/native-executors/src/codec.rs new file mode 100644 index 00000000000..b8b9a7f9320 --- /dev/null +++ b/native/native-executors/src/codec.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Serializes Comet's operators inside the stage and task plans Ballista sends to executors. + +use std::sync::Arc; + +use ballista_core::serde::BallistaPhysicalExtensionCodec; +use comet::execution::native_executors::CometFragmentExec; +use datafusion::common::{internal_datafusion_err, Result}; +use datafusion::execution::TaskContext; +use datafusion::physical_plan::ExecutionPlan; +use datafusion_comet_proto::spark_native_executors::Fragment; +use datafusion_proto::physical_plan::{PhysicalExtensionCodec, PhysicalProtoConverterExtension}; +use prost::Message; + +/// Prefix of an encoded [`CometFragmentExec`], which tells it apart from Ballista's own nodes. +/// The rest is its `Fragment`; its children travel separately, as the plan's. +const FRAGMENT_MAGIC: &[u8] = b"COMET_FRAGMENT\0"; + +/// Ballista's physical extension codec, extended with Comet's operators. +#[derive(Debug, Default)] +pub struct CometPhysicalCodec { + ballista: BallistaPhysicalExtensionCodec, +} + +impl PhysicalExtensionCodec for CometPhysicalCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + ctx: &TaskContext, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result> { + let Some(buf) = buf.strip_prefix(FRAGMENT_MAGIC) else { + return self.ballista.try_decode(buf, inputs, ctx, proto_converter); + }; + let fragment = Fragment::decode(buf) + .map_err(|e| internal_datafusion_err!("Cannot decode a CometFragmentExec: {e}"))?; + Ok(Arc::new(CometFragmentExec::try_new( + Arc::new(fragment), + inputs.to_vec(), + )?)) + } + + fn try_encode( + &self, + node: Arc, + buf: &mut Vec, + proto_converter: &dyn PhysicalProtoConverterExtension, + ) -> Result<()> { + let Some(fragment) = node.downcast_ref::() else { + return self.ballista.try_encode(node, buf, proto_converter); + }; + buf.extend_from_slice(FRAGMENT_MAGIC); + fragment + .fragment() + .encode(buf) + .map_err(|e| internal_datafusion_err!("Cannot encode a CometFragmentExec: {e}")) + } +} diff --git a/native/native-executors/src/frontend.rs b/native/native-executors/src/frontend.rs new file mode 100644 index 00000000000..b6077669819 --- /dev/null +++ b/native/native-executors/src/frontend.rs @@ -0,0 +1,259 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The Arrow Flight service that Spark drivers submit queries to. +//! +//! `GetFlightInfo` takes a [`QueryRequest`] as its command, runs the query, and returns one +//! endpoint per result partition. Each endpoint names the executor that holds the partition and +//! carries an opaque ticket for its `DoGet`, so the driver never speaks Ballista's protocol. It +//! also says when the partition goes: Ballista deletes a query's results a set time after the +//! query ends, whether or not they have been read (see the comet-scheduler binary). + +use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; + +use arrow_flight::flight_service_server::FlightService; +use arrow_flight::{ + Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, + HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, +}; +use ballista_core::extension::SessionConfigExt; +use ballista_core::serde::protobuf::{job_status, PartitionLocation, SuccessfulJob}; +use ballista_core::serde::scheduler::{Action as BallistaAction, ShuffleFileKind}; +use ballista_scheduler::scheduler_server::SchedulerServer; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::SessionConfig; +use datafusion_comet_proto::spark_native_executors::QueryRequest; +use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode}; +use futures::stream::BoxStream; +use log::info; +use prost::Message; +use prost_types::Timestamp; +use tonic::{Request, Response, Status, Streaming}; + +use crate::assembly::{build_plan, check_version}; + +/// The Ballista session every query runs in. Every query uses Ballista's default settings, and +/// executors cache a runtime per session, so sharing one lets queries share those runtimes. +const SESSION_ID: &str = "comet"; + +/// The Flight service of a Comet scheduler, which runs queries on the Ballista scheduler it +/// wraps. +pub struct CometFlightService { + scheduler: SchedulerServer, +} + +impl CometFlightService { + pub fn new(scheduler: SchedulerServer) -> Self { + Self { scheduler } + } + + /// Runs `plan` to completion and returns its result: where its output partitions are, in + /// partition order, and when it ended. + async fn run(&self, plan: Arc) -> Result { + let ctx = self + .scheduler + .state + .session_manager + .create_or_update_session(SESSION_ID, &SessionConfig::new_with_ballista()) + .await + .map_err(|e| Status::internal(format!("Cannot create a session: {e}")))?; + let (sender, mut receiver) = tokio::sync::mpsc::channel(16); + let job_id = self + .scheduler + .submit_physical_plan("comet", ctx, plan, Some(sender)) + .await + .map_err(|e| Status::internal(format!("Cannot submit the query: {e}")))?; + info!("Submitted query as job {job_id}"); + while let Some(status) = receiver.recv().await { + match status.status { + Some(job_status::Status::Successful(mut job)) => { + job.partition_location.sort_by_key(|location| { + let partition = location.partition_id.as_ref().map(|p| p.partition_id); + (partition, location.map_partition_id) + }); + return Ok(job); + } + Some(job_status::Status::Failed(job)) => { + return Err(Status::internal(format!( + "Job {job_id} failed: {}", + job.error + ))) + } + _ => {} + } + } + Err(Status::internal(format!( + "Job {job_id} ended without a final status" + ))) + } + + /// When the executors delete the results of a query that ended at `ended_at`, in milliseconds + /// since the epoch. Ballista deletes them that long after the query ends, unless the scheduler + /// keeps them. Ballista starts that wait only after it records the end, so they are still + /// there until this time. + fn results_expire_at(&self, ended_at: u64) -> Option { + let retention = self + .scheduler + .state + .config + .finished_job_data_clean_up_interval_seconds; + if retention == 0 { + return None; + } + UNIX_EPOCH + .checked_add(Duration::from_millis(ended_at))? + .checked_add(Duration::from_secs(retention)) + .map(Timestamp::from) + } +} + +/// The Flight endpoint for a result partition: the executor's location, a ticket the executor's +/// Flight service reads as a Ballista partition fetch, and when the partition is deleted. +fn endpoint( + location: &PartitionLocation, + expires_at: Option, +) -> Result { + let executor = location + .executor_meta + .as_ref() + .ok_or_else(|| Status::internal("A result partition has no executor"))?; + let partition = location + .partition_id + .as_ref() + .ok_or_else(|| Status::internal("A result partition has no ID"))?; + let action = BallistaAction::FetchPartition { + job_id: partition.job_id.clone().into(), + stage_id: partition.stage_id as usize, + partition_id: partition.partition_id as usize, + host: executor.host.clone(), + port: executor.port as u16, + file_id: location.file_id, + layout: location.layout(), + file_kind: ShuffleFileKind::Data, + byte_ranges: vec![], + }; + let action: ballista_core::serde::protobuf::Action = action + .try_into() + .map_err(|e| Status::internal(format!("Cannot encode a partition fetch: {e}")))?; + let mut endpoint = FlightEndpoint::new() + .with_ticket(Ticket::new(action.encode_to_vec())) + .with_location(format!("grpc+tcp://{}:{}", executor.host, executor.port)); + endpoint.expiration_time = expires_at; + Ok(endpoint) +} + +fn unimplemented(method: &str) -> Result { + Err(Status::unimplemented(format!( + "A Comet scheduler doesn't implement {method}" + ))) +} + +#[tonic::async_trait] +impl FlightService for CometFlightService { + type HandshakeStream = BoxStream<'static, Result>; + type ListFlightsStream = BoxStream<'static, Result>; + type DoGetStream = BoxStream<'static, Result>; + type DoPutStream = BoxStream<'static, Result>; + type DoExchangeStream = BoxStream<'static, Result>; + type DoActionStream = BoxStream<'static, Result>; + type ListActionsStream = BoxStream<'static, Result>; + + async fn get_flight_info( + &self, + request: Request, + ) -> Result, Status> { + let descriptor = request.into_inner(); + let query = QueryRequest::decode(descriptor.cmd.as_ref()) + .map_err(|e| Status::invalid_argument(format!("Cannot decode the query: {e}")))?; + check_version(&query).map_err(|e| Status::failed_precondition(e.to_string()))?; + let plan = tokio::task::spawn_blocking(move || build_plan(query)) + .await + .map_err(|e| Status::internal(e.to_string()))? + .map_err(|e| Status::invalid_argument(e.to_string()))?; + let job = self.run(plan).await?; + let expires_at = self.results_expire_at(job.ended_at); + let mut info = FlightInfo::new(); + for location in &job.partition_location { + info = info.with_endpoint(endpoint(location, expires_at)?); + } + Ok(Response::new(info)) + } + + async fn handshake( + &self, + _request: Request>, + ) -> Result, Status> { + unimplemented("Handshake") + } + + async fn list_flights( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("ListFlights") + } + + async fn poll_flight_info( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("PollFlightInfo") + } + + async fn get_schema( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("GetSchema") + } + + async fn do_get( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("DoGet; fetch result partitions from the executors") + } + + async fn do_put( + &self, + _request: Request>, + ) -> Result, Status> { + unimplemented("DoPut") + } + + async fn do_exchange( + &self, + _request: Request>, + ) -> Result, Status> { + unimplemented("DoExchange") + } + + async fn do_action( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("DoAction") + } + + async fn list_actions( + &self, + _request: Request, + ) -> Result, Status> { + unimplemented("ListActions") + } +} diff --git a/native/native-executors/src/lib.rs b/native/native-executors/src/lib.rs new file mode 100644 index 00000000000..02251c83cb2 --- /dev/null +++ b/native/native-executors/src/lib.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Comet Native Executors: a Comet scheduler and executors built on Apache DataFusion Ballista, +//! which run whole Spark queries planned by a Spark driver with no JVM. See +//! . +//! +//! A driver submits a query as a stage graph of Comet native blocks (a +//! [`QueryRequest`](datafusion_comet_proto::spark_native_executors::QueryRequest)) to the +//! scheduler's Arrow Flight service ([`frontend`]). The scheduler checks it, turns it into a +//! Ballista physical plan ([`assembly`]) and runs it on the executors, which rebuild each block +//! with Comet's planner. The driver then fetches the result partitions from the executors. + +use std::fmt::Display; +use std::str::FromStr; + +pub mod assembly; +pub mod codec; +pub mod frontend; + +/// The value of the environment variable `key`, if it is set, for the binaries' settings. +pub fn env_var(key: &str) -> Result, String> +where + T::Err: Display, +{ + std::env::var(key) + .ok() + .map(|value| { + value + .parse() + .map_err(|e| format!("Invalid {key} {value:?}: {e}")) + }) + .transpose() +} + +/// Logs at `info` unless `RUST_LOG` says otherwise. +pub fn init_logging() { + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); +} diff --git a/native/native-executors/tests/assembly.rs b/native/native-executors/tests/assembly.rs new file mode 100644 index 00000000000..63fcd103b9c --- /dev/null +++ b/native/native-executors/tests/assembly.rs @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Runs assembled stage graphs on DataFusion alone, without Ballista: each exchange runs as a +//! DataFusion repartition in this process. + +mod common; + +use comet_native_executors::assembly::build_plan; +use common::*; +use datafusion::arrow::datatypes::{DataType, Int32Type, Int64Type}; +use datafusion::physical_plan::{collect, ExecutionPlan, ExecutionPlanProperties}; +use datafusion::prelude::SessionContext; +use datafusion_comet_proto::spark_expression::{ + expr::ExprStruct, literal::Value, Expr, IfExpr, Literal, Subquery, +}; +use datafusion_comet_proto::spark_native_executors::{QueryRequest, Stage}; +use datafusion_comet_proto::spark_operator::{ + operator::OpStruct, NativeScan, NativeScanCommon, Operator, Projection, SparkFilePartition, + SparkPartitionedFile, +}; +use prost::Message; +use std::sync::Arc; + +async fn assemble(request: QueryRequest) -> datafusion::common::Result> { + tokio::task::spawn_blocking(move || build_plan(request)) + .await + .unwrap() +} + +/// Runs `request` and returns its single column, sorted. +async fn run(request: QueryRequest) -> Vec { + let plan = assemble(request).await.unwrap(); + let batches = collect(plan, SessionContext::new().task_ctx()) + .await + .unwrap(); + sorted_values::(&batches) +} + +#[tokio::test(flavor = "multi_thread")] +async fn leaf_stage_runs_every_partition() { + let request = query(vec![range_stage(100, 4)]); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn hash_exchange_moves_every_row() { + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, hash_on_first_column(3), 3), + ]); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn single_partition_exchange_collects_every_row() { + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, single_partition(), 1), + ]); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn range_exchange_is_read_as_one_partition() { + // The driver asks for the exchange's partition count. + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, range_partitioning(4), 4), + ]); + let plan = assemble(request.clone()).await.unwrap(); + assert_eq!(plan.output_partitioning().partition_count(), 1); + assert_eq!(run(request).await, (0..100).collect::>()); +} + +#[tokio::test(flavor = "multi_thread")] +async fn partition_plans_must_match_the_partitions() { + let mut stage = range_stage(100, 4); + let block = stage.plan.as_ref().unwrap().encode_to_vec(); + stage.partition_plans = vec![block.clone(), block]; + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!( + error + .to_string() + .contains("has 4 partitions but 2 partition plans"), + "{error}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn reading_stage_has_as_many_partitions_as_its_exchange() { + // The partition count a driver gives a stage that reads an exchange doesn't matter. + let request = query(vec![ + range_stage(100, 4), + reading_stage(0, hash_on_first_column(3), 5), + ]); + let plan = assemble(request).await.unwrap(); + assert_eq!(plan.output_partitioning().partition_count(), 3); +} + +#[tokio::test(flavor = "multi_thread")] +async fn output_is_cast_to_the_types_the_stage_declares() { + // A range produces longs. A stage declaring an int column gets ints, as a Spark task's + // shuffle writer casts a column whose native type drifted from the one Spark declared. + let mut stage = range_stage(100, 4); + stage.output_schema = vec![field(int_type())]; + let plan = assemble(query(vec![stage])).await.unwrap(); + assert_eq!(plan.schema().field(0).data_type(), &DataType::Int32); + let batches = collect(plan, SessionContext::new().task_ctx()) + .await + .unwrap(); + assert_eq!( + sorted_values::(&batches), + (0..100).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn batches_have_the_schema_the_stage_declares() { + // `if(true, id, null)` has no nulls, but the block's output leaves it nullable. Spark can + // declare such a column not null, as it does a join key that a filter has checked, and a + // distributed engine checks each batch a stage sends against the stage's schema. + let literal = |value: Option, data_type| Expr { + expr_struct: Some(ExprStruct::Literal(Literal { + is_null: value.is_none(), + value, + datatype: Some(data_type), + })), + ..Default::default() + }; + let column = Expr { + expr_struct: Some(ExprStruct::If(Box::new(IfExpr { + if_expr: Some(Box::new(literal(Some(Value::BoolVal(true)), bool_type()))), + true_expr: Some(Box::new(bound(0, long_type()))), + false_expr: Some(Box::new(literal(None, long_type()))), + }))), + ..Default::default() + }; + let mut stage = range_stage(100, 4); + stage.plan = Some(Operator { + op_struct: Some(OpStruct::Projection(Projection { + project_list: vec![column], + })), + children: vec![stage.plan.take().unwrap()], + ..Default::default() + }); + stage.output_schema[0].nullable = false; + let plan = assemble(query(vec![stage])).await.unwrap(); + let batches = collect(Arc::clone(&plan), SessionContext::new().task_ctx()) + .await + .unwrap(); + assert!(batches.iter().all(|batch| batch.schema() == plan.schema())); + assert_eq!( + sorted_values::(&batches), + (0..100).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn declared_columns_must_match_the_block() { + let mut stage = range_stage(100, 4); + stage.output_schema = vec![field(long_type()), field(long_type())]; + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!( + error + .to_string() + .contains("produces 1 columns but the stage declares 2"), + "{error}" + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn stage_without_partitions_is_rejected() { + let mut stage = range_stage(100, 4); + stage.num_partitions = 0; + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!(error.to_string().contains("no partitions"), "{error}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn block_that_calls_into_the_jvm_is_rejected() { + // A scalar subquery reads its value from the JVM, which a native executor doesn't have. + let subquery = Expr { + expr_struct: Some(ExprStruct::Subquery(Subquery { + id: 1, + datatype: Some(long_type()), + })), + ..Default::default() + }; + let mut stage = range_stage(100, 4); + stage.plan = Some(Operator { + op_struct: Some(OpStruct::Projection(Projection { + project_list: vec![subquery], + })), + children: vec![stage.plan.take().unwrap()], + ..Default::default() + }); + let error = assemble(query(vec![stage])).await.unwrap_err(); + assert!(error.to_string().contains("calls into the JVM"), "{error}"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn file_stage_is_checked_without_its_object_store() { + // The scheduler checks a stage without reading its files, so it builds no object store and + // uses no storage credentials. Nothing can build a store for this URL, so only the stage's + // run finds that out. + let scan = |partitioned_file| Operator { + op_struct: Some(OpStruct::NativeScan(NativeScan { + common: Some(NativeScanCommon { + required_schema: vec![field(long_type())], + data_schema: vec![field(long_type())], + projection_vector: vec![0], + session_timezone: "UTC".to_string(), + ..Default::default() + }), + file_partition: Some(SparkFilePartition { partitioned_file }), + ..Default::default() + })), + ..Default::default() + }; + let file = SparkPartitionedFile { + file_path: "no-such-store://bucket/part-0.parquet".to_string(), + length: 16, + file_size: 16, + ..Default::default() + }; + let stage = Stage { + plan: Some(scan(vec![])), + partition_plans: vec![scan(vec![file]).encode_to_vec()], + num_partitions: 1, + output_schema: vec![field(long_type())], + ..Default::default() + }; + let plan = assemble(query(vec![stage])).await.unwrap(); + let error = collect(plan, SessionContext::new().task_ctx()) + .await + .unwrap_err(); + assert!(error.to_string().contains("no-such-store"), "{error}"); +} diff --git a/native/native-executors/tests/common/mod.rs b/native/native-executors/tests/common/mod.rs new file mode 100644 index 00000000000..5dc6b530cf7 --- /dev/null +++ b/native/native-executors/tests/common/mod.rs @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Stage graphs for the tests, built from Comet operators that need no files: `RangeScan` leaf +//! stages, and stages that return their input. + +// Each test file uses its own subset. +#![allow(dead_code)] + +use comet::execution::native_executors::COMET_VERSION; +use datafusion::arrow::array::{AsArray, RecordBatch}; +use datafusion::arrow::datatypes::ArrowPrimitiveType; +use datafusion_comet_proto::spark_expression::data_type::DataTypeId; +use datafusion_comet_proto::spark_expression::{self, expr::ExprStruct, BoundReference, Expr}; +use datafusion_comet_proto::spark_native_executors::{QueryRequest, Stage, StageInput}; +use datafusion_comet_proto::spark_operator::{ + operator::OpStruct, Operator, RangeScan, Scan, SparkStructField, +}; +use datafusion_comet_proto::spark_partitioning::{ + partitioning::PartitioningStruct, HashPartition, Partitioning, RangePartition, SinglePartition, +}; +use datafusion_comet_proto::PROTO_FINGERPRINT; + +/// A query of `stages` from a driver of this Comet build. +pub fn query(stages: Vec) -> QueryRequest { + QueryRequest { + comet_version: COMET_VERSION.to_string(), + proto_fingerprint: PROTO_FINGERPRINT.to_string(), + stages, + ..Default::default() + } +} + +fn data_type(type_id: DataTypeId) -> spark_expression::DataType { + spark_expression::DataType { + type_id: type_id as i32, + type_info: None, + } +} + +pub fn long_type() -> spark_expression::DataType { + data_type(DataTypeId::Int64) +} + +pub fn int_type() -> spark_expression::DataType { + data_type(DataTypeId::Int32) +} + +pub fn bool_type() -> spark_expression::DataType { + data_type(DataTypeId::Bool) +} + +/// Column `index` of an operator's input, of type `data_type`. +pub fn bound(index: i32, data_type: spark_expression::DataType) -> Expr { + Expr { + expr_struct: Some(ExprStruct::Bound(BoundReference { + index, + datatype: Some(data_type), + })), + ..Default::default() + } +} + +/// A nullable column `id` of type `data_type`, as a stage declares its output. +pub fn field(data_type: spark_expression::DataType) -> SparkStructField { + SparkStructField { + name: "id".to_string(), + data_type: Some(data_type), + nullable: true, + ..Default::default() + } +} + +/// A stage that runs `range(0, num_elements)` in `num_slices` partitions, like `spark.range`. +pub fn range_stage(num_elements: i64, num_slices: i32) -> Stage { + Stage { + plan: Some(Operator { + op_struct: Some(OpStruct::RangeScan(RangeScan { + start: 0, + step: 1, + num_elements, + num_slices, + })), + ..Default::default() + }), + num_partitions: num_slices as u32, + output_schema: vec![field(long_type())], + ..Default::default() + } +} + +/// A stage that returns what stage `producer` sends it through an exchange with `partitioning`. +/// `num_partitions` is the count the driver asks for, which is the exchange's. +pub fn reading_stage(producer: u32, partitioning: Partitioning, num_partitions: u32) -> Stage { + Stage { + plan: Some(Operator { + op_struct: Some(OpStruct::Scan(Scan { + fields: vec![long_type()], + source: "test".to_string(), + })), + ..Default::default() + }), + inputs: vec![StageInput { + producer, + partitioning: Some(partitioning), + broadcast: false, + }], + num_partitions, + output_schema: vec![field(long_type())], + ..Default::default() + } +} + +pub fn hash_on_first_column(num_partitions: i32) -> Partitioning { + Partitioning { + partitioning_struct: Some(PartitioningStruct::HashPartition(HashPartition { + hash_expression: vec![bound(0, long_type())], + num_partitions, + })), + } +} + +pub fn single_partition() -> Partitioning { + Partitioning { + partitioning_struct: Some(PartitioningStruct::SinglePartition(SinglePartition {})), + } +} + +/// A range exchange as a driver sends it: its sort keys don't matter here, and it has no bounds. +pub fn range_partitioning(num_partitions: i32) -> Partitioning { + Partitioning { + partitioning_struct: Some(PartitioningStruct::RangePartition(RangePartition { + num_partitions, + ..Default::default() + })), + } +} + +/// The values of the batches' single column, of type `T`, sorted. +pub fn sorted_values(batches: &[RecordBatch]) -> Vec +where + T::Native: Ord, +{ + let mut values: Vec = batches + .iter() + .flat_map(|batch| batch.column(0).as_primitive::().values().to_vec()) + .collect(); + values.sort_unstable(); + values +} From 47ccd3fb2762ce082fcbb7005cbfed4facbc939f Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 4/8] test: run queries on comet-scheduler and comet-executor through Flight tests/cluster.rs starts the two binaries, submits stage graphs to the scheduler's Flight service as a driver would, and reads every result partition from the executors with DoGet. It covers a hash shuffle, a range exchange, and a query from another Comet build, which the scheduler refuses. --- native/native-executors/tests/cluster.rs | 265 +++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 native/native-executors/tests/cluster.rs diff --git a/native/native-executors/tests/cluster.rs b/native/native-executors/tests/cluster.rs new file mode 100644 index 00000000000..dee8dcaa7b5 --- /dev/null +++ b/native/native-executors/tests/cluster.rs @@ -0,0 +1,265 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Runs queries on a cluster of the comet-scheduler and comet-executor binaries, through the +//! scheduler's Arrow Flight service, as a Spark driver does. + +mod common; + +use std::future::Future; +use std::net::{TcpListener, TcpStream}; +use std::path::PathBuf; +use std::process::{Child, Command, Stdio}; +use std::sync::Mutex; +use std::time::{Duration, Instant, SystemTime}; + +use arrow_flight::error::FlightError; +use arrow_flight::{FlightClient, FlightDescriptor, FlightEndpoint, FlightInfo}; +use common::*; +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::Int64Type; +use datafusion_comet_proto::spark_native_executors::QueryRequest; +use futures::TryStreamExt; +use prost::Message; +use tonic::transport::Endpoint; +use tonic::Code; + +/// A comet-scheduler and a comet-executor, stopped when dropped. +struct Cluster { + processes: Vec, + work_dir: PathBuf, + flight_url: String, +} + +/// Held while a cluster starts, so that two tests don't pick the same free port in between. +static STARTING: Mutex<()> = Mutex::new(()); + +impl Cluster { + fn start() -> Self { + Self::start_with(&[]) + } + + /// Starts a cluster whose scheduler also gets the settings `scheduler_env`. + fn start_with(scheduler_env: &[(&str, String)]) -> Self { + let _starting = STARTING.lock().unwrap_or_else(|e| e.into_inner()); + let scheduler_port = free_port(); + let flight_port = free_port(); + let executor_port = free_port(); + let executor_grpc_port = free_port(); + let mut cluster = Cluster { + processes: vec![], + work_dir: std::env::temp_dir().join(format!( + "comet-native-executors-{}-{flight_port}", + std::process::id() + )), + flight_url: format!("http://127.0.0.1:{flight_port}"), + }; + let mut env = vec![ + ("COMET_SCHEDULER_BIND_HOST", "127.0.0.1".to_string()), + ("COMET_SCHEDULER_PORT", scheduler_port.to_string()), + ("COMET_SCHEDULER_FLIGHT_PORT", flight_port.to_string()), + ]; + env.extend_from_slice(scheduler_env); + cluster.spawn(env!("CARGO_BIN_EXE_comet-scheduler"), &env); + wait_for(scheduler_port); + wait_for(flight_port); + let work_dir = cluster.work_dir.to_string_lossy().into_owned(); + cluster.spawn( + env!("CARGO_BIN_EXE_comet-executor"), + &[ + ("COMET_EXECUTOR_BIND_HOST", "127.0.0.1".to_string()), + ("COMET_EXECUTOR_PORT", executor_port.to_string()), + ("COMET_EXECUTOR_GRPC_PORT", executor_grpc_port.to_string()), + ("COMET_SCHEDULER_HOST", "127.0.0.1".to_string()), + ("COMET_SCHEDULER_PORT", scheduler_port.to_string()), + ("COMET_EXECUTOR_TASKS", "4".to_string()), + ("COMET_EXECUTOR_WORK_DIR", work_dir), + ], + ); + wait_for(executor_port); + cluster + } + + fn spawn(&mut self, binary: &str, env: &[(&str, String)]) { + let child = Command::new(binary) + .envs(env.iter().map(|(key, value)| (*key, value.as_str()))) + .env("RUST_LOG", "warn") + .stdout(Stdio::null()) + .spawn() + .unwrap_or_else(|e| panic!("Cannot start {binary}: {e}")); + self.processes.push(child); + } + + async fn get_flight_info(&self, request: &QueryRequest) -> Result { + let mut scheduler = connect(&self.flight_url).await; + within_time_limit( + scheduler.get_flight_info(FlightDescriptor::new_cmd(request.encode_to_vec())), + ) + .await + } + + /// Runs `request` and returns the batches of each result partition, in partition order. + async fn run(&self, request: &QueryRequest) -> Vec> { + let info = self.get_flight_info(request).await.unwrap(); + let mut partitions = vec![]; + for endpoint in &info.endpoint { + partitions.push(read(endpoint).await.unwrap()); + } + partitions + } +} + +/// Reads a result partition from the executor that holds it. +async fn read(endpoint: &FlightEndpoint) -> Result, FlightError> { + let mut executor = connect(&endpoint.location[0].uri).await; + let ticket = endpoint.ticket.clone().unwrap(); + within_time_limit(async { executor.do_get(ticket).await?.try_collect().await }).await +} + +impl Drop for Cluster { + fn drop(&mut self) { + // The executor first, so that it doesn't report to a scheduler that has gone. + for process in self.processes.iter_mut().rev() { + let _ = process.kill(); + let _ = process.wait(); + } + let _ = std::fs::remove_dir_all(&self.work_dir); + } +} + +/// Runs `call`, failing the test if it takes too long, as when no executor ever runs the query. +async fn within_time_limit(call: impl Future) -> T { + tokio::time::timeout(Duration::from_secs(300), call) + .await + .expect("The cluster took too long") +} + +/// Connects to a Flight location, such as an endpoint's `grpc+tcp://host:port`. +async fn connect(location: &str) -> FlightClient { + let uri = match location.strip_prefix("grpc+tcp://") { + Some(address) => format!("http://{address}"), + None => location.to_string(), + }; + let channel = Endpoint::from_shared(uri).unwrap().connect().await.unwrap(); + FlightClient::new(channel) +} + +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port() +} + +fn wait_for(port: u16) { + let deadline = Instant::now() + Duration::from_secs(60); + while TcpStream::connect(("127.0.0.1", port)).is_err() { + assert!( + Instant::now() < deadline, + "Nothing is listening on port {port}" + ); + std::thread::sleep(Duration::from_millis(100)); + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn runs_a_query_across_a_hash_shuffle() { + let cluster = Cluster::start(); + let request = query(vec![ + range_stage(1000, 4), + reading_stage(0, hash_on_first_column(3), 3), + ]); + let partitions = cluster.run(&request).await; + assert_eq!(partitions.len(), 3); + assert_eq!( + sorted_values::(&partitions.concat()), + (0..1000).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn runs_a_query_across_a_range_exchange() { + let cluster = Cluster::start(); + let request = query(vec![ + range_stage(1000, 4), + reading_stage(0, range_partitioning(4), 4), + ]); + let partitions = cluster.run(&request).await; + assert_eq!(partitions.len(), 1); + assert_eq!( + sorted_values::(&partitions.concat()), + (0..1000).collect::>() + ); +} + +#[tokio::test(flavor = "multi_thread")] +async fn refuses_a_query_from_another_build() { + let cluster = Cluster::start(); + let mut request = query(vec![range_stage(10, 1)]); + request.comet_version = "0.0.0".to_string(); + match cluster.get_flight_info(&request).await { + Err(FlightError::Tonic(status)) => { + assert_eq!(status.code(), Code::FailedPrecondition, "{status}"); + assert!(status.message().contains("same Comet build"), "{status}"); + } + other => panic!("Expected the query to be refused, got {other:?}"), + } +} + +#[tokio::test(flavor = "multi_thread")] +async fn endpoints_say_when_their_results_are_deleted() { + // A query's results stay readable for the scheduler's retention after the query ends, whether + // or not a client reads them, and each endpoint says until when. + let retention = Duration::from_secs(5); + let cluster = Cluster::start_with(&[( + "COMET_SCHEDULER_RESULT_RETENTION_SECONDS", + retention.as_secs().to_string(), + )]); + let submitted = SystemTime::now(); + let info = cluster + .get_flight_info(&query(vec![range_stage(100, 2)])) + .await + .unwrap(); + assert_eq!(info.endpoint.len(), 2); + let expiration = info.endpoint[0].expiration_time; + assert!(info + .endpoint + .iter() + .all(|endpoint| endpoint.expiration_time == expiration)); + let expires_at = SystemTime::try_from(expiration.expect("an expiration time")).unwrap(); + // The query ended between its submission and its answer. + assert!(expires_at >= submitted + retention); + assert!(expires_at <= SystemTime::now() + retention); + + // A client that reads one partition and then takes too long finds the other gone. + read(&info.endpoint[0]).await.unwrap(); + tokio::time::sleep( + expires_at + .duration_since(SystemTime::now()) + .unwrap_or_default(), + ) + .await; + let deadline = Instant::now() + Duration::from_secs(60); + while read(&info.endpoint[1]).await.is_ok() { + assert!( + Instant::now() < deadline, + "A result partition was still readable a minute after its endpoint expired" + ); + tokio::time::sleep(Duration::from_millis(200)).await; + } +} From 16f004ab2ab4500cb72814ee9acd8218bf59b4f6 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:26:09 -0600 Subject: [PATCH 5/8] ci: build and test the native-executors workspace native/native-executors is a Cargo workspace of its own, so the Rust jobs don't cover it. The new linux-test-native-executors job runs clippy and the workspace's tests, including the ones that start a scheduler and an executor, and the lint job checks its formatting. --- .github/workflows/pr_build_linux.yml | 31 ++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml index 52e86f59ffd..4990e93dd6a 100644 --- a/.github/workflows/pr_build_linux.yml +++ b/.github/workflows/pr_build_linux.yml @@ -106,6 +106,7 @@ jobs: run: | rustup component add rustfmt cd native && cargo fmt --all -- --check + cargo fmt --manifest-path native-executors/Cargo.toml --all -- --check # Fast syntactic-only scalafix check. Parses sources without compiling, so it # surfaces version-independent style issues (e.g. RedundantSyntax) in seconds, @@ -476,6 +477,36 @@ jobs: - name: Rust test steps uses: ./.github/actions/rust-test + # native/native-executors, the Comet Native Executors cluster (issue #7288), + # is a Cargo workspace of its own so that the main build never depends on + # Ballista. linux-test-rust doesn't build it, so this job does, and the lint + # job checks its formatting. When Comet moves to a DataFusion major that + # Ballista hasn't released yet, the upgrade can turn this job off until + # Ballista catches up. + linux-test-native-executors: + needs: lint + name: ubuntu-latest/native-executors + if: ${{ !inputs.cache-refresh-only && inputs.profiles != 'nightly' }} + runs-on: ubuntu-24.04 + container: + image: amd64/rust + steps: + - uses: actions/checkout@v7 + + - name: Setup Rust & Java toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: ${{ env.RUST_VERSION }} + jdk-version: 17 + + - name: Check Cargo clippy + run: | + cargo clippy --manifest-path native/native-executors/Cargo.toml --color=never \ + --all-targets -- -D warnings + + - name: Run Cargo test + run: cargo test --manifest-path native/native-executors/Cargo.toml --color=never + linux-test: # `lint` is already upstream via build-native; it is listed here so this # job can read its `profile-matrix` output. From 3dbee23bd7a43c2c1bfe0df23256abfa8d4e4799 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 04:14:28 -0600 Subject: [PATCH 6/8] feat: run queries on Comet Native Executors from a Spark driver With spark.comet.nativeExecutors.enabled, CometRule replaces the plan Comet would execute with a CometNativeExecutorsExec leaf, which runs the whole query on the cluster at spark.comet.nativeExecutors.scheduler.url. This works with AQE on, where Comet's query-stage-prep rule sees the whole plan, and off. The leaf is columnar, so Spark Connect's execute-and-submitJob path and collect() both read it through local tasks on the driver, each of which fetches one result partition. The stage graph builder turns each native block into a stage and each native shuffle into an input, and refuses, naming the operator, any query that cannot run entirely natively, including a block whose plan calls back into the JVM. Operators that only turn the result into JVM objects, as Dataset.rdd adds, stay on the driver. The driver's Flight client is a JNI library in libcomet behind the native-executors Cargo feature, off by default. CometNativeExecutorsSuite starts a scheduler and an executor and checks queries against Spark; it runs when COMET_NATIVE_EXECUTORS_BIN_DIR names the binaries. See #7288. --- native/Cargo.lock | 143 +++++++ native/core/Cargo.toml | 6 + native/core/src/execution/jni_api.rs | 2 +- .../src/execution/native_executors/client.rs | 179 +++++++++ .../src/execution/native_executors/mod.rs | 2 + native/proto/src/proto/native_executors.proto | 14 + .../scala/org/apache/comet/CometConf.scala | 22 ++ .../apache/comet/NativeExecutorsClient.scala | 62 +++ .../org/apache/comet/rules/CometRule.scala | 5 + .../sql/comet/CometNativeExecutorsExec.scala | 369 ++++++++++++++++++ .../exec/CometNativeExecutorsSuite.scala | 231 +++++++++++ 11 files changed, 1034 insertions(+), 1 deletion(-) create mode 100644 native/core/src/execution/native_executors/client.rs create mode 100644 spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala create mode 100644 spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala create mode 100644 spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala diff --git a/native/Cargo.lock b/native/Cargo.lock index 80741b85406..80922c6ce4e 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -347,6 +347,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-flight" +version = "59.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b543210778ac15d5f7a10153dc39e099f1eee756363589a842029e6c5c76e423" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-schema", + "base64 0.23.1", + "bytes", + "futures", + "prost", + "prost-types", + "tonic", + "tonic-prost", +] + [[package]] name = "arrow-ipc" version = "59.3.1" @@ -1033,6 +1054,49 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + [[package]] name = "backon" version = "1.6.0" @@ -1979,6 +2043,7 @@ name = "datafusion-comet" version = "1.3.0" dependencies = [ "arrow", + "arrow-flight", "assertables", "async-trait", "aws-config", @@ -2044,6 +2109,7 @@ dependencies = [ "tikv-jemalloc-ctl", "tikv-jemallocator", "tokio", + "tonic", "typetag", "url", "uuid", @@ -3465,6 +3531,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.4.0" @@ -3494,6 +3566,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -3517,6 +3590,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.21" @@ -4241,6 +4327,12 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + [[package]] name = "md-5" version = "0.10.6" @@ -4295,6 +4387,12 @@ dependencies = [ "libmimalloc-sys", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -6884,6 +6982,47 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1fa9b1a51188f3266a823208078315811269d06d272c3a61be0b9a194194ff" +dependencies = [ + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "socket2", + "sync_wrapper", + "tokio", + "tokio-stream", + "tokio-util", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-prost" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63c616ffaa648f7c5dd5657e97d9a285fcb44ff3b70da761d436f5a13ff7be6" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" version = "0.5.3" @@ -6892,11 +7031,15 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", + "indexmap 2.14.2", "pin-project-lite", + "slab", "sync_wrapper", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index a01bbb7f829..e207bbdf166 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -54,6 +54,9 @@ async-trait = { workspace = true } log = "0.4" log4rs = "1.4.0" prost = "0.14.3" +# The driver's client of a Comet Native Executor cluster; see the native-executors feature. +arrow-flight = { version = "59.2.0", optional = true } +tonic = { version = "0.14", optional = true } jni = { workspace = true } rand = { workspace = true } num = { workspace = true } @@ -132,6 +135,9 @@ python-udf = ["dep:pyo3", "dep:libc"] contrib-lance = ["dep:comet-contrib-lance"] hdfs-opendal = ["opendal/services-hdfs", "object_store_opendal", "hdfs-sys"] jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"] +# The JNI client a Spark driver uses to run queries on Comet Native Executors (issue #7288). Off +# by default while the feature is experimental, so the standard build links no gRPC client. +native-executors = ["dep:arrow-flight", "dep:tonic"] # Native Delta Lake scan support for the JVM-planned path (contrib/delta-spark). # In the default set so trying the contrib needs only the jar and the config, # not a custom native build. It is inert at runtime unless the contrib jar is diff --git a/native/core/src/execution/jni_api.rs b/native/core/src/execution/jni_api.rs index 00af7f9aac9..c6035aba930 100644 --- a/native/core/src/execution/jni_api.rs +++ b/native/core/src/execution/jni_api.rs @@ -1232,7 +1232,7 @@ fn prepare_output( /// # Safety /// Each address must point to a writable `FFI_ArrowArray` / `FFI_ArrowSchema` the caller owns. /// Whatever the structs hold is overwritten without being released. -unsafe fn export_batch( +pub(crate) unsafe fn export_batch( array_addrs: &[i64], schema_addrs: &[i64], output_batch: RecordBatch, diff --git a/native/core/src/execution/native_executors/client.rs b/native/core/src/execution/native_executors/client.rs new file mode 100644 index 00000000000..d268316070e --- /dev/null +++ b/native/core/src/execution/native_executors/client.rs @@ -0,0 +1,179 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! The JNI client a Spark driver uses to run queries on a Comet Native Executor cluster: the +//! native methods of `org.apache.comet.NativeExecutorsClient`. +//! +//! The driver submits a query to the scheduler's Arrow Flight service, which answers with one +//! endpoint per result partition, and then reads each partition from its executor with `DoGet`. + +use arrow_flight::decode::FlightRecordBatchStream; +use arrow_flight::flight_service_client::FlightServiceClient; +use arrow_flight::{FlightClient, FlightDescriptor, Ticket}; +use datafusion_comet_proto::spark_native_executors::{ + QueryRequest, ResultEndpoint, ResultEndpoints, +}; +use datafusion_comet_proto::PROTO_FINGERPRINT; +use futures::StreamExt; +use jni::objects::{JByteArray, JClass, JLongArray, JString, ReleaseMode}; +use jni::sys::{jbyteArray, jlong}; +use jni::EnvUnowned; +use prost::Message; +use tonic::transport::{Channel, Endpoint}; + +use super::COMET_VERSION; +use crate::errors::{try_unwrap_or_throw, CometError, CometResult}; +use crate::execution::jni_api::{export_batch, get_runtime}; + +fn client_error(context: &str, error: impl std::fmt::Display) -> CometError { + CometError::Internal(format!("Comet Native Executors: {context}: {error}")) +} + +/// Connects to the Flight service at `location`, a Flight location such as +/// `grpc+tcp://host:port` or an `http://` URL. +async fn connect(location: &str) -> CometResult { + let uri = match location.strip_prefix("grpc+tcp://") { + Some(address) => format!("http://{address}"), + None => location.to_string(), + }; + let channel: Channel = Endpoint::from_shared(uri) + .map_err(|e| client_error(&format!("invalid location {location}"), e))? + .connect() + .await + .map_err(|e| client_error(&format!("cannot connect to {location}"), e))?; + // Query plans and result batches can be larger than gRPC's default 4 MiB message limit. + let inner = FlightServiceClient::new(channel) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX); + Ok(FlightClient::new_from_inner(inner)) +} + +/// Submits `query`, a serialized `QueryRequest`, to the scheduler at `scheduler_url` and waits +/// for it to finish. Returns where its result partitions are. +async fn submit(scheduler_url: &str, query: &[u8]) -> CometResult { + let mut request = QueryRequest::decode(query).map_err(|e| client_error("bad query", e))?; + request.comet_version = COMET_VERSION.to_string(); + request.proto_fingerprint = PROTO_FINGERPRINT.to_string(); + let mut client = connect(scheduler_url).await?; + let info = client + .get_flight_info(FlightDescriptor::new_cmd(request.encode_to_vec())) + .await + .map_err(|e| client_error("the query failed", e))?; + let endpoints = info + .endpoint + .into_iter() + .map(|endpoint| { + let location = endpoint + .location + .first() + .map(|location| location.uri.clone()) + .ok_or_else(|| client_error("bad result", "a partition has no location"))?; + let ticket = endpoint + .ticket + .map(|ticket| ticket.ticket.to_vec()) + .ok_or_else(|| client_error("bad result", "a partition has no ticket"))?; + Ok(ResultEndpoint { location, ticket }) + }) + .collect::>>()?; + Ok(ResultEndpoints { endpoints }) +} + +/// Runs a query: `query` is a serialized `QueryRequest`, which this fills in with the driver's +/// Comet version and plan format. Returns a serialized `ResultEndpoints`. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_submit( + e: EnvUnowned, + _class: JClass, + scheduler_url: JString, + query: JByteArray, +) -> jbyteArray { + try_unwrap_or_throw(&e, |env| { + let scheduler_url = scheduler_url + .try_to_string(env) + .map_err(|e| client_error("bad scheduler URL", e))?; + let query = env.convert_byte_array(query)?; + let endpoints = get_runtime().block_on(submit(&scheduler_url, &query))?; + Ok(env + .byte_array_from_slice(&endpoints.encode_to_vec())? + .into_raw()) + }) +} + +/// Starts reading a result partition from the executor at `location`. Returns a handle for +/// `nextBatch` and `closePartition`. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_openPartition( + e: EnvUnowned, + _class: JClass, + location: JString, + ticket: JByteArray, +) -> jlong { + try_unwrap_or_throw(&e, |env| { + let location = location + .try_to_string(env) + .map_err(|e| client_error("bad location", e))?; + let ticket = env.convert_byte_array(ticket)?; + let stream = get_runtime().block_on(async { + connect(&location) + .await? + .do_get(Ticket::new(ticket)) + .await + .map_err(|e| client_error(&format!("cannot read a partition from {location}"), e)) + })?; + Ok(Box::into_raw(Box::new(stream)) as jlong) + }) +} + +/// Moves the partition's next batch into the Arrow C Data Interface structs at `array_addrs` +/// and `schema_addrs`, one pair per column. Returns its row count, or -1 at the end. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_nextBatch( + e: EnvUnowned, + _class: JClass, + handle: jlong, + array_addrs: JLongArray, + schema_addrs: JLongArray, +) -> jlong { + try_unwrap_or_throw(&e, |env| { + // SAFETY: `handle` comes from `openPartition` and hasn't been closed. + let stream = unsafe { &mut *(handle as *mut FlightRecordBatchStream) }; + let Some(batch) = get_runtime().block_on(stream.next()) else { + return Ok(-1); + }; + let batch = batch.map_err(|e| client_error("cannot read a partition", e))?; + let array_addrs = unsafe { array_addrs.get_elements(env, ReleaseMode::NoCopyBack)? }; + let schema_addrs = unsafe { schema_addrs.get_elements(env, ReleaseMode::NoCopyBack)? }; + // SAFETY: the JVM allocated one array and one schema struct per column. + unsafe { export_batch(&array_addrs, &schema_addrs, batch, false) } + }) +} + +/// Stops reading a partition and releases its handle. +#[no_mangle] +pub extern "system" fn Java_org_apache_comet_NativeExecutorsClient_closePartition( + e: EnvUnowned, + _class: JClass, + handle: jlong, +) { + try_unwrap_or_throw(&e, |_env| { + if handle != 0 { + // SAFETY: `handle` comes from `openPartition`, and the JVM closes it once. + drop(unsafe { Box::from_raw(handle as *mut FlightRecordBatchStream) }); + } + Ok(()) + }) +} diff --git a/native/core/src/execution/native_executors/mod.rs b/native/core/src/execution/native_executors/mod.rs index fc0b9d6b2f4..f7c152d3fba 100644 --- a/native/core/src/execution/native_executors/mod.rs +++ b/native/core/src/execution/native_executors/mod.rs @@ -24,6 +24,8 @@ //! partition of a stage, reading its inputs from other DataFusion plans, such as a distributed //! engine's shuffle readers, instead of from the JVM. +#[cfg(feature = "native-executors")] +mod client; mod fragment; pub use fragment::{block_schema, partition_descriptor, CometFragmentExec}; diff --git a/native/proto/src/proto/native_executors.proto b/native/proto/src/proto/native_executors.proto index 6c0ba50afb4..a324dc690a1 100644 --- a/native/proto/src/proto/native_executors.proto +++ b/native/proto/src/proto/native_executors.proto @@ -80,3 +80,17 @@ message StageInput { // Every partition of this stage reads the producer's whole output. bool broadcast = 3; } + +// The result partitions of a query, in order. The driver's native library converts the +// endpoints of the FlightInfo the scheduler returns into this message for the JVM. +message ResultEndpoints { + repeated ResultEndpoint endpoints = 1; +} + +message ResultEndpoint { + // Arrow Flight location of the executor that holds the partition, such as + // grpc+tcp://host:port. + string location = 1; + // Opaque ticket to pass to DoGet. + bytes ticket = 2; +} diff --git a/spark/src/main/scala/org/apache/comet/CometConf.scala b/spark/src/main/scala/org/apache/comet/CometConf.scala index 6003703a630..6281f6924af 100644 --- a/spark/src/main/scala/org/apache/comet/CometConf.scala +++ b/spark/src/main/scala/org/apache/comet/CometConf.scala @@ -489,6 +489,28 @@ object CometConf extends ShimCometConf { .checkValue(v => v > 0 && v <= Int.MaxValue, "Must be positive and fit in an Int.") .createWithDefault(1024 * 1024) + val COMET_NATIVE_EXECUTORS_ENABLED: ConfigEntry[Boolean] = + conf("spark.comet.nativeExecutors.enabled") + .category(CATEGORY_EXEC) + .doc( + "Experimental. When enabled, Comet runs each query on a Comet Native Executor cluster " + + "instead of on Spark's executors: the driver plans the query and submits it to the " + + "cluster's scheduler (`spark.comet.nativeExecutors.scheduler.url`), and the results " + + "come back to the driver. A query that cannot run entirely natively fails, naming " + + "the operator that cannot. See https://github.com/apache/datafusion-comet/issues/7288.") + .internal() + .booleanConf + .createWithDefault(false) + + val COMET_NATIVE_EXECUTORS_SCHEDULER_URL: ConfigEntry[String] = + conf("spark.comet.nativeExecutors.scheduler.url") + .category(CATEGORY_EXEC) + .doc("Experimental. The Arrow Flight URL of a Comet Native Executor cluster's scheduler, " + + "such as `http://host:50060`. See `spark.comet.nativeExecutors.enabled`.") + .internal() + .stringConf + .createWithDefault("http://localhost:50060") + val COMET_NATIVE_COLUMNAR_TO_ROW_ENABLED: ConfigEntry[Boolean] = conf(s"$COMET_EXEC_CONFIG_PREFIX.columnarToRow.native.enabled") .category(CATEGORY_EXEC) diff --git a/spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala b/spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala new file mode 100644 index 00000000000..fc4d3c98d9c --- /dev/null +++ b/spark/src/main/scala/org/apache/comet/NativeExecutorsClient.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet + +/** + * The JNI client a Spark driver uses to run queries on a Comet Native Executor cluster. See + * https://github.com/apache/datafusion-comet/issues/7288. + * + * The native methods exist only in a libcomet built with the `native-executors` Cargo feature. + */ +class NativeExecutorsClient extends NativeBase { + + /** + * Runs a query on the cluster and waits for it to finish. + * + * @param schedulerUrl + * the Arrow Flight URL of the cluster's scheduler + * @param query + * a serialized `QueryRequest`, whose Comet version and plan format the native library fills + * in + * @return + * a serialized `ResultEndpoints`: where each result partition is, in order + */ + @native def submit(schedulerUrl: String, query: Array[Byte]): Array[Byte] + + /** + * Starts reading a result partition. + * + * @return + * a handle for [[nextBatch]] and [[closePartition]] + */ + @native def openPartition(location: String, ticket: Array[Byte]): Long + + /** + * Moves the partition's next batch into the Arrow C Data Interface structs at `arrayAddrs` and + * `schemaAddrs`, one pair per column. + * + * @return + * the batch's row count, or -1 at the end of the partition + */ + @native def nextBatch(handle: Long, arrayAddrs: Array[Long], schemaAddrs: Array[Long]): Long + + /** Stops reading a partition and releases its handle. */ + @native def closePartition(handle: Long): Unit +} diff --git a/spark/src/main/scala/org/apache/comet/rules/CometRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometRule.scala index fbde4a79090..07b2eea6863 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometRule.scala @@ -25,6 +25,7 @@ import scala.util.control.NonFatal import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.catalyst.trees.TreeNodeTag +import org.apache.spark.sql.comet.CometNativeExecutors import org.apache.spark.sql.execution.{ApplyColumnarRulesAndInsertTransitions, BaseSubqueryExec, ColumnarToRowExec, ExecSubqueryExpression, InputAdapter, QueryExecution, ReusedSubqueryExec, RowToColumnarExec, SparkPlan, WholeStageCodegenExec} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, InsertAdaptiveSparkPlan, QueryStageExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, Exchange} @@ -163,6 +164,10 @@ case class CometRule(session: SparkSession, queryStagePrep: Boolean = false) if (CometRule.planOnlyApplies(conf, plan)) { reportPlanOnlyCoverage(plan) plan + } else if (CometNativeExecutors.applies(conf, plan, queryStagePrep)) { + // The whole query runs on a Comet Native Executor cluster. The conversion alone leaves + // every native block serialized, and nothing reverts a stage to Spark. + CometNativeExecutors.offload(plan, convert(plan)) } else { convert(plan) } diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala new file mode 100644 index 00000000000..ef9c4de17c5 --- /dev/null +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.spark.sql.comet + +import java.util.IdentityHashMap + +import scala.collection.mutable.ArrayBuffer +import scala.jdk.CollectionConverters._ + +import org.apache.spark.{Partition, SparkContext, TaskContext} +import org.apache.spark.rdd.RDD +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, SinglePartition} +import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec} +import org.apache.spark.sql.execution.{ColumnarToRowExec, DeserializeToObjectExec, LeafExecNode, SparkPlan} +import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} +import org.apache.spark.sql.execution.command.{DataWritingCommandExec, ExecutedCommandExec} +import org.apache.spark.sql.execution.datasources.v2.V2CommandExec +import org.apache.spark.sql.execution.exchange.Exchange +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.types.StructField +import org.apache.spark.sql.vectorized.ColumnarBatch + +import com.google.protobuf.Descriptors.FieldDescriptor +import com.google.protobuf.Message + +import org.apache.comet.{CometConf, CometExecIterator, CometExplainInfo, CometRuntimeException, NativeExecutorsClient} +import org.apache.comet.serde.{PartitioningOuterClass, QueryPlanSerde} +import org.apache.comet.serde.Config.ConfigMap +import org.apache.comet.serde.NativeExecutors.{QueryRequest, ResultEndpoints, Stage, StageInput} +import org.apache.comet.serde.OperatorOuterClass.Operator +import org.apache.comet.serde.operator.schema2Proto +import org.apache.comet.vector.NativeUtil + +/** + * Runs a whole query on a Comet Native Executor cluster instead of on Spark's executors. See + * https://github.com/apache/datafusion-comet/issues/7288. + * + * This leaf replaces the plan Comet would otherwise execute. `doExecuteColumnar` submits the + * query to the cluster's scheduler and waits for it on the driver; each partition of the RDD it + * returns then reads one result partition from the executor that holds it. + * + * @param query + * the serialized `QueryRequest` built by [[CometNativeExecutors.buildQuery]] + */ +case class CometNativeExecutorsExec( + override val output: Seq[Attribute], + query: Array[Byte], + schedulerUrl: String) + extends LeafExecNode { + + override def supportsColumnar: Boolean = true + + override protected def doExecute(): RDD[InternalRow] = + throw new UnsupportedOperationException(s"$nodeName only produces columnar output") + + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val endpoints = + ResultEndpoints.parseFrom(new NativeExecutorsClient().submit(schedulerUrl, query)) + val partitions = endpoints.getEndpointsList.asScala.map { endpoint => + (endpoint.getLocation, endpoint.getTicket.toByteArray) + } + new CometNativeExecutorsRDD(sparkContext, partitions.toSeq, output.length) + } + + override def stringArgs: Iterator[Any] = + Iterator(output, s"${query.length} bytes", schedulerUrl) +} + +private class CometNativeExecutorsPartition( + override val index: Int, + val location: String, + val ticket: Array[Byte]) + extends Partition + +/** + * The result partitions of a query that ran on a Comet Native Executor cluster, each read from + * the executor that holds it. + */ +private class CometNativeExecutorsRDD( + sc: SparkContext, + endpoints: Seq[(String, Array[Byte])], + numColumns: Int) + extends RDD[ColumnarBatch](sc, Nil) { + + override protected def getPartitions: Array[Partition] = + endpoints.zipWithIndex.map { case ((location, ticket), index) => + new CometNativeExecutorsPartition(index, location, ticket): Partition + }.toArray + + override def compute(split: Partition, context: TaskContext): Iterator[ColumnarBatch] = { + val partition = split.asInstanceOf[CometNativeExecutorsPartition] + val client = new NativeExecutorsClient() + val nativeUtil = new NativeUtil() + val handle = client.openPartition(partition.location, partition.ticket) + val batches = new Iterator[ColumnarBatch] with AutoCloseable { + private var current: Option[ColumnarBatch] = None + private var pending: Option[ColumnarBatch] = None + private var finished = false + private var closed = false + + override def hasNext: Boolean = { + if (pending.isEmpty && !finished) { + // The consumer has moved past the batch it last got. + current.foreach(_.close()) + current = None + pending = nativeUtil.getNextBatch( + numColumns, + (arrays, schemas) => client.nextBatch(handle, arrays, schemas)) + if (pending.isEmpty) { + finished = true + close() + } + } + pending.isDefined + } + + override def next(): ColumnarBatch = { + if (!hasNext) { + throw new NoSuchElementException("No more batches in this result partition") + } + current = pending + pending = None + current.get + } + + override def close(): Unit = if (!closed) { + closed = true + current.foreach(_.close()) + current = None + pending.foreach(_.close()) + pending = None + client.closePartition(handle) + nativeUtil.close() + } + } + context.addTaskCompletionListener[Unit](_ => batches.close()) + batches + } +} + +/** + * Plans queries for Comet Native Executors: turns the plan Comet would execute into the stage + * graph a cluster runs, and refuses any query that cannot run entirely natively. + */ +object CometNativeExecutors { + + /** Messages of native plans that call back into the JVM, which a cluster doesn't have. */ + private val jvmCallbacks = Set("JvmScalarUdf", "Subquery", "ArrowPythonUdf") + + /** + * Whether Comet runs `plan` on a Comet Native Executor cluster. It must be a whole query: not a + * command, not a stage AQE has split off, and not a plan already replaced. + */ + def applies(conf: SQLConf, plan: SparkPlan, queryStagePrep: Boolean): Boolean = + CometConf.COMET_NATIVE_EXECUTORS_ENABLED.get(conf) && + !(conf.adaptiveExecutionEnabled && !queryStagePrep && plan.isInstanceOf[Exchange]) && + !plan.exists { + case _: CometNativeExecutorsExec | _: AdaptiveSparkPlanExec | _: QueryStageExec | + _: ExecutedCommandExec | _: DataWritingCommandExec | _: V2CommandExec => + true + case _ => false + } + + /** + * Replaces `converted`, the plan Comet would execute for `original`, with a leaf that runs it + * on a Comet Native Executor cluster. An operator on top that only turns the query's result + * into JVM objects, as `Dataset.rdd` adds, stays on the driver above the leaf. + */ + def offload(original: SparkPlan, converted: SparkPlan): SparkPlan = converted match { + case toObjects: DeserializeToObjectExec => + toObjects.withNewChildren(Seq(offload(toObjects.child, toObjects.child))) + case native => + val exec = CometNativeExecutorsExec( + native.output, + buildQuery(native).toByteArray, + CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.get()) + native.logicalLink.orElse(original.logicalLink).foreach(exec.setLogicalLink) + exec + } + + /** + * The query a Comet Native Executor cluster runs for `plan`, the plan Comet would execute: one + * stage per native block, with an input for each exchange that feeds the block. + * + * @throws CometRuntimeException + * naming the first operator that cannot run on a cluster + */ + def buildQuery(plan: SparkPlan): QueryRequest = { + val builder = new StageGraphBuilder(plan) + val root = plan match { + case ColumnarToRowExec(child) => child + case CometColumnarToRowExec(child) => child + case CometNativeColumnarToRowExec(child) => child + case other => other + } + builder.stageOf(root) + QueryRequest + .newBuilder() + .putAllConf(nativeConf()) + .addAllStages(builder.stages.asJava) + .build() + } + + /** The configs a Spark task would pass to native code with a plan. */ + private def nativeConf(): java.util.Map[String, String] = { + val conf = new java.util.HashMap[String, String]( + ConfigMap.parseFrom(CometExecIterator.serializeCometSQLConfs()).getEntriesMap) + // A Spark task passes the batch size apart from the other configs. + conf.put(CometConf.COMET_BATCH_SIZE.key, CometConf.COMET_BATCH_SIZE.get().toString) + conf + } + + private class StageGraphBuilder(query: SparkPlan) { + val stages: ArrayBuffer[Stage] = ArrayBuffer.empty + private val stageIndexes = new IdentityHashMap[SparkPlan, Integer]() + + /** The index of the stage that runs the native block rooted at `plan`, adding it first. */ + def stageOf(plan: SparkPlan): Int = { + val known = stageIndexes.get(plan) + if (known != null) { + return known + } + val block = plan match { + case native: CometNativeExec if native.serializedPlanOpt.isDefined => native + case other => fail(other, "it does not run natively") + } + val exchanges = inputsOf(block) + val inputs = exchanges.map { exchange => + StageInput + .newBuilder() + .setProducer(stageOf(exchange.child)) + .setPartitioning(partitioningOf(exchange)) + .build() + } + val blockPlan = Operator.parseFrom(block.serializedPlanOpt.plan.get) + findJvmCallback(blockPlan).foreach { name => + fail(block, s"its native plan calls back into the JVM ($name)") + } + + // A block that reads files runs one copy per partition, each naming its own files. + val (common, perPartition) = PlanDataInjector.findAllPlanData(block) + val partitionPlans = if (perPartition.isEmpty) { + Seq.empty + } else { + val base = new PlanDataInjector.CachedPlanData(blockPlan) + (0 until perPartition.values.head.length).map { i => + val partitionData = perPartition.map { case (key, data) => key -> data(i) } + PlanDataInjector.injectPlanData(base, common, partitionData).toByteString + } + } + // The cluster reads it only for a stage that reads no exchange. A stage that does has as + // many partitions as its exchanges deliver. + val numPartitions = + if (partitionPlans.nonEmpty) partitionPlans.length + else block.outputPartitioning.numPartitions + // Declared as Comet's shuffle writer declares its input, since the cluster casts the + // block's output to it as that writer does. + val outputSchema = schema2Proto( + block.output.map(a => StructField(a.name, a.dataType, a.nullable, a.metadata))) + + stages += Stage + .newBuilder() + .setPlan(blockPlan) + .addAllInputs(inputs.asJava) + .addAllPartitionPlans(partitionPlans.asJava) + .setNumPartitions(numPartitions) + .addAllOutputSchema(outputSchema.asJava) + .build() + val index = stages.length - 1 + stageIndexes.put(plan, index) + index + } + + /** + * The exchanges that feed `block`, in the order its native plan's input leaves read them. + */ + private def inputsOf(block: CometNativeExec): Seq[CometShuffleExchangeExec] = { + val inputs = ArrayBuffer.empty[CometShuffleExchangeExec] + def walk(plan: SparkPlan): Unit = plan match { + case exchange: CometShuffleExchangeExec if exchange.shuffleType == CometNativeShuffle => + inputs += exchange + case _: CometNativeScanExec | _: CometIcebergNativeScanExec | _: CometRangeExec => + // Read inside the block. + // Part of the block, even when it has a serialized plan of its own: the conversion + // serializes some operators before their parent joins them into a larger block. + case native: CometNativeExec => + native.children.foreach(walk) + case other => fail(other, "Comet Native Executors do not support it yet") + } + walk(block) + inputs.toSeq + } + + private def partitioningOf( + exchange: CometShuffleExchangeExec): PartitioningOuterClass.Partitioning = { + val partitioning = PartitioningOuterClass.Partitioning.newBuilder() + exchange.outputPartitioning match { + case HashPartitioning(expressions, numPartitions) => + val keys = expressions.map { key => + QueryPlanSerde + .exprToProto(key, exchange.child.output) + .getOrElse(fail(exchange, s"its partitioning key $key cannot run natively")) + } + partitioning.setHashPartition( + PartitioningOuterClass.HashPartition + .newBuilder() + .addAllHashExpression(keys.asJava) + .setNumPartitions(numPartitions)) + case SinglePartition => + partitioning.setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder()) + case other => + fail(exchange, s"Comet Native Executors do not support $other exchanges yet") + } + partitioning.build() + } + + private def fail(node: SparkPlan, reason: String): Nothing = { + val fallbackReasons = node + .getTagValue(CometExplainInfo.FALLBACK_REASONS) + .getOrElse(Set.empty[String]) + val because = if (fallbackReasons.isEmpty) { + "" + } else { + fallbackReasons.toSeq.sorted.mkString(" Comet could not convert it: ", "; ", ".") + } + throw new CometRuntimeException( + s"Comet Native Executors cannot run this query, because of ${node.nodeName}: " + + s"$reason.$because\n${query.treeString}") + } + } + + /** The first message in `message` that calls back into the JVM, if any. */ + private def findJvmCallback(message: Message): Option[String] = { + val name = message.getDescriptorForType.getName + if (jvmCallbacks.contains(name)) { + Some(name) + } else { + val found = message.getAllFields.asScala.iterator.flatMap { case (field, value) => + if (field.getJavaType != FieldDescriptor.JavaType.MESSAGE) { + Iterator.empty + } else if (field.isRepeated) { + value.asInstanceOf[java.util.List[Message]].asScala.iterator.flatMap(findJvmCallback) + } else { + findJvmCallback(value.asInstanceOf[Message]).iterator + } + } + if (found.hasNext) Some(found.next()) else None + } + } +} diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala new file mode 100644 index 00000000000..bbc5c1476e0 --- /dev/null +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.comet.exec + +import java.io.File +import java.net.{ServerSocket, Socket} +import java.nio.file.Files +import java.util.concurrent.TimeUnit + +import scala.util.Try + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{CometTestBase, DataFrame} +import org.apache.spark.sql.comet.CometNativeExecutorsExec +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.functions.{col, count, max, sum} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.comet.CometConf + +/** + * Runs queries on a Comet Native Executor cluster (see + * https://github.com/apache/datafusion-comet/issues/7288): a comet-scheduler and a comet-executor + * started from the binaries in the directory `COMET_NATIVE_EXECUTORS_BIN_DIR` names. The tests + * are skipped when it isn't set. To run them: + * + * {{{ + * cargo build --manifest-path native/Cargo.toml -p datafusion-comet --features native-executors + * CARGO_TARGET_DIR=native/target cargo build --manifest-path native/native-executors/Cargo.toml + * COMET_NATIVE_EXECUTORS_BIN_DIR=native/target/debug ./mvnw test -Dtest=none \ + * -Dsuites=org.apache.comet.exec.CometNativeExecutorsSuite + * }}} + */ +class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelper { + + private val binDir: Option[File] = + sys.env.get("COMET_NATIVE_EXECUTORS_BIN_DIR").map(new File(_).getAbsoluteFile) + private var processes: Seq[Process] = Nil + private var schedulerUrl: String = _ + + override protected def sparkConf: SparkConf = { + val conf = super.sparkConf + conf.set(CometConf.COMET_SHUFFLE_MODE.key, "native") + conf.set(SQLConf.SHUFFLE_PARTITIONS.key, "4") + conf + } + + override def beforeAll(): Unit = { + super.beforeAll() + binDir.foreach(startCluster) + } + + override def afterAll(): Unit = { + try { + // The executor first, so that it doesn't report to a scheduler that has gone. + processes.reverse.foreach { process => + process.destroy() + if (!process.waitFor(10, TimeUnit.SECONDS)) { + process.destroyForcibly() + } + } + } finally { + super.afterAll() + } + } + + private def startCluster(dir: File): Unit = { + val schedulerPort = freePort() + val flightPort = freePort() + val executorPort = freePort() + val executorGrpcPort = freePort() + val logs = Files.createTempDirectory("comet-native-executors").toFile + logInfo(s"Comet Native Executor cluster logs: $logs") + start( + new File(dir, "comet-scheduler"), + Map( + "COMET_SCHEDULER_BIND_HOST" -> "127.0.0.1", + "COMET_SCHEDULER_PORT" -> schedulerPort.toString, + "COMET_SCHEDULER_FLIGHT_PORT" -> flightPort.toString), + new File(logs, "scheduler.log")) + waitForPort(schedulerPort) + waitForPort(flightPort) + start( + new File(dir, "comet-executor"), + Map( + "COMET_EXECUTOR_BIND_HOST" -> "127.0.0.1", + "COMET_EXECUTOR_PORT" -> executorPort.toString, + "COMET_EXECUTOR_GRPC_PORT" -> executorGrpcPort.toString, + "COMET_SCHEDULER_HOST" -> "127.0.0.1", + "COMET_SCHEDULER_PORT" -> schedulerPort.toString, + "COMET_EXECUTOR_TASKS" -> "4", + "COMET_EXECUTOR_WORK_DIR" -> new File(logs, "work").getPath), + new File(logs, "executor.log")) + waitForPort(executorPort) + schedulerUrl = s"http://127.0.0.1:$flightPort" + } + + private def start(binary: File, env: Map[String, String], log: File): Unit = { + val builder = new ProcessBuilder(binary.getPath) + env.foreach { case (key, value) => builder.environment().put(key, value) } + builder.redirectErrorStream(true).redirectOutput(log) + processes :+= builder.start() + } + + private def freePort(): Int = { + val socket = new ServerSocket(0) + try socket.getLocalPort + finally socket.close() + } + + private def waitForPort(port: Int): Unit = { + val deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60) + while (Try(new Socket("127.0.0.1", port).close()).isFailure) { + assert(System.nanoTime() < deadline, s"Nothing is listening on port $port") + Thread.sleep(100) + } + } + + private def nativeExecutorsTest(name: String)(body: => Unit): Unit = test(name) { + assume(binDir.isDefined, "COMET_NATIVE_EXECUTORS_BIN_DIR is not set") + body + } + + private def onCluster[T](body: => T): T = withSQLConf( + CometConf.COMET_NATIVE_EXECUTORS_ENABLED.key -> "true", + CometConf.COMET_NATIVE_EXECUTORS_SCHEDULER_URL.key -> schedulerUrl)(body) + + /** Checks that the cluster runs `query` and returns what Spark returns. */ + private def checkOnCluster(query: => DataFrame): Unit = { + val expected = query.collect().toSeq + onCluster { + val df = query + checkAnswer(df, expected) + val plan = df.queryExecution.executedPlan + assert(collect(plan) { case exec: CometNativeExecutorsExec => exec }.size == 1, plan) + } + } + + /** A Parquet table with several files, read as several partitions. */ + private def withTable(body: String => Unit): Unit = withTempPath { dir => + spark + .range(0, 10000, 1, 8) + .selectExpr("id", "id % 7 AS k", "CAST(id AS DOUBLE) * 1.5 AS v") + .write + .parquet(dir.getCanonicalPath) + withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> "16384") { + body(dir.getCanonicalPath) + } + } + + nativeExecutorsTest("scan, filter and project") { + withTable { path => + checkOnCluster(spark.read.parquet(path).where("k > 2").selectExpr("id", "v * 2 AS w")) + } + } + + nativeExecutorsTest("aggregate across a hash shuffle") { + withTable { path => + checkOnCluster(spark.read.parquet(path).groupBy("k").agg(sum("v"), count("*"))) + } + } + + nativeExecutorsTest("aggregate across a hash shuffle without AQE") { + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + withTable { path => + checkOnCluster(spark.read.parquet(path).groupBy("k").agg(sum("v"), max("id"))) + } + } + } + + nativeExecutorsTest("global aggregate across a single-partition shuffle") { + withTable { path => + checkOnCluster(spark.read.parquet(path).agg(sum("id"), max("v"), count("*"))) + } + } + + Seq(true, false).foreach { sortMerge => + val join = if (sortMerge) "sort-merge" else "hash" + nativeExecutorsTest(s"shuffled $join join") { + withSQLConf( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.ADAPTIVE_AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.PREFER_SORTMERGEJOIN.key -> sortMerge.toString) { + withTable { path => + val left = spark.read.parquet(path).select("id", "k") + val right = spark.read + .parquet(path) + .where("id < 10") + .select(col("k").as("k2"), col("v")) + checkOnCluster(left.join(right, left("k") === right("k2")).select("id", "v")) + } + } + } + } + + nativeExecutorsTest("a query the cluster cannot run fails, naming the operator") { + withTable { path => + onCluster { + // A global sort needs a range exchange, which the prototype doesn't support. + val error = intercept[Exception](spark.read.parquet(path).orderBy("v").collect()) + val messages = Iterator + .iterate[Throwable](error)(_.getCause) + .takeWhile(_ != null) + .map(e => String.valueOf(e.getMessage)) + .toSeq + assert( + messages.exists(message => + message.contains("Comet Native Executors cannot run this query") && + message.contains("rangepartitioning")), + error) + } + } + } +} From 5cd7afa25fe03a31c962eecab9cc2542e9583498 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 09:53:31 -0600 Subject: [PATCH 7/8] feat: run sorts, and with them TPC-H q1, on Comet Native Executors The driver now serializes a range exchange as a RangePartition with its sort keys and no bounds. Until Ballista can plan a range shuffle for a physical plan (apache/datafusion-ballista#2680), the scheduler reads a range exchange as one partition, so the stage that sorts it runs in a single task. That is correct for a global sort, if serial. CometNativeExecutorsSuite runs TPC-H q1, with and without its order by, on the lineitem table under SPARK_TPCH_DATA when it is set, and on generated data with the same schema otherwise. The query that the cluster must refuse is now a broadcast join. See #7288. --- .../sql/comet/CometNativeExecutorsExec.scala | 14 +++- .../exec/CometNativeExecutorsSuite.scala | 73 ++++++++++++++++++- 2 files changed, 83 insertions(+), 4 deletions(-) diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala index ef9c4de17c5..1125fc45d0c 100644 --- a/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala +++ b/spark/src/main/scala/org/apache/spark/sql/comet/CometNativeExecutorsExec.scala @@ -28,7 +28,7 @@ import org.apache.spark.{Partition, SparkContext, TaskContext} import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, SinglePartition} +import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning, SinglePartition} import org.apache.spark.sql.comet.execution.shuffle.{CometNativeShuffle, CometShuffleExchangeExec} import org.apache.spark.sql.execution.{ColumnarToRowExec, DeserializeToObjectExec, LeafExecNode, SparkPlan} import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, QueryStageExec} @@ -325,6 +325,18 @@ object CometNativeExecutors { .newBuilder() .addAllHashExpression(keys.asJava) .setNumPartitions(numPartitions)) + case RangePartitioning(ordering, numPartitions) => + // Without bounds: the cluster decides how to range-partition. + val sortOrders = ordering.map { order => + QueryPlanSerde + .exprToProto(order, exchange.child.output) + .getOrElse(fail(exchange, s"its sort key $order cannot run natively")) + } + partitioning.setRangePartition( + PartitioningOuterClass.RangePartition + .newBuilder() + .addAllSortOrders(sortOrders.asJava) + .setNumPartitions(numPartitions)) case SinglePartition => partitioning.setSinglePartition(PartitioningOuterClass.SinglePartition.newBuilder()) case other => diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala index bbc5c1476e0..0ae64710c26 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeExecutorsSuite.scala @@ -210,11 +210,78 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp } } + /** + * TPC-H's `lineitem`: the table under `SPARK_TPCH_DATA` when it is set, as for + * `CometTPCHQuerySuite`, and otherwise generated data with the same schema. + */ + private def withLineitem(body: => Unit): Unit = sys.env.get("SPARK_TPCH_DATA") match { + case Some(path) => + spark.read.parquet(s"$path/lineitem").createOrReplaceTempView("lineitem") + withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> "32m")(body) + case None => + withTempPath { dir => + spark + .range(0, 100000, 1, 8) + .selectExpr( + "CAST(id % 50 + 1 AS DECIMAL(15, 2)) AS l_quantity", + "CAST((id % 1000) * 1.37 + 900 AS DECIMAL(15, 2)) AS l_extendedprice", + "CAST((id % 11) / 100.0 AS DECIMAL(15, 2)) AS l_discount", + "CAST((id % 9) / 100.0 AS DECIMAL(15, 2)) AS l_tax", + "ELT(CAST(id % 3 AS INT) + 1, 'A', 'N', 'R') AS l_returnflag", + "ELT(CAST(id % 2 AS INT) + 1, 'F', 'O') AS l_linestatus", + "DATE_ADD(DATE '1992-01-01', CAST(id % 2500 AS INT)) AS l_shipdate") + .write + .parquet(dir.getCanonicalPath) + spark.read.parquet(dir.getCanonicalPath).createOrReplaceTempView("lineitem") + withSQLConf(SQLConf.FILES_MAX_PARTITION_BYTES.key -> "131072")(body) + } + } + + private val q1Aggregate = """ + |select + | l_returnflag, + | l_linestatus, + | sum(l_quantity) as sum_qty, + | sum(l_extendedprice) as sum_base_price, + | sum(l_extendedprice * (1 - l_discount)) as sum_disc_price, + | sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) as sum_charge, + | avg(l_quantity) as avg_qty, + | avg(l_extendedprice) as avg_price, + | avg(l_discount) as avg_disc, + | count(*) as count_order + |from + | lineitem + |where + | l_shipdate <= date '1998-12-01' - interval '90' day + |group by + | l_returnflag, + | l_linestatus""".stripMargin + + nativeExecutorsTest("TPC-H q1 without its order by") { + withLineitem { + checkOnCluster(sql(q1Aggregate)) + } + } + + nativeExecutorsTest("TPC-H q1") { + withLineitem { + // A new Dataset each time, since a Dataset keeps the plan it was first run with. + def q1 = sql(s"$q1Aggregate\norder by\n l_returnflag,\n l_linestatus") + checkOnCluster(q1) + // checkAnswer ignores the order of the rows. + val expected = q1.collect().toSeq + onCluster(assert(q1.collect().toSeq == expected)) + } + } + nativeExecutorsTest("a query the cluster cannot run fails, naming the operator") { withTable { path => onCluster { - // A global sort needs a range exchange, which the prototype doesn't support. - val error = intercept[Exception](spark.read.parquet(path).orderBy("v").collect()) + // A small join is broadcast, and the prototype doesn't support broadcast exchanges. + val left = spark.read.parquet(path) + val right = spark.read.parquet(path).where("id < 10").select(col("k").as("k2")) + val error = + intercept[Exception](left.join(right, left("k") === right("k2")).collect()) val messages = Iterator .iterate[Throwable](error)(_.getCause) .takeWhile(_ != null) @@ -223,7 +290,7 @@ class CometNativeExecutorsSuite extends CometTestBase with AdaptiveSparkPlanHelp assert( messages.exists(message => message.contains("Comet Native Executors cannot run this query") && - message.contains("rangepartitioning")), + message.contains("CometBroadcastExchange")), error) } } From eca9510b6b098b3523f18f8eb8e3225e8e7eb820 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Wed, 28 Oct 2026 08:06:34 -0600 Subject: [PATCH 8/8] ci: list CometNativeExecutorsSuite as a manual suite It runs only against comet-scheduler and comet-executor binaries named by COMET_NATIVE_EXECUTORS_BIN_DIR, so the PR build matrix would only skip it. --- dev/ci/check-suites.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dev/ci/check-suites.py b/dev/ci/check-suites.py index 68bf972c5cf..5ced2ba6971 100644 --- a/dev/ci/check-suites.py +++ b/dev/ci/check-suites.py @@ -42,6 +42,7 @@ def file_to_class_name(path: Path) -> str | None: "org.apache.comet.CometIcebergSchedulerFailureSuite", # manual, multiple hosts + SIGKILL "org.apache.comet.cloud.s3.CometS3CredentialBridgeSuite", # manual test suite "org.apache.comet.cloud.s3.HadoopS3ACredentialProviderAdapterBridgeSuite", # manual test suite + "org.apache.comet.exec.CometNativeExecutorsSuite", # manual, needs comet-scheduler and comet-executor binaries "org.apache.comet.shuffle.CelebornReflectionCompatibilitySuite", # dedicated version matrix "org.apache.spark.sql.comet.CometPlanStabilitySuite", # abstract "org.apache.spark.sql.comet.ParquetDatetimeRebaseSuite", # abstract