From ad42523ff637a5975c2bce6c4aaba91466bd2ec7 Mon Sep 17 00:00:00 2001 From: Vaibhave Sekar Date: Sun, 9 Aug 2026 06:07:32 +0000 Subject: [PATCH 01/16] fix: scope information schema to catalog --- datafusion/catalog/src/information_schema.rs | 164 +++++++----------- .../core/src/execution/session_state.rs | 27 +-- .../information_schema_multiple_catalogs.slt | 31 ++-- 3 files changed, 96 insertions(+), 126 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index d9ad7791af67c..d399f49e4ed7e 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -21,7 +21,7 @@ use crate::streaming::StreamingTable; use crate::table::TableFunction; -use crate::{CatalogProviderList, SchemaProvider, TableProvider}; +use crate::{CatalogProvider, SchemaProvider, TableProvider}; use arrow::array::builder::{BooleanBuilder, UInt8Builder}; use arrow::{ array::{StringBuilder, UInt64Builder}, @@ -79,11 +79,15 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { - /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` - pub fn new(catalog_list: Arc) -> Self { + /// Creates a new [`InformationSchemaProvider`] for the specified catalog. + pub fn new( + catalog_name: impl Into, + catalog: Arc, + ) -> Self { Self { config: InformationSchemaConfig { - catalog_list, + catalog_name: catalog_name.into(), + catalog, table_functions: HashMap::new(), }, } @@ -102,7 +106,8 @@ impl InformationSchemaProvider { #[derive(Clone, Debug)] struct InformationSchemaConfig { - catalog_list: Arc, + catalog_name: String, + catalog: Arc, table_functions: HashMap>, } @@ -114,54 +119,44 @@ impl InformationSchemaConfig { ) -> Result<(), DataFusionError> { // create a mem table with the names of tables - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table_type) = - schema.table_type(&table_name).await? - { - builder.add_table( - &catalog_name, - &schema_name, - &table_name, - table_type, - ); - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = self.catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table_type) = schema.table_type(&table_name).await? { + builder.add_table( + &self.catalog_name, + &schema_name, + &table_name, + table_type, + ); } } } } + } - // Add a final list for the information schema tables themselves - for table_name in INFORMATION_SCHEMA_TABLES { - builder.add_table( - &catalog_name, - INFORMATION_SCHEMA, - table_name, - TableType::View, - ); - } + // Add a final list for the information schema tables themselves + for table_name in INFORMATION_SCHEMA_TABLES { + builder.add_table( + &self.catalog_name, + INFORMATION_SCHEMA, + table_name, + TableType::View, + ); } Ok(()) } fn make_schemata(&self, builder: &mut InformationSchemataBuilder) { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA - && let Some(schema) = catalog.schema(&schema_name) - { - let schema_owner = schema.owner_name(); - builder.add_schemata(&catalog_name, &schema_name, schema_owner); - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA + && let Some(schema) = self.catalog.schema(&schema_name) + { + let schema_owner = schema.owner_name(); + builder.add_schemata(&self.catalog_name, &schema_name, schema_owner); } } } @@ -170,22 +165,18 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaViewBuilder, ) -> Result<(), DataFusionError> { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - builder.add_view( - &catalog_name, - &schema_name, - &table_name, - table.get_table_definition(), - ) - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = self.catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + builder.add_view( + &self.catalog_name, + &schema_name, + &table_name, + table.get_table_definition(), + ) } } } @@ -200,26 +191,22 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaColumnsBuilder, ) -> Result<(), DataFusionError> { - for catalog_name in self.catalog_list.catalog_names() { - let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); - - for schema_name in catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - for (field_position, field) in - table.schema().fields().iter().enumerate() - { - builder.add_column( - &catalog_name, - &schema_name, - &table_name, - field_position, - field, - ) - } + for schema_name in self.catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = self.catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + for (field_position, field) in + table.schema().fields().iter().enumerate() + { + builder.add_column( + &self.catalog_name, + &schema_name, + &table_name, + field_position, + field, + ) } } } @@ -1564,7 +1551,8 @@ mod tests { #[tokio::test] async fn make_tables_uses_table_type() { let config = InformationSchemaConfig { - catalog_list: Arc::new(Fixture), + catalog_name: "acatalog".to_string(), + catalog: Arc::new(Fixture), table_functions: HashMap::new(), }; let mut builder = InformationSchemaTablesBuilder { @@ -1607,24 +1595,6 @@ mod tests { } } - impl CatalogProviderList for Fixture { - fn register_catalog( - &self, - _: String, - _: Arc, - ) -> Option> { - unimplemented!("not required for these tests") - } - - fn catalog_names(&self) -> Vec { - vec!["acatalog".to_string()] - } - - fn catalog(&self, _: &str) -> Option> { - Some(Arc::new(Self)) - } - } - impl CatalogProvider for Fixture { fn schema_names(&self) -> Vec { vec!["aschema".to_string()] diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index bfd38faacf816..782205dc9c003 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -369,26 +369,27 @@ impl SessionState { table_ref: impl Into, ) -> datafusion_common::Result> { let resolved_ref = self.resolve_table_ref(table_ref); + let catalog = self + .catalog_list + .catalog(&resolved_ref.catalog) + .ok_or_else(|| { + plan_datafusion_err!( + "failed to resolve catalog: {}", + resolved_ref.catalog + ) + })?; + if self.config.information_schema() && *resolved_ref.schema == *INFORMATION_SCHEMA { return Ok(Arc::new( - InformationSchemaProvider::new(Arc::clone(&self.catalog_list)) + InformationSchemaProvider::new(resolved_ref.catalog.to_string(), catalog) .with_table_functions(self.table_functions.clone()), )); } - self.catalog_list - .catalog(&resolved_ref.catalog) - .ok_or_else(|| { - plan_datafusion_err!( - "failed to resolve catalog: {}", - resolved_ref.catalog - ) - })? - .schema(&resolved_ref.schema) - .ok_or_else(|| { - plan_datafusion_err!("failed to resolve schema: {}", resolved_ref.schema) - }) + catalog.schema(&resolved_ref.schema).ok_or_else(|| { + plan_datafusion_err!("failed to resolve schema: {}", resolved_ref.schema) + }) } /// Add `analyzer_rule` to the end of the list of diff --git a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt index f57ecbc32a57a..8a204a1df0763 100644 --- a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt +++ b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt @@ -77,25 +77,24 @@ set datafusion.catalog.default_schema = my_other_schema; statement ok create table t3 as values(1); +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_other_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_other_catalog my_other_schema t3 + query TTTT rowsort SELECT * from information_schema.tables; ---- -datafusion information_schema columns VIEW -datafusion information_schema df_settings VIEW -datafusion information_schema parameters VIEW -datafusion information_schema routines VIEW -datafusion information_schema schemata VIEW -datafusion information_schema tables VIEW -datafusion information_schema views VIEW -my_catalog information_schema columns VIEW -my_catalog information_schema df_settings VIEW -my_catalog information_schema parameters VIEW -my_catalog information_schema routines VIEW -my_catalog information_schema schemata VIEW -my_catalog information_schema tables VIEW -my_catalog information_schema views VIEW -my_catalog my_schema t1 BASE TABLE -my_catalog my_schema t2 BASE TABLE my_other_catalog information_schema columns VIEW my_other_catalog information_schema df_settings VIEW my_other_catalog information_schema parameters VIEW From ea2d35f32fe9549135a64e1d1a0211a2c7ac86a1 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 9 Sep 2026 18:28:43 +0200 Subject: [PATCH 02/16] Add a 'system' catalog that provides a global information schema --- datafusion-cli/src/main.rs | 5 +- datafusion/catalog/src/information_schema.rs | 283 +++++++++++++----- datafusion/common/src/config.rs | 3 + datafusion/core/src/execution/context/mod.rs | 4 +- .../core/src/execution/session_state.rs | 112 +++++-- datafusion/execution/src/config.rs | 14 +- datafusion/sql/src/statement.rs | 70 +++-- 7 files changed, 366 insertions(+), 125 deletions(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index f82206a5bd184..f08cb6eeb37ef 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -370,8 +370,9 @@ fn get_session_config(args: &Args) -> Result { config_options.sql_parser.dialect = Dialect::Spark; } - let mut session_config = - SessionConfig::from(config_options).with_information_schema(true); + let mut session_config = SessionConfig::from(config_options) + .with_system_catalog(Some(String::from("system"))) + .with_information_schema(true); if args.reads_sql_from_stdin() { // When stdin carries the session's SQL — the REPL (including any rc diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index d399f49e4ed7e..9fd70cf9f7d0b 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -21,7 +21,7 @@ use crate::streaming::StreamingTable; use crate::table::TableFunction; -use crate::{CatalogProvider, SchemaProvider, TableProvider}; +use crate::{CatalogProviderList, SchemaProvider, TableProvider}; use arrow::array::builder::{BooleanBuilder, UInt8Builder}; use arrow::{ array::{StringBuilder, UInt64Builder}, @@ -79,15 +79,12 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { - /// Creates a new [`InformationSchemaProvider`] for the specified catalog. - pub fn new( - catalog_name: impl Into, - catalog: Arc, - ) -> Self { + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` + pub fn new(catalog_list: Arc) -> Self { Self { config: InformationSchemaConfig { - catalog_name: catalog_name.into(), - catalog, + system_catalog_name: None, + catalog_list, table_functions: HashMap::new(), }, } @@ -102,12 +99,17 @@ impl InformationSchemaProvider { self.config.table_functions = table_functions; self } + + pub fn with_system_catalog(mut self, catalog_name: String) -> Self { + self.config.system_catalog_name = Some(catalog_name); + self + } } #[derive(Clone, Debug)] struct InformationSchemaConfig { - catalog_name: String, - catalog: Arc, + system_catalog_name: Option, + catalog_list: Arc, table_functions: HashMap>, } @@ -119,45 +121,74 @@ impl InformationSchemaConfig { ) -> Result<(), DataFusionError> { // create a mem table with the names of tables - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = self.catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table_type) = schema.table_type(&table_name).await? { - builder.add_table( - &self.catalog_name, - &schema_name, - &table_name, - table_type, - ); + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + for table_name in schema.table_names() { + if let Some(table_type) = + schema.table_type(&table_name).await? + { + builder.add_table( + &catalog_name, + &schema_name, + &table_name, + table_type, + ); + } } } } } + + // Add a final list for the information schema tables themselves + for table_name in INFORMATION_SCHEMA_TABLES { + builder.add_table( + &catalog_name, + INFORMATION_SCHEMA, + table_name, + TableType::View, + ); + } } - // Add a final list for the information schema tables themselves - for table_name in INFORMATION_SCHEMA_TABLES { - builder.add_table( - &self.catalog_name, - INFORMATION_SCHEMA, - table_name, - TableType::View, - ); + if let Some(system_catalog_name) = &self.system_catalog_name { + for table_name in INFORMATION_SCHEMA_TABLES { + builder.add_table( + system_catalog_name, + INFORMATION_SCHEMA, + table_name, + TableType::View, + ); + } } Ok(()) } fn make_schemata(&self, builder: &mut InformationSchemataBuilder) { - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA - && let Some(schema) = self.catalog.schema(&schema_name) - { - let schema_owner = schema.owner_name(); - builder.add_schemata(&self.catalog_name, &schema_name, schema_owner); + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA + && let Some(schema) = catalog.schema(&schema_name) + { + let schema_owner = schema.owner_name(); + builder.add_schemata(&catalog_name, &schema_name, schema_owner); + } } + + // Add the information schema itself + builder.add_schemata(&catalog_name, INFORMATION_SCHEMA, None); + } + + // Add the system information schema + if let Some(system_catalog_name) = &self.system_catalog_name { + builder.add_schemata(system_catalog_name, INFORMATION_SCHEMA, None); } } @@ -165,24 +196,55 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaViewBuilder, ) -> Result<(), DataFusionError> { - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = self.catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - builder.add_view( - &self.catalog_name, - &schema_name, - &table_name, - table.get_table_definition(), - ) - } + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + Self::add_views( + builder, + &catalog_name, + &schema_name, + schema.as_ref(), + ) + .await?; } } + + // Add the information schema views themselves + Self::add_views(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; } } + // Add the system information schema + if let Some(system_catalog_name) = &self.system_catalog_name { + Self::add_views(builder, system_catalog_name, INFORMATION_SCHEMA, self) + .await?; + } + + Ok(()) + } + + async fn add_views( + builder: &mut InformationSchemaViewBuilder, + catalog_name: &str, + schema_name: &str, + schema: &dyn SchemaProvider, + ) -> Result<(), DataFusionError> { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? + && table.table_type() == TableType::View + { + builder.add_view( + catalog_name, + schema_name, + &table_name, + table.get_table_definition(), + ) + } + } Ok(()) } @@ -191,32 +253,60 @@ impl InformationSchemaConfig { &self, builder: &mut InformationSchemaColumnsBuilder, ) -> Result<(), DataFusionError> { - for schema_name in self.catalog.schema_names() { - if schema_name != INFORMATION_SCHEMA { - // schema name may not exist in the catalog, so we need to check - if let Some(schema) = self.catalog.schema(&schema_name) { - for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { - for (field_position, field) in - table.schema().fields().iter().enumerate() - { - builder.add_column( - &self.catalog_name, - &schema_name, - &table_name, - field_position, - field, - ) - } - } + for catalog_name in self.catalog_list.catalog_names() { + let catalog = self.catalog_list.catalog(&catalog_name).unwrap(); + + for schema_name in catalog.schema_names() { + if schema_name != INFORMATION_SCHEMA { + // schema name may not exist in the catalog, so we need to check + if let Some(schema) = catalog.schema(&schema_name) { + Self::add_columns( + builder, + &catalog_name, + &schema_name, + schema.as_ref(), + ) + .await?; } } } + + // Add the information schema view columns themselves + Self::add_columns(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; + } + + // Add the system information schema view columns + if let Some(system_catalog_name) = &self.system_catalog_name { + Self::add_columns(builder, system_catalog_name, INFORMATION_SCHEMA, self) + .await?; } Ok(()) } + async fn add_columns( + builder: &mut InformationSchemaColumnsBuilder, + catalog_name: &str, + schema_name: &str, + schema: &dyn SchemaProvider, + ) -> Result<(), DataFusionError> { + for table_name in schema.table_names() { + if let Some(table) = schema.table(&table_name).await? { + for (field_position, field) in table.schema().fields().iter().enumerate() + { + builder.add_column( + catalog_name, + schema_name, + &table_name, + field_position, + field, + ) + } + } + } + Ok(()) + } + /// Construct the `information_schema.df_settings` virtual table fn make_df_settings( &self, @@ -562,6 +652,24 @@ fn remove_native_type_prefix(native_type: &NativeType) -> String { #[async_trait] impl SchemaProvider for InformationSchemaProvider { + fn table_names(&self) -> Vec { + self.config.table_names() + } + + async fn table( + &self, + name: &str, + ) -> Result>, DataFusionError> { + self.config.get_table(name) + } + + fn table_exist(&self, name: &str) -> bool { + self.config.table_exist(name) + } +} + +#[async_trait] +impl SchemaProvider for InformationSchemaConfig { fn table_names(&self) -> Vec { INFORMATION_SCHEMA_TABLES .iter() @@ -573,7 +681,20 @@ impl SchemaProvider for InformationSchemaProvider { &self, name: &str, ) -> Result>, DataFusionError> { - let config = self.config.clone(); + self.get_table(name) + } + + fn table_exist(&self, name: &str) -> bool { + INFORMATION_SCHEMA_TABLES.contains(&name.to_ascii_lowercase().as_str()) + } +} + +impl InformationSchemaConfig { + fn get_table( + &self, + name: &str, + ) -> Result>, DataFusionError> { + let config = self.clone(); let table: Arc = match name.to_ascii_lowercase().as_str() { TABLES => Arc::new(InformationSchemaTables::new(config)), COLUMNS => Arc::new(InformationSchemaColumns::new(config)), @@ -589,10 +710,6 @@ impl SchemaProvider for InformationSchemaProvider { StreamingTable::try_new(Arc::clone(table.schema()), vec![table]).unwrap(), ))) } - - fn table_exist(&self, name: &str) -> bool { - INFORMATION_SCHEMA_TABLES.contains(&name.to_ascii_lowercase().as_str()) - } } #[derive(Debug)] @@ -1551,8 +1668,8 @@ mod tests { #[tokio::test] async fn make_tables_uses_table_type() { let config = InformationSchemaConfig { - catalog_name: "acatalog".to_string(), - catalog: Arc::new(Fixture), + system_catalog_name: None, + catalog_list: Arc::new(Fixture), table_functions: HashMap::new(), }; let mut builder = InformationSchemaTablesBuilder { @@ -1595,6 +1712,24 @@ mod tests { } } + impl CatalogProviderList for Fixture { + fn register_catalog( + &self, + _: String, + _: Arc, + ) -> Option> { + unimplemented!("not required for these tests") + } + + fn catalog_names(&self) -> Vec { + vec!["acatalog".to_string()] + } + + fn catalog(&self, _: &str) -> Option> { + Some(Arc::new(Self)) + } + } + impl CatalogProvider for Fixture { fn schema_names(&self) -> Vec { vec!["aschema".to_string()] diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index f770cebf88f95..6f92cae3c42eb 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -235,6 +235,9 @@ config_namespace! { /// The default schema name - this impacts what SQL queries use if not specified pub default_schema: String, default = "public".to_string() + /// The system schema name + pub system_catalog: Option, default = None + /// Should DataFusion provide access to `information_schema` /// virtual tables for displaying schema information pub information_schema: bool, default = false diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index ff1ad25811440..1919a83bd7853 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -2764,7 +2764,7 @@ mod tests { async fn sql_create_catalog() -> Result<()> { // the information schema used to introduce cyclic Arcs let ctx = SessionContext::new_with_config( - SessionConfig::new().with_information_schema(true), + SessionConfig::new().with_information_schema(true).with_system_catalog(Some(String::from("system"))), ); // Create catalog @@ -2780,7 +2780,7 @@ mod tests { .await?; // Check table exists in schema - let results = ctx.sql("SELECT * FROM information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap(); + let results = ctx.sql("SELECT * FROM system.information_schema.tables WHERE table_catalog='test' AND table_schema='abc' AND table_name = 'y'").await.unwrap().collect().await.unwrap(); assert_eq!(results[0].num_rows(), 1); Ok(()) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 869c3002a24d1..27114a89e1438 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -46,7 +46,7 @@ use datafusion_common::display::{PlanType, StringifiedPlan, ToStringifiedPlan}; use datafusion_common::tree_node::TreeNode; use datafusion_common::{ DFSchema, DataFusionError, ResolvedTableReference, TableReference, config_err, - exec_err, plan_datafusion_err, + exec_err, plan_datafusion_err, plan_err, }; use datafusion_execution::TaskContext; use datafusion_execution::config::SessionConfig; @@ -76,7 +76,9 @@ use datafusion_physical_expr_common::physical_expr::PhysicalExpr; use datafusion_physical_optimizer::optimizer::PhysicalOptimizer; use datafusion_physical_plan::ExecutionPlan; use datafusion_physical_plan::operator_statistics::StatisticsRegistry; -use datafusion_session::{PhysicalOptimizerContext, PhysicalOptimizerRule, Session}; +use datafusion_session::{ + CatalogProvider, PhysicalOptimizerContext, PhysicalOptimizerRule, Session, +}; #[cfg(feature = "sql")] use datafusion_sql::{ parser::{DFParserBuilder, Statement}, @@ -404,28 +406,75 @@ impl SessionState { table_ref: impl Into, ) -> datafusion_common::Result> { let resolved_ref = self.resolve_table_ref(table_ref); - let catalog = self - .inner - .catalog_list - .catalog(&resolved_ref.catalog) - .ok_or_else(|| { - plan_datafusion_err!( - "failed to resolve catalog: {}", - resolved_ref.catalog - ) - })?; - - if self.config.information_schema() && *resolved_ref.schema == *INFORMATION_SCHEMA + let catalog_name = resolved_ref.catalog.as_ref(); + + if let Some(system_catalog) = self.inner.config.system_catalog() + && system_catalog == catalog_name { - return Ok(Arc::new( - InformationSchemaProvider::new(resolved_ref.catalog.to_string(), catalog) - .with_table_functions(self.table_functions.clone()), - )); + if *resolved_ref.schema == *INFORMATION_SCHEMA { + return Ok(Arc::new( + InformationSchemaProvider::new(Arc::clone(&self.inner.catalog_list)) + .with_table_functions(self.inner.table_functions.clone()) + .with_system_catalog(String::from(system_catalog)), + )); + } + } else { + let catalog = + self.inner + .catalog_list + .catalog(catalog_name) + .ok_or_else(|| { + plan_datafusion_err!("failed to resolve catalog: {catalog_name}") + })?; + + if self.inner.config.information_schema() + && *resolved_ref.schema == *INFORMATION_SCHEMA + { + #[derive(Debug)] + struct SingleCatalogList { + catalog_name: String, + catalog: Arc, + } + + impl CatalogProviderList for SingleCatalogList { + fn register_catalog( + &self, + _: String, + _: Arc, + ) -> Option> { + None + } + + fn catalog_names(&self) -> Vec { + vec![self.catalog_name.clone()] + } + + fn catalog(&self, name: &str) -> Option> { + if name == self.catalog_name { + Some(Arc::clone(&self.catalog)) + } else { + None + } + } + } + + let catalog_list = Arc::new(SingleCatalogList { + catalog_name: catalog_name.to_string(), + catalog, + }); + + return Ok(Arc::new( + InformationSchemaProvider::new(catalog_list) + .with_table_functions(self.inner.table_functions.clone()), + )); + } + + if let Some(schema) = catalog.schema(&resolved_ref.schema) { + return Ok(schema); + } } - catalog.schema(&resolved_ref.schema).ok_or_else(|| { - plan_datafusion_err!("failed to resolve schema: {}", resolved_ref.schema) - }) + plan_err!("failed to resolve schema: {}", resolved_ref.schema) } /// Add `analyzer_rule` to the end of the list of @@ -588,6 +637,27 @@ impl SessionState { statement, enable_ident_normalization, )?; + + let table_refs = if let Some(system_catalog) = self.config().system_catalog() { + table_refs + .iter() + .map(|table_ref| match table_ref { + TableReference::Partial { schema, table } + if schema.as_ref() == INFORMATION_SCHEMA => + { + TableReference::Full { + catalog: Arc::from(system_catalog), + schema: Arc::clone(schema), + table: Arc::clone(table), + } + } + r => r.clone(), + }) + .collect() + } else { + table_refs + }; + Ok(table_refs) } diff --git a/datafusion/execution/src/config.rs b/datafusion/execution/src/config.rs index efaedebdadb33..2d21772016ccd 100644 --- a/datafusion/execution/src/config.rs +++ b/datafusion/execution/src/config.rs @@ -219,6 +219,11 @@ impl SessionConfig { self.options.execution.target_partitions } + /// Is the information schema enabled? + pub fn system_catalog(&self) -> Option<&str> { + self.options.catalog.system_catalog.as_deref() + } + /// Is the information schema enabled? pub fn information_schema(&self) -> bool { self.options.catalog.information_schema @@ -285,7 +290,14 @@ impl SessionConfig { self } - /// Enables or disables the inclusion of `information_schema` virtual tables + /// Enables or disables the inclusion of the system virtual catalog + pub fn with_system_catalog(mut self, catalog_name: Option) -> Self { + self.options_mut().catalog.system_catalog = catalog_name; + self + } + + /// Enables or disables the inclusion of `information_schema` virtual tables in each user + /// defined catalog pub fn with_information_schema(mut self, enabled: bool) -> Self { self.options_mut().catalog.information_schema = enabled; self diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 1a9072212f2f3..543c63eeb9e49 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -16,6 +16,7 @@ // under the License. use std::collections::{BTreeMap, HashMap, HashSet}; +use std::ops::Deref; use std::path::Path; use std::str::FromStr; use std::sync::Arc; @@ -1637,14 +1638,16 @@ impl SqlToRel<'_, S> { /// Generate a logical plan from a "SHOW TABLES" query fn show_tables_to_plan(&self) -> Result { - if self.has_table("information_schema", "tables") { - let query = "SELECT * FROM information_schema.tables;"; - let mut rewrite = DFParser::parse_sql(query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 - } else { - plan_err!("SHOW TABLES is not supported unless information_schema is enabled") - } + let Some(tables_table_ref) = self.resolve_info_table("tables") else { + return plan_err!( + "SHOW TABLES is not supported unless information_schema is enabled" + ); + }; + + let query = format!("SELECT * FROM {tables_table_ref};"); + let mut rewrite = DFParser::parse_sql(&query)?; + assert_eq!(rewrite.len(), 1); + self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 } fn describe_table_to_plan(&self, table_name: ObjectName) -> Result { @@ -2120,11 +2123,11 @@ impl SqlToRel<'_, S> { } fn show_variable_to_plan(&self, variable: &[Ident]) -> Result { - if !self.has_table("information_schema", "df_settings") { + let Some(df_settings_table_ref) = self.resolve_info_table("df_settings") else { return plan_err!( "SHOW [VARIABLE] is not supported unless information_schema is enabled" ); - } + }; let verbose = variable .last() @@ -2139,7 +2142,7 @@ impl SqlToRel<'_, S> { } let variable = object_name_to_string(&ObjectName::from(variable_vec)); - let base_query = format!("SELECT {columns} FROM information_schema.df_settings"); + let base_query = format!("SELECT {columns} FROM {df_settings_table_ref}"); let query = if variable == "all" { // Add an ORDER BY so the output comes out in a consistent order format!("{base_query} ORDER BY name") @@ -2988,11 +2991,11 @@ impl SqlToRel<'_, S> { self.options.enable_ident_normalization, )?; - if !self.has_table("information_schema", "columns") { + let Some(columns_table_ref) = self.resolve_info_table("columns") else { return plan_err!( "SHOW COLUMNS is not supported unless information_schema is enabled" ); - } + }; // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(sql_table_name)?; @@ -3005,9 +3008,8 @@ impl SqlToRel<'_, S> { "table_catalog, table_schema, table_name, column_name, data_type, is_nullable" }; - let query = format!( - "SELECT {select_list} FROM information_schema.columns WHERE {where_clause}" - ); + let query = + format!("SELECT {select_list} FROM {columns_table_ref} WHERE {where_clause}"); let mut rewrite = DFParser::parse_sql(&query)?; assert_eq!(rewrite.len(), 1); @@ -3136,11 +3138,12 @@ FROM ( &self, sql_table_name: ObjectName, ) -> Result { - if !self.has_table("information_schema", "tables") { + let Some(tables_table_ref) = self.resolve_info_table("tables") else { return plan_err!( "SHOW CREATE TABLE is not supported unless information_schema is enabled" ); - } + }; + // Figure out the where clause let where_clause = object_name_to_qualifier( &sql_table_name, @@ -3152,7 +3155,7 @@ FROM ( let _ = self.context_provider.get_table_source(table_ref)?; let query = format!( - "SELECT table_catalog, table_schema, table_name, definition FROM information_schema.views WHERE {where_clause}" + "SELECT table_catalog, table_schema, table_name, definition FROM {tables_table_ref} WHERE {where_clause}" ); let mut rewrite = DFParser::parse_sql(&query)?; @@ -3161,14 +3164,31 @@ FROM ( } /// Return true if there is a table provider available for "schema.table" - fn has_table(&self, schema: &str, table: &str) -> bool { - let tables_reference = TableReference::Partial { - schema: schema.into(), - table: table.into(), + fn resolve_info_table(&self, table: &str) -> Option { + let table_reference = if let Some(system_catalog) = + &self.context_provider.options().catalog.system_catalog + { + TableReference::Full { + catalog: system_catalog.deref().into(), + schema: "information_schema".into(), + table: table.into(), + } + } else { + TableReference::Partial { + schema: "information_schema".into(), + table: table.into(), + } }; - self.context_provider - .get_table_source(tables_reference) + + if self + .context_provider + .get_table_source(table_reference.clone()) .is_ok() + { + Some(table_reference) + } else { + None + } } fn validate_transaction_kind( From db98eb912fbbc2e5dc51a85ccfd7881f80fd2fb6 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 30 Sep 2026 00:22:49 +0200 Subject: [PATCH 03/16] Fix resolution of partial information_schema table references --- .../core/src/execution/session_state.rs | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 7959b171597e7..43cd46502c0bb 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -662,27 +662,6 @@ impl SessionState { statement, enable_ident_normalization, )?; - - let table_refs = if let Some(system_catalog) = self.config().system_catalog() { - table_refs - .iter() - .map(|table_ref| match table_ref { - TableReference::Partial { schema, table } - if schema.as_ref() == INFORMATION_SCHEMA => - { - TableReference::Full { - catalog: Arc::from(system_catalog), - schema: Arc::clone(schema), - table: Arc::clone(table), - } - } - r => r.clone(), - }) - .collect() - } else { - table_refs - }; - Ok(table_refs) } From 6da6d7224baa52c2f9a2a4a03a1913301583e33f Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Wed, 30 Sep 2026 00:23:13 +0200 Subject: [PATCH 04/16] Formatting --- datafusion/core/src/execution/context/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index 1d7efe1ae9eaf..edb6361340144 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -2846,7 +2846,9 @@ mod tests { async fn sql_create_catalog() -> Result<()> { // the information schema used to introduce cyclic Arcs let ctx = SessionContext::new_with_config( - SessionConfig::new().with_information_schema(true).with_system_catalog(Some(String::from("system"))), + SessionConfig::new() + .with_information_schema(true) + .with_system_catalog(Some(String::from("system"))), ); // Create catalog From 1b1ed047d6ef3fed81ea1f37dad147a68b07ce08 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Fri, 2 Oct 2026 21:59:25 +0200 Subject: [PATCH 05/16] Update config documentation --- datafusion/common/src/config.rs | 2 +- docs/source/user-guide/configs.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 795cad6a5a9f7..2b22de90470fa 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -237,7 +237,7 @@ config_namespace! { /// The default schema name - this impacts what SQL queries use if not specified pub default_schema: String, default = "public".to_string() - /// The system schema name + /// The system schema name - when set a global system and `information_schema` catalog will be available pub system_catalog: Option, default = None /// Should DataFusion provide access to `information_schema` diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index ba8ea52bb0850..6537031c613a0 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -68,6 +68,7 @@ The following configuration settings are available: | datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | | datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | | datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | +| datafusion.catalog.system_catalog | NULL | The system schema name - when set a global system and `information_schema` catalog will be available | | datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | | datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | | datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | From 1735fc569a5134a1e8661e3a648026d9a08ac662 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 3 Oct 2026 00:17:37 +0200 Subject: [PATCH 06/16] Move information schema resolution to session state --- .../core/src/execution/session_state.rs | 35 +++++++++++-- datafusion/sql/src/resolve.rs | 50 ------------------- 2 files changed, 32 insertions(+), 53 deletions(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 43cd46502c0bb..31ce00f16b81d 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -37,7 +37,7 @@ use arrow_schema::DataType; use arrow_schema::FieldRef; use datafusion_catalog::MemoryCatalogProviderList; use datafusion_catalog::information_schema::{ - INFORMATION_SCHEMA, InformationSchemaProvider, + INFORMATION_SCHEMA, INFORMATION_SCHEMA_TABLES, InformationSchemaProvider, }; use datafusion_catalog::{TableFunction, TableFunctionImpl}; use datafusion_common::alias::AliasGenerator; @@ -95,7 +95,9 @@ use log::{debug, info}; use object_store::ObjectStore; #[cfg(feature = "sql")] use sqlparser::{ - ast::{Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias}, + ast::{ + Expr as SQLExpr, ExprWithAlias as SQLExprWithAlias, Statement as SQLStatement, + }, dialect::dialect_from_str, }; use url::Url; @@ -671,7 +673,34 @@ impl SessionState { &self, statement: Statement, ) -> datafusion_common::Result { - let references = self.resolve_table_references(&statement)?; + let mut references = self.resolve_table_references(&statement)?; + let require_info_schema = match &statement { + Statement::Statement(s) => matches!( + s.as_ref(), + SQLStatement::ShowCreate { .. } + | SQLStatement::ShowTables { .. } + | SQLStatement::ShowColumns { .. } + | SQLStatement::ShowVariable { .. } + | SQLStatement::ShowFunctions { .. } + ), + _ => false, + }; + + if require_info_schema { + let info_tables = INFORMATION_SCHEMA_TABLES.iter(); + let info_table_refs: Vec<_> = if let Some(system_catalog) = + self.config().system_catalog() + { + info_tables + .map(|n| TableReference::full(system_catalog, INFORMATION_SCHEMA, *n)) + .collect() + } else { + info_tables + .map(|n| TableReference::partial(INFORMATION_SCHEMA, *n)) + .collect() + }; + references.extend(info_table_refs); + } let mut provider = SessionContextProvider { state: self, diff --git a/datafusion/sql/src/resolve.rs b/datafusion/sql/src/resolve.rs index b6b1a74849be0..9abc687a2c39b 100644 --- a/datafusion/sql/src/resolve.rs +++ b/datafusion/sql/src/resolve.rs @@ -25,28 +25,6 @@ use crate::planner::object_name_to_table_reference; use datafusion_common::TableReference; use sqlparser::ast::*; -// following constants are used in `resolve_table_references` -// and should be same as `datafusion/catalog/src/information_schema.rs` -const INFORMATION_SCHEMA: &str = "information_schema"; -const TABLES: &str = "tables"; -const VIEWS: &str = "views"; -const COLUMNS: &str = "columns"; -const DF_SETTINGS: &str = "df_settings"; -const SCHEMATA: &str = "schemata"; -const ROUTINES: &str = "routines"; -const PARAMETERS: &str = "parameters"; - -/// All information schema tables -const INFORMATION_SCHEMA_TABLES: &[&str] = &[ - TABLES, - VIEWS, - COLUMNS, - DF_SETTINGS, - SCHEMATA, - ROUTINES, - PARAMETERS, -]; - // Collect table/CTE references as `TableReference`s and normalize them during traversal. // This avoids a second normalization/conversion pass after visiting the AST. struct RelationVisitor { @@ -127,34 +105,6 @@ impl Visitor for RelationVisitor { self.insert_relation(obj_name)?; } - // SHOW statements will later be rewritten into a SELECT from the information_schema - let requires_information_schema = matches!( - statement, - Statement::ShowFunctions { .. } - | Statement::ShowVariable { .. } - | Statement::ShowStatus { .. } - | Statement::ShowVariables { .. } - | Statement::ShowCreate { .. } - | Statement::ShowColumns { .. } - | Statement::ShowTables { .. } - | Statement::ShowCollation { .. } - ); - if requires_information_schema { - for s in INFORMATION_SCHEMA_TABLES { - // Information schema references are synthesized here, so convert directly. - let obj = ObjectName::from(vec![ - Ident::new(INFORMATION_SCHEMA), - Ident::new(*s), - ]); - match object_name_to_table_reference(obj, self.enable_ident_normalization) - { - Ok(tbl_ref) => { - self.relations.insert(tbl_ref); - } - Err(e) => return ControlFlow::Break(e), - } - } - } ControlFlow::Continue(()) } } From 1e5b40e3c05cd1e6ab277e01a4d0b5a3f61a8473 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Sat, 3 Oct 2026 15:19:16 +0200 Subject: [PATCH 07/16] Revert information schema content changes --- datafusion/catalog/src/information_schema.rs | 21 +------------------ datafusion/sql/src/statement.rs | 2 +- .../test_files/information_schema.slt | 2 ++ 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 9fd70cf9f7d0b..30ed1a3753a98 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -181,14 +181,6 @@ impl InformationSchemaConfig { builder.add_schemata(&catalog_name, &schema_name, schema_owner); } } - - // Add the information schema itself - builder.add_schemata(&catalog_name, INFORMATION_SCHEMA, None); - } - - // Add the system information schema - if let Some(system_catalog_name) = &self.system_catalog_name { - builder.add_schemata(system_catalog_name, INFORMATION_SCHEMA, None); } } @@ -234,9 +226,7 @@ impl InformationSchemaConfig { schema: &dyn SchemaProvider, ) -> Result<(), DataFusionError> { for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? - && table.table_type() == TableType::View - { + if let Some(table) = schema.table(&table_name).await? { builder.add_view( catalog_name, schema_name, @@ -270,15 +260,6 @@ impl InformationSchemaConfig { } } } - - // Add the information schema view columns themselves - Self::add_columns(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; - } - - // Add the system information schema view columns - if let Some(system_catalog_name) = &self.system_catalog_name { - Self::add_columns(builder, system_catalog_name, INFORMATION_SCHEMA, self) - .await?; } Ok(()) diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index a88048a0cbcf2..75a7e6170fe6f 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -3146,7 +3146,7 @@ FROM ( &self, sql_table_name: ObjectName, ) -> Result { - let Some(tables_table_ref) = self.resolve_info_table("tables") else { + let Some(tables_table_ref) = self.resolve_info_table("views") else { return plan_err!( "SHOW CREATE TABLE is not supported unless information_schema is enabled" ); diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index f6633a165df18..b34c772f618fe 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -213,6 +213,7 @@ datafusion.catalog.has_header true datafusion.catalog.information_schema true datafusion.catalog.location NULL datafusion.catalog.newlines_in_values false +datafusion.catalog.system_catalog NULL datafusion.execution.batch_size 8192 datafusion.execution.coalesce_batches true datafusion.execution.collect_statistics true @@ -375,6 +376,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. +datafusion.catalog.system_catalog NULL The system schema name - when set a global system and `information_schema` catalog will be available datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. From f728f064c3b5f94b4183576280556a2538f1fe4d Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 20 Oct 2026 14:51:54 +0200 Subject: [PATCH 08/16] Make segregated information schema opt-in --- datafusion/catalog/src/information_schema.rs | 45 ++++ .../core/src/execution/session_state.rs | 118 ++++---- .../information_schema_multiple_catalogs.slt | 252 ++++++++++++++++++ 3 files changed, 355 insertions(+), 60 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index f466533cfa05f..7a2ad76bf1e17 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -44,6 +44,7 @@ use datafusion_expr::{TableType, Volatility}; use datafusion_physical_plan::SendableRecordBatchStream; use datafusion_physical_plan::stream::RecordBatchStreamAdapter; use datafusion_physical_plan::streaming::PartitionStream; +use datafusion_session::CatalogProvider; use itertools::Itertools; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; @@ -81,6 +82,50 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` + pub fn for_catalog(catalog_name: String, catalog: Arc) -> Self { + #[derive(Debug)] + struct SingleCatalogList { + catalog_name: String, + catalog: Arc, + } + + impl CatalogProviderList for SingleCatalogList { + fn register_catalog( + &self, + _: String, + _: Arc, + ) -> Option> { + None + } + + fn catalog_names(&self) -> Vec { + vec![self.catalog_name.clone()] + } + + fn catalog(&self, name: &str) -> Option> { + if name == self.catalog_name { + Some(Arc::clone(&self.catalog)) + } else { + None + } + } + } + + let catalog_list = Arc::new(SingleCatalogList { + catalog_name, + catalog, + }); + + Self { + config: InformationSchemaConfig { + system_catalog_name: None, + catalog_list, + table_functions: HashMap::new(), + }, + } + } + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` pub fn new(catalog_list: Arc) -> Self { Self { diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 09105dd953dbb..10f507d1d5dc4 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -423,73 +423,71 @@ impl SessionState { let resolved_ref = self.resolve_table_ref(table_ref); let catalog_name = resolved_ref.catalog.as_ref(); - if let Some(system_catalog) = self.inner.config.system_catalog() - && system_catalog == catalog_name + // Special case handling for `information_schema`. + if *resolved_ref.schema == *INFORMATION_SCHEMA + && self.inner.config.information_schema() { - if *resolved_ref.schema == *INFORMATION_SCHEMA { - return Ok(Arc::new( - InformationSchemaProvider::new(Arc::clone(&self.inner.catalog_list)) - .with_table_functions(self.inner.table_functions.clone()) - .with_system_catalog(String::from(system_catalog)), - )); - } - } else { - let catalog = - self.inner - .catalog_list - .catalog(catalog_name) - .ok_or_else(|| { - plan_datafusion_err!("failed to resolve catalog: {catalog_name}") - })?; - - if self.inner.config.information_schema() - && *resolved_ref.schema == *INFORMATION_SCHEMA - { - #[derive(Debug)] - struct SingleCatalogList { - catalog_name: String, - catalog: Arc, - } - - impl CatalogProviderList for SingleCatalogList { - fn register_catalog( - &self, - _: String, - _: Arc, - ) -> Option> { - None - } - - fn catalog_names(&self) -> Vec { - vec![self.catalog_name.clone()] - } - - fn catalog(&self, name: &str) -> Option> { - if name == self.catalog_name { - Some(Arc::clone(&self.catalog)) - } else { - None - } + match self.inner.config.system_catalog() { + Some(system_catalog) => { + if system_catalog == catalog_name { + // Global catalog configured, catalog name references it + // => information schema provider for full catalog list + return Ok(Arc::new( + InformationSchemaProvider::new(Arc::clone( + &self.inner.catalog_list, + )) + .with_table_functions(self.inner.table_functions.clone()) + .with_system_catalog(String::from(system_catalog)), + )); + } else { + // Global catalog configured, catalog name is other catalog + // => information schema provider for named catalog + let catalog = self.resolve_catalog(catalog_name)?; + return Ok(Arc::new( + InformationSchemaProvider::for_catalog( + catalog_name.to_string(), + catalog, + ) + .with_table_functions(self.inner.table_functions.clone()), + )); } } - - let catalog_list = Arc::new(SingleCatalogList { - catalog_name: catalog_name.to_string(), - catalog, - }); - - return Ok(Arc::new( - InformationSchemaProvider::new(catalog_list) + None => { + // Global catalog not configured + // => information schema provider for full catalog list + // Resolve the catalog (even though it's not needed for the InformationSchemaProvider) + // so references to non-existent catalogs report an error. + self.resolve_catalog(catalog_name)?; + return Ok(Arc::new( + InformationSchemaProvider::new(Arc::clone( + &self.inner.catalog_list, + )) .with_table_functions(self.inner.table_functions.clone()), - )); - } - - if let Some(schema) = catalog.schema(&resolved_ref.schema) { - return Ok(schema); + )); + } } } - plan_err!("failed to resolve schema: {}", resolved_ref.schema) + let catalog = self.resolve_catalog(catalog_name)?; + let Some(schema) = catalog.schema(&resolved_ref.schema) else { + return plan_err!("failed to resolve schema: {}", resolved_ref.schema); + }; + + Ok(schema) + } + + fn resolve_catalog( + &self, + catalog_name: &str, + ) -> Result, DataFusionError> { + let catalog = self + .inner + .catalog_list + .catalog(catalog_name) + .ok_or_else(|| { + plan_datafusion_err!("failed to resolve catalog: {catalog_name}") + })?; + Ok(catalog) } /// Add `analyzer_rule` to the end of the list of diff --git a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt index ae5a54902f23f..d4048a8b6b327 100644 --- a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt +++ b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt @@ -83,17 +83,71 @@ WHERE table_schema <> 'information_schema'; ---- my_catalog my_schema t1 my_catalog my_schema t2 +my_other_catalog my_other_schema t3 query TTT rowsort SELECT table_catalog, table_schema, table_name FROM my_other_catalog.information_schema.tables WHERE table_schema <> 'information_schema'; ---- +my_catalog my_schema t1 +my_catalog my_schema t2 my_other_catalog my_other_schema t3 query TTTT rowsort SELECT * from information_schema.tables; ---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE + +############ +# Configure a system catalog +# +# When `datafusion.catalog.system_catalog` is set, the `information_schema` +# of each catalog only describes the objects in that catalog. Only the +# `information_schema` of the system catalog describes all catalogs. +########### +statement ok +set datafusion.catalog.system_catalog = system; + +query TTTT rowsort +SELECT * FROM my_catalog.information_schema.tables; +---- +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE + +query TTTT rowsort +SELECT * FROM my_other_catalog.information_schema.tables; +---- my_other_catalog information_schema columns VIEW my_other_catalog information_schema df_settings VIEW my_other_catalog information_schema parameters VIEW @@ -103,6 +157,204 @@ my_other_catalog information_schema tables VIEW my_other_catalog information_schema views VIEW my_other_catalog my_other_schema t3 BASE TABLE +query TTTT rowsort +SELECT * FROM datafusion.information_schema.tables; +---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW + +# Unqualified `information_schema` resolves against the default catalog +query TTTT rowsort +SELECT * FROM information_schema.tables; +---- +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE + +query TTTT rowsort +SELECT * FROM system.information_schema.tables; +---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE +system information_schema columns VIEW +system information_schema df_settings VIEW +system information_schema parameters VIEW +system information_schema routines VIEW +system information_schema schemata VIEW +system information_schema tables VIEW +system information_schema views VIEW + +query TTTTTTT rowsort +SELECT * FROM my_catalog.information_schema.schemata; +---- +my_catalog my_schema NULL NULL NULL NULL NULL + +query TTTTTTT rowsort +SELECT * FROM my_other_catalog.information_schema.schemata; +---- +my_other_catalog my_other_schema NULL NULL NULL NULL NULL + +query TTTTTTT rowsort +SELECT * FROM system.information_schema.schemata; +---- +datafusion public NULL NULL NULL NULL NULL +my_catalog my_schema NULL NULL NULL NULL NULL +my_other_catalog my_other_schema NULL NULL NULL NULL NULL + +query TTTT rowsort +SELECT table_catalog, table_schema, table_name, column_name +FROM my_catalog.information_schema.columns; +---- +my_catalog my_schema t1 column1 +my_catalog my_schema t2 column1 + +query TTTT rowsort +SELECT table_catalog, table_schema, table_name, column_name +FROM my_other_catalog.information_schema.columns; +---- +my_other_catalog my_other_schema t3 column1 + +query TTTT rowsort +SELECT table_catalog, table_schema, table_name, column_name +FROM system.information_schema.columns; +---- +my_catalog my_schema t1 column1 +my_catalog my_schema t2 column1 +my_other_catalog my_other_schema t3 column1 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_catalog.information_schema.views +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM system.information_schema.views +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 +my_other_catalog my_other_schema t3 + +# SHOW statements use the system catalog's information schema +query TTTT rowsort +SHOW TABLES; +---- +datafusion information_schema columns VIEW +datafusion information_schema df_settings VIEW +datafusion information_schema parameters VIEW +datafusion information_schema routines VIEW +datafusion information_schema schemata VIEW +datafusion information_schema tables VIEW +datafusion information_schema views VIEW +my_catalog information_schema columns VIEW +my_catalog information_schema df_settings VIEW +my_catalog information_schema parameters VIEW +my_catalog information_schema routines VIEW +my_catalog information_schema schemata VIEW +my_catalog information_schema tables VIEW +my_catalog information_schema views VIEW +my_catalog my_schema t1 BASE TABLE +my_catalog my_schema t2 BASE TABLE +my_other_catalog information_schema columns VIEW +my_other_catalog information_schema df_settings VIEW +my_other_catalog information_schema parameters VIEW +my_other_catalog information_schema routines VIEW +my_other_catalog information_schema schemata VIEW +my_other_catalog information_schema tables VIEW +my_other_catalog information_schema views VIEW +my_other_catalog my_other_schema t3 BASE TABLE +system information_schema columns VIEW +system information_schema df_settings VIEW +system information_schema parameters VIEW +system information_schema routines VIEW +system information_schema schemata VIEW +system information_schema tables VIEW +system information_schema views VIEW + +query TTTTTT +SHOW COLUMNS FROM my_catalog.my_schema.t1; +---- +my_catalog my_schema t1 column1 Int64 NO + +# The information schema of a catalog that does not exist cannot be queried +statement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not found +SELECT * FROM no_such_catalog.information_schema.tables; + +# A system catalog can be an existing catalog. Its information schema then +# describes all catalogs +statement ok +set datafusion.catalog.system_catalog = my_catalog; + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 +my_other_catalog my_other_schema t3 + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_other_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_other_catalog my_other_schema t3 + +# Resetting the system catalog makes every information schema global again +statement ok +RESET datafusion.catalog.system_catalog; + +query TTT rowsort +SELECT table_catalog, table_schema, table_name +FROM my_other_catalog.information_schema.tables +WHERE table_schema <> 'information_schema'; +---- +my_catalog my_schema t1 +my_catalog my_schema t2 +my_other_catalog my_other_schema t3 + +# Without a system catalog, the information schema of a catalog that does not +# exist cannot be queried either +statement error DataFusion error: Error during planning: table 'no_such_catalog.information_schema.tables' not found +SELECT * FROM no_such_catalog.information_schema.tables; + # Cleanup statement ok From b43795f3be627a20ce1b742d857bd447fe610114 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 20 Oct 2026 15:09:32 +0200 Subject: [PATCH 09/16] Add extra detail to `system_catalog` description --- datafusion/common/src/config.rs | 9 ++++++--- .../sqllogictest/test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 91c347cd6ea45..7efa7775f6325 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -238,13 +238,16 @@ config_namespace! { /// The default schema name - this impacts what SQL queries use if not specified pub default_schema: String, default = "public".to_string() - /// The system schema name - when set a global system and `information_schema` catalog will be available - pub system_catalog: Option, default = None - /// Should DataFusion provide access to `information_schema` /// virtual tables for displaying schema information pub information_schema: bool, default = false + /// The system schema name - when set together with `information_schema`, the + /// `information_schema` for each catalog will only describe objects contained in the + /// catalog itself. The global, cross-catalog `information_schema` is available under + /// the virtual catalog named by this option. + pub system_catalog: Option, default = None + /// Location scanned to load tables for `default` schema pub location: Option, default = None diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 7420463d3cc4d..4e8f9959540a7 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -383,7 +383,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.catalog.system_catalog NULL The system schema name - when set a global system and `information_schema` catalog will be available +datafusion.catalog.system_catalog NULL The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index bbc444e0af51b..cdd2246fbff8b 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -68,8 +68,8 @@ The following configuration settings are available: | datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | | datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | | datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | -| datafusion.catalog.system_catalog | NULL | The system schema name - when set a global system and `information_schema` catalog will be available | | datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | +| datafusion.catalog.system_catalog | NULL | The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. | | datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | | datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | | datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | From de365473fa974d81b6f3a3601727902a06f43268 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Tue, 20 Oct 2026 15:19:00 +0200 Subject: [PATCH 10/16] Correct `system_catalog` short description --- datafusion/common/src/config.rs | 2 +- datafusion/sqllogictest/test_files/information_schema.slt | 2 +- docs/source/user-guide/configs.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 7efa7775f6325..6da0216f7e093 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -242,7 +242,7 @@ config_namespace! { /// virtual tables for displaying schema information pub information_schema: bool, default = false - /// The system schema name - when set together with `information_schema`, the + /// The system catalog name - when set together with `information_schema`, the /// `information_schema` for each catalog will only describe objects contained in the /// catalog itself. The global, cross-catalog `information_schema` is available under /// the virtual catalog named by this option. diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 4e8f9959540a7..03a1976306bf0 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -383,7 +383,7 @@ datafusion.catalog.has_header true Default value for `format.has_header` for `CR datafusion.catalog.information_schema true Should DataFusion provide access to `information_schema` virtual tables for displaying schema information datafusion.catalog.location NULL Location scanned to load tables for `default` schema datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. -datafusion.catalog.system_catalog NULL The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. +datafusion.catalog.system_catalog NULL The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Defaults to true. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index cdd2246fbff8b..d13c650061ff4 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -69,7 +69,7 @@ The following configuration settings are available: | datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | | datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | | datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | -| datafusion.catalog.system_catalog | NULL | The system schema name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. | +| datafusion.catalog.system_catalog | NULL | The system catalog name - when set together with `information_schema`, the `information_schema` for each catalog will only describe objects contained in the catalog itself. The global, cross-catalog `information_schema` is available under the virtual catalog named by this option. | | datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | | datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | | datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | From b40d5877efe8187a7c99667cc051b061603bb2f7 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 22:59:32 +0200 Subject: [PATCH 11/16] Fix 'explain show tables' --- .../core/src/execution/session_state.rs | 29 +++++++++++-------- .../test_files/information_schema.slt | 7 +++++ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 10f507d1d5dc4..0debae505b6b5 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -102,6 +102,7 @@ use sqlparser::{ }; use url::Url; use uuid::Uuid; +use datafusion_sql::parser::ExplainStatement; /// `SessionState` contains all the necessary state to plan and execute queries, /// such as configuration, functions, and runtime environment. Please see the @@ -672,19 +673,8 @@ impl SessionState { statement: Statement, ) -> datafusion_common::Result { let mut references = self.resolve_table_references(&statement)?; - let require_info_schema = match &statement { - Statement::Statement(s) => matches!( - s.as_ref(), - SQLStatement::ShowCreate { .. } - | SQLStatement::ShowTables { .. } - | SQLStatement::ShowColumns { .. } - | SQLStatement::ShowVariable { .. } - | SQLStatement::ShowFunctions { .. } - ), - _ => false, - }; - if require_info_schema { + if Self::requires_information_schema(&statement) { let info_tables = INFORMATION_SCHEMA_TABLES.iter(); let info_table_refs: Vec<_> = if let Some(system_catalog) = self.config().system_catalog() @@ -721,6 +711,21 @@ impl SessionState { query.statement_to_plan(statement) } + fn requires_information_schema(statement: &Statement) -> bool { + match &statement { + Statement::Explain(ExplainStatement { statement, .. }) => Self::requires_information_schema(statement), + Statement::Statement(s) => matches!( + s.as_ref(), + SQLStatement::ShowCreate { .. } + | SQLStatement::ShowTables { .. } + | SQLStatement::ShowColumns { .. } + | SQLStatement::ShowVariable { .. } + | SQLStatement::ShowFunctions { .. } + ), + _ => false, + } + } + #[cfg(feature = "sql")] fn get_parser_options(&self) -> ParserOptions { let sql_parser_options = &self.config().options().sql_parser; diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 03a1976306bf0..e9d7e861fbb5c 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -653,6 +653,13 @@ datafusion information_schema tables VIEW datafusion information_schema views VIEW +query TT +EXPLAIN SHOW TABLES +---- +logical_plan TableScan: information_schema.tables projection=[table_catalog, table_schema, table_name, table_type] +physical_plan StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, table_type] + + # information_schema_show_tables_no_information_schema statement ok From 94c64e60def8170a392aa1b5d1d5d63a5cbb54db Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 23:00:15 +0200 Subject: [PATCH 12/16] Do not use partioned information schema in CLI by default --- datafusion-cli/src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index 16786bb3a8c1f..a3c3ad8ca9720 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -381,7 +381,6 @@ fn get_session_config(args: &Args) -> Result { } let mut session_config = SessionConfig::from(config_options) - .with_system_catalog(Some(String::from("system"))) .with_information_schema(true); if args.reads_sql_from_stdin() { From 4bc355381c27609b42c06890bb63abf842bc889d Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 23:06:30 +0200 Subject: [PATCH 13/16] Formatting --- datafusion-cli/src/main.rs | 4 ++-- datafusion/core/src/execution/session_state.rs | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/datafusion-cli/src/main.rs b/datafusion-cli/src/main.rs index a3c3ad8ca9720..d24f924bfb5bd 100644 --- a/datafusion-cli/src/main.rs +++ b/datafusion-cli/src/main.rs @@ -380,8 +380,8 @@ fn get_session_config(args: &Args) -> Result { config_options.sql_parser.dialect = Dialect::Spark; } - let mut session_config = SessionConfig::from(config_options) - .with_information_schema(true); + let mut session_config = + SessionConfig::from(config_options).with_information_schema(true); if args.reads_sql_from_stdin() { // When stdin carries the session's SQL — the REPL (including any rc diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 0debae505b6b5..9abf9e92ba0b2 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -89,6 +89,7 @@ use datafusion_sql::{ use async_trait::async_trait; use chrono::{DateTime, Utc}; +use datafusion_sql::parser::ExplainStatement; use futures::future::BoxFuture; use itertools::Itertools; use log::{debug, info}; @@ -102,7 +103,6 @@ use sqlparser::{ }; use url::Url; use uuid::Uuid; -use datafusion_sql::parser::ExplainStatement; /// `SessionState` contains all the necessary state to plan and execute queries, /// such as configuration, functions, and runtime environment. Please see the @@ -713,7 +713,9 @@ impl SessionState { fn requires_information_schema(statement: &Statement) -> bool { match &statement { - Statement::Explain(ExplainStatement { statement, .. }) => Self::requires_information_schema(statement), + Statement::Explain(ExplainStatement { statement, .. }) => { + Self::requires_information_schema(statement) + } Statement::Statement(s) => matches!( s.as_ref(), SQLStatement::ShowCreate { .. } From 2678ab791a93869c57cbfe5f5fd2d18ac88f44d9 Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 22 Oct 2026 23:35:17 +0200 Subject: [PATCH 14/16] Revert adding information schema views to information_schema.views --- datafusion/catalog/src/information_schema.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 7a2ad76bf1e17..97b55169e0e88 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -251,18 +251,9 @@ impl InformationSchemaConfig { .await?; } } - - // Add the information schema views themselves - Self::add_views(builder, &catalog_name, INFORMATION_SCHEMA, self).await?; } } - // Add the system information schema - if let Some(system_catalog_name) = &self.system_catalog_name { - Self::add_views(builder, system_catalog_name, INFORMATION_SCHEMA, self) - .await?; - } - Ok(()) } From 24496566dfa7d923b659aa178da81aa54455418d Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Mon, 5 Oct 2026 17:59:37 +0200 Subject: [PATCH 15/16] Decouple 'show' statements from information schema tables --- datafusion-cli/src/exec.rs | 4 +- datafusion/catalog/src/information_schema.rs | 1150 ++++++----------- datafusion/common/src/information_schema.rs | 1066 +++++++++++++++ datafusion/common/src/lib.rs | 3 + datafusion/common/src/table_type.rs | 42 + datafusion/core/src/physical_planner.rs | 124 +- datafusion/expr/src/logical_plan/builder.rs | 71 +- datafusion/expr/src/logical_plan/display.rs | 63 +- datafusion/expr/src/logical_plan/mod.rs | 8 +- datafusion/expr/src/logical_plan/plan.rs | 141 +- datafusion/expr/src/logical_plan/tree_node.rs | 6 +- datafusion/expr/src/table_source.rs | 22 +- .../optimizer/src/common_subexpr_eliminate.rs | 2 +- datafusion/optimizer/src/decorrelate.rs | 2 +- .../optimizer/src/optimize_projections/mod.rs | 4 +- datafusion/optimizer/src/optimizer.rs | 2 +- datafusion/proto/src/logical_plan/mod.rs | 4 +- datafusion/sql/src/statement.rs | 276 +--- datafusion/sql/src/unparser/plan.rs | 2 +- .../runner.rs | 4 +- .../test_files/information_schema.slt | 39 +- .../information_schema_multiple_catalogs.slt | 14 +- .../src/logical_plan/producer/rel/mod.rs | 4 +- 23 files changed, 1893 insertions(+), 1160 deletions(-) create mode 100644 datafusion/common/src/information_schema.rs create mode 100644 datafusion/common/src/table_type.rs diff --git a/datafusion-cli/src/exec.rs b/datafusion-cli/src/exec.rs index 288ce4b7351b6..f4cc3911be510 100644 --- a/datafusion-cli/src/exec.rs +++ b/datafusion-cli/src/exec.rs @@ -373,9 +373,7 @@ impl AdjustedPrintOptions { // all rows if matches!( plan, - LogicalPlan::Explain(_) - | LogicalPlan::DescribeTable(_) - | LogicalPlan::Analyze(_) + LogicalPlan::Explain(_) | LogicalPlan::Show(_) | LogicalPlan::Analyze(_) ) { self.inner.maxrows = MaxRows::Unlimited; } diff --git a/datafusion/catalog/src/information_schema.rs b/datafusion/catalog/src/information_schema.rs index 97b55169e0e88..b94f59005028a 100644 --- a/datafusion/catalog/src/information_schema.rs +++ b/datafusion/catalog/src/information_schema.rs @@ -22,29 +22,46 @@ use crate::streaming::StreamingTable; use crate::table::TableFunction; use crate::{CatalogProviderList, SchemaProvider, TableProvider}; -use arrow::array::builder::{BooleanBuilder, UInt8Builder}; -use arrow::{ - array::{StringBuilder, UInt64Builder}, - datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}, - record_batch::RecordBatch, -}; +use arrow::array::StringArray; +use arrow::array::builder::{ListBuilder, StringBuilder}; +use arrow::compute::kernels::filter::filter_record_batch; +use arrow::compute::like; +use arrow::datatypes::{DataType, Field, FieldRef, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; use async_trait::async_trait; -use datafusion_common::DataFusionError; -use datafusion_common::config::{ConfigEntry, ConfigOptions}; +use datafusion_common::config::ConfigOptions; use datafusion_common::error::Result; +use datafusion_common::information_schema::{ + ColumnsDetail, InformationSchemaColumnsBuilder, InformationSchemaDfSettingsBuilder, + InformationSchemaParametersBuilder, InformationSchemaRoutinesBuilder, + InformationSchemaTablesBuilder, InformationSchemaViewBuilder, columns_schema, + df_settings_schema, parameters_schema, routines_schema, show_functions_schema, + tables_schema, views_schema, +}; +use datafusion_common::{ + DataFusionError, ResolvedTableReference, plan_datafusion_err, plan_err, +}; +// Re-exported (rather than just `use`d) because these were already part of +// this module's public API before it started sharing its schema/builder +// definitions with `datafusion_common::information_schema`. +pub use datafusion_common::information_schema::{ + InformationSchemataBuilder, schemata_schema, +}; use datafusion_common::types::{LogicalType, NativeType}; +use datafusion_datasource::memory::MemorySourceConfig; use datafusion_execution::TaskContext; use datafusion_execution::runtime_env::RuntimeEnv; use datafusion_expr::function::WindowUDFFieldArgs; use datafusion_expr::type_coercion::functions::fields_with_udf; use datafusion_expr::{ - AggregateUDF, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature, WindowUDF, + AggregateUDF, Documentation, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature, + WindowUDF, }; use datafusion_expr::{TableType, Volatility}; -use datafusion_physical_plan::SendableRecordBatchStream; use datafusion_physical_plan::stream::RecordBatchStreamAdapter; -use datafusion_physical_plan::streaming::PartitionStream; -use datafusion_session::CatalogProvider; +use datafusion_physical_plan::streaming::{PartitionStream, StreamingTableExec}; +use datafusion_physical_plan::{ExecutionPlan, SendableRecordBatchStream}; +use datafusion_session::{CatalogProvider, Session}; use itertools::Itertools; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Debug; @@ -82,6 +99,15 @@ pub struct InformationSchemaProvider { } impl InformationSchemaProvider { + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` + pub fn from(session: &dyn Session) -> Self { + let mut provider = Self::new(session.catalog_list()); + provider.config.information_schema = session.config().information_schema(); + provider.config.system_catalog_name = + session.config().system_catalog().map(str::to_owned); + provider + } + /// Creates a new [`InformationSchemaProvider`] for the provided `catalog_list` pub fn for_catalog(catalog_name: String, catalog: Arc) -> Self { #[derive(Debug)] @@ -120,6 +146,7 @@ impl InformationSchemaProvider { Self { config: InformationSchemaConfig { system_catalog_name: None, + information_schema: true, catalog_list, table_functions: HashMap::new(), }, @@ -131,6 +158,7 @@ impl InformationSchemaProvider { Self { config: InformationSchemaConfig { system_catalog_name: None, + information_schema: true, catalog_list, table_functions: HashMap::new(), }, @@ -151,11 +179,295 @@ impl InformationSchemaProvider { self.config.system_catalog_name = Some(catalog_name); self } + + /// `SHOW TABLES`. + pub fn show_tables(&self) -> Result> { + Ok(Arc::new(StreamingTableExec::try_new( + SchemaRef::from(tables_schema()), + vec![Arc::new(InformationSchemaTables::new(self.config.clone()))], + None, + vec![], + false, + None, + )?)) + } + + /// `DESCRIBE `. + pub fn describe_schema(&self, schema: &Schema) -> Result> { + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Describe); + for (position, field) in schema.fields().iter().enumerate() { + builder.add_column("", "", "", position, field); + } + record_batch_to_exec(builder.finish()) + } + + /// `DESCRIBE ` / `SHOW [FULL|EXTENDED] COLUMNS FROM
`, at + /// the given [`ColumnsDetail`] level. + pub async fn show_columns( + &self, + table_ref: &ResolvedTableReference, + detail: ColumnsDetail, + ) -> Result> { + let table = self.resolve_table(table_ref).await?; + + let mut builder = InformationSchemaColumnsBuilder::new(detail); + for (position, field) in table.schema().fields().iter().enumerate() { + builder.add_column( + &table_ref.catalog, + &table_ref.schema, + &table_ref.table, + position, + field, + ); + } + record_batch_to_exec(builder.finish()) + } + + /// `SHOW CREATE TABLE
`. + pub async fn show_create_table( + &self, + table_ref: &ResolvedTableReference, + ) -> Result> { + let table = self.resolve_table(table_ref).await?; + + let mut builder = InformationSchemaViewBuilder::new(); + builder.add_view( + table_ref.catalog.as_ref(), + table_ref.schema.as_ref(), + table_ref.table.as_ref(), + table.get_table_definition(), + ); + let batch = builder.finish(); + record_batch_to_exec(batch) + } + + async fn resolve_table( + &self, + table_ref: &ResolvedTableReference, + ) -> Result, DataFusionError> { + let not_found = || plan_datafusion_err!("table '{table_ref}' not found"); + let schema_provider = self + .config + .catalog_list + .catalog(&table_ref.catalog) + .ok_or_else(not_found)? + .schema(&table_ref.schema) + .ok_or_else(not_found)?; + let table = schema_provider + .table(&table_ref.table) + .await? + .ok_or_else(not_found)?; + Ok(table) + } + + /// `SHOW ` / `SHOW ALL`. `name` is `None` for `SHOW ALL`. + pub fn show_variables( + &self, + config_options: &ConfigOptions, + runtime_env: &Arc, + name: Option<&str>, + verbose: bool, + ) -> Result> { + let name = name.map(|n| { + if n == "timezone" || n == "time.zone" { + // we could introduce alias in OptionDefinition if this string matching thing grows + "datafusion.execution.time_zone" + } else { + n + } + }); + + if let Some(name) = name { + // These values are what are used to make the information_schema + // table, so we just check here, before actually executing the + // query, if it would produce no results, and error preemptively + // if it would (for a better UX). + let is_valid_variable = + config_options.entries().iter().any(|opt| opt.key == name); + let is_runtime_variable = name.starts_with("datafusion.runtime."); + if !is_valid_variable && !is_runtime_variable { + return plan_err!( + "'{name}' is not a variable which can be viewed with 'SHOW'" + ); + } + } + let mut builder = InformationSchemaDfSettingsBuilder::new(verbose); + self.config + .make_df_settings(config_options, runtime_env, name, &mut builder); + let batch = builder.finish(); + record_batch_to_exec(batch) + } + + /// `SHOW FUNCTIONS [LIKE ]`. + /// + /// Built directly from the session's scalar/aggregate/window UDF and + /// UDTF registries (the same sources `information_schema.routines` / + /// `information_schema.parameters` are built from, see + /// `InformationSchemaConfig::make_routines`/`::make_parameters`) by a + /// tight loop over each function and its overloads, rather than by + /// planning and executing the equivalent join/aggregate/union SQL + /// against those two tables. + pub fn show_functions( + &self, + udfs: &HashMap>, + udafs: &HashMap>, + udwfs: &HashMap>, + filter: Option<&str>, + ) -> Result> { + let mut builder = ShowFunctionsBuilder::new(); + + for (name, udf) in udfs { + for (arg_types, return_type) in get_udf_args_and_return_types(udf)? { + builder.add_overload( + name, + &arg_types, + return_type.as_deref(), + udf.documentation(), + "SCALAR", + ); + } + } + for (name, udaf) in udafs { + for (arg_types, return_type) in get_udaf_args_and_return_types(udaf)? { + builder.add_overload( + name, + &arg_types, + return_type.as_deref(), + udaf.documentation(), + "AGGREGATE", + ); + } + } + for (name, udwf) in udwfs { + for (arg_types, return_type) in get_udwf_args_and_return_types(udwf)? { + builder.add_overload( + name, + &arg_types, + return_type.as_deref(), + udwf.documentation(), + "WINDOW", + ); + } + } + // Table functions (UDTFs) have no scalar signature (no parameters, + // no return type beyond "TABLE") and no documentation, so they get + // their own, much simpler, row shape. + for name in self.config.table_functions.keys() { + builder.add_table_function(name); + } + + record_batch_to_exec(builder.finish(filter)?) + } +} + +/// Wraps a single [`RecordBatch`] (already matching the plan's output +/// schema) in a one-partition in-memory [`ExecutionPlan`]. +fn record_batch_to_exec(record_batch: RecordBatch) -> Result> { + let schema = record_batch.schema(); + let partitions = vec![vec![record_batch]]; + let mem_exec = MemorySourceConfig::try_new_exec(&partitions, schema, None)?; + Ok(mem_exec) +} + +/// Builds `SHOW FUNCTIONS [LIKE ]` rows: one per concrete overload +/// of each scalar/aggregate/window UDF, plus one per UDTF. +struct ShowFunctionsBuilder { + function_names: StringBuilder, + return_types: StringBuilder, + parameters: ListBuilder, + parameter_types: ListBuilder, + function_types: StringBuilder, + descriptions: StringBuilder, + syntax_examples: StringBuilder, +} + +impl ShowFunctionsBuilder { + fn new() -> Self { + Self { + function_names: StringBuilder::new(), + return_types: StringBuilder::new(), + parameters: ListBuilder::new(StringBuilder::new()), + parameter_types: ListBuilder::new(StringBuilder::new()), + function_types: StringBuilder::new(), + descriptions: StringBuilder::new(), + syntax_examples: StringBuilder::new(), + } + } + + /// Append one row for a concrete scalar/aggregate/window overload. + /// `arg_types` is this overload's full, ordered argument type list; + /// argument *names* (when documented) are matched to them by position. + fn add_overload( + &mut self, + name: &str, + arg_types: &[String], + return_type: Option<&str>, + documentation: Option<&Documentation>, + function_type: &str, + ) { + self.function_names.append_value(name); + self.return_types.append_option(return_type); + + let arg_names = documentation.and_then(|d| d.arguments.as_ref()); + for (position, type_name) in arg_types.iter().enumerate() { + let param_name = arg_names + .and_then(|args| args.get(position)) + .map(|(arg_name, _)| arg_name.as_str()); + self.parameters.values().append_option(param_name); + self.parameter_types.values().append_value(type_name); + } + self.parameters.append(true); + self.parameter_types.append(true); + + self.function_types.append_value(function_type); + self.descriptions + .append_option(documentation.map(|d| d.description.as_str())); + self.syntax_examples + .append_option(documentation.map(|d| d.syntax_example.as_str())); + } + + /// Append one row for a table function (UDTF): no parameter or + /// documentation info is available for these. + fn add_table_function(&mut self, name: &str) { + self.function_names.append_value(name); + self.return_types.append_value("TABLE"); + self.parameters.append(false); + self.parameter_types.append(false); + self.function_types.append_value("TABLE"); + self.descriptions.append_null(); + self.syntax_examples.append_null(); + } + + /// Finalize into a [`RecordBatch`], applying the optional `LIKE` + /// `filter` to `function_name` (matching `col("function_name") + /// .like(lit(filter))`'s semantics). + fn finish(mut self, filter: Option<&str>) -> Result { + let batch = RecordBatch::try_new( + SchemaRef::from(show_functions_schema()), + vec![ + Arc::new(self.function_names.finish()), + Arc::new(self.return_types.finish()), + Arc::new(self.parameters.finish()), + Arc::new(self.parameter_types.finish()), + Arc::new(self.function_types.finish()), + Arc::new(self.descriptions.finish()), + Arc::new(self.syntax_examples.finish()), + ], + )?; + match filter { + Some(pattern) => { + let mask = like(batch.column(0), &StringArray::new_scalar(pattern))?; + Ok(filter_record_batch(&batch, &mask)?) + } + None => Ok(batch), + } + } } #[derive(Clone, Debug)] struct InformationSchemaConfig { system_catalog_name: Option, + information_schema: bool, catalog_list: Arc, table_functions: HashMap>, } @@ -190,19 +502,22 @@ impl InformationSchemaConfig { } } } - - // Add a final list for the information schema tables themselves - for table_name in INFORMATION_SCHEMA_TABLES { - builder.add_table( - &catalog_name, - INFORMATION_SCHEMA, - table_name, - TableType::View, - ); + if self.information_schema { + // Add a final list for the information schema tables themselves + for table_name in INFORMATION_SCHEMA_TABLES { + builder.add_table( + &catalog_name, + INFORMATION_SCHEMA, + table_name, + TableType::View, + ); + } } } - if let Some(system_catalog_name) = &self.system_catalog_name { + if self.information_schema + && let Some(system_catalog_name) = &self.system_catalog_name + { for table_name in INFORMATION_SCHEMA_TABLES { builder.add_table( system_catalog_name, @@ -264,7 +579,9 @@ impl InformationSchemaConfig { schema: &dyn SchemaProvider, ) -> Result<(), DataFusionError> { for table_name in schema.table_names() { - if let Some(table) = schema.table(&table_name).await? { + if let Some(table) = schema.table(&table_name).await? + && table.table_type() == TableType::View + { builder.add_view( catalog_name, schema_name, @@ -326,18 +643,28 @@ impl InformationSchemaConfig { Ok(()) } - /// Construct the `information_schema.df_settings` virtual table + /// Construct the `information_schema.df_settings` virtual table. + /// + /// `name` restricts the output to just that setting (`SHOW `); + /// when `None` (`SHOW ALL` / the real `information_schema.df_settings` + /// table), every setting is included, ordered by name for a consistent, + /// deterministic result. fn make_df_settings( &self, config_options: &ConfigOptions, runtime_env: &Arc, + name: Option<&str>, builder: &mut InformationSchemaDfSettingsBuilder, ) { - for entry in config_options.entries() { - builder.add_setting(entry); + let mut entries = config_options.entries(); + entries.extend(runtime_env.config_entries()); + + match name { + Some(name) => entries.retain(|entry| entry.key == name), + None => entries.sort_unstable_by(|a, b| a.key.cmp(&b.key)), } - // Add runtime configuration entries - for entry in runtime_env.config_entries() { + + for entry in entries { builder.add_setting(entry); } } @@ -723,23 +1050,9 @@ struct InformationSchemaTables { impl InformationSchemaTables { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("table_catalog", DataType::Utf8, false), - Field::new("table_schema", DataType::Utf8, false), - Field::new("table_name", DataType::Utf8, false), - Field::new("table_type", DataType::Utf8, false), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaTablesBuilder { - InformationSchemaTablesBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - table_types: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + schema: Arc::new(tables_schema()), + config, } } } @@ -750,7 +1063,7 @@ impl PartitionStream for InformationSchemaTables { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemaTablesBuilder::new(); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), @@ -763,50 +1076,6 @@ impl PartitionStream for InformationSchemaTables { } } -/// Builds the `information_schema.TABLE` table row by row -/// -/// Columns are based on -struct InformationSchemaTablesBuilder { - schema: SchemaRef, - catalog_names: StringBuilder, - schema_names: StringBuilder, - table_names: StringBuilder, - table_types: StringBuilder, -} - -impl InformationSchemaTablesBuilder { - fn add_table( - &mut self, - catalog_name: impl AsRef, - schema_name: impl AsRef, - table_name: impl AsRef, - table_type: TableType, - ) { - // Note: append_value is actually infallible. - self.catalog_names.append_value(catalog_name.as_ref()); - self.schema_names.append_value(schema_name.as_ref()); - self.table_names.append_value(table_name.as_ref()); - self.table_types.append_value(match table_type { - TableType::Base => "BASE TABLE", - TableType::View => "VIEW", - TableType::Temporary => "LOCAL TEMPORARY", - }); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_names.finish()), - Arc::new(self.schema_names.finish()), - Arc::new(self.table_names.finish()), - Arc::new(self.table_types.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemaViews { schema: SchemaRef, @@ -815,23 +1084,9 @@ struct InformationSchemaViews { impl InformationSchemaViews { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("table_catalog", DataType::Utf8, false), - Field::new("table_schema", DataType::Utf8, false), - Field::new("table_name", DataType::Utf8, false), - Field::new("definition", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaViewBuilder { - InformationSchemaViewBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - definitions: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + schema: Arc::new(views_schema()), + config, } } } @@ -842,7 +1097,7 @@ impl PartitionStream for InformationSchemaViews { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemaViewBuilder::new(); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), @@ -855,46 +1110,6 @@ impl PartitionStream for InformationSchemaViews { } } -/// Builds the `information_schema.VIEWS` table row by row -/// -/// Columns are based on -struct InformationSchemaViewBuilder { - schema: SchemaRef, - catalog_names: StringBuilder, - schema_names: StringBuilder, - table_names: StringBuilder, - definitions: StringBuilder, -} - -impl InformationSchemaViewBuilder { - fn add_view( - &mut self, - catalog_name: impl AsRef, - schema_name: impl AsRef, - table_name: impl AsRef, - definition: Option<&(impl AsRef + ?Sized)>, - ) { - // Note: append_value is actually infallible. - self.catalog_names.append_value(catalog_name.as_ref()); - self.schema_names.append_value(schema_name.as_ref()); - self.table_names.append_value(table_name.as_ref()); - self.definitions.append_option(definition.as_ref()); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_names.finish()), - Arc::new(self.schema_names.finish()), - Arc::new(self.table_names.finish()), - Arc::new(self.definitions.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemaColumns { schema: SchemaRef, @@ -903,50 +1118,12 @@ struct InformationSchemaColumns { impl InformationSchemaColumns { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("table_catalog", DataType::Utf8, false), - Field::new("table_schema", DataType::Utf8, false), - Field::new("table_name", DataType::Utf8, false), - Field::new("column_name", DataType::Utf8, false), - Field::new("ordinal_position", DataType::UInt64, false), - Field::new("column_default", DataType::Utf8, true), - Field::new("is_nullable", DataType::Utf8, false), - Field::new("data_type", DataType::Utf8, false), - Field::new("character_maximum_length", DataType::UInt64, true), - Field::new("character_octet_length", DataType::UInt64, true), - Field::new("numeric_precision", DataType::UInt64, true), - Field::new("numeric_precision_radix", DataType::UInt64, true), - Field::new("numeric_scale", DataType::UInt64, true), - Field::new("datetime_precision", DataType::UInt64, true), - Field::new("interval_type", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaColumnsBuilder { - // StringBuilder requires providing an initial capacity, so - // pick 10 here arbitrarily as this is not performance - // critical code and the number of tables is unavailable here. - let default_capacity = 10; - - InformationSchemaColumnsBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - column_names: StringBuilder::new(), - ordinal_positions: UInt64Builder::with_capacity(default_capacity), - column_defaults: StringBuilder::new(), - is_nullables: StringBuilder::new(), - data_types: StringBuilder::new(), - character_maximum_lengths: UInt64Builder::with_capacity(default_capacity), - character_octet_lengths: UInt64Builder::with_capacity(default_capacity), - numeric_precisions: UInt64Builder::with_capacity(default_capacity), - numeric_precision_radixes: UInt64Builder::with_capacity(default_capacity), - numeric_scales: UInt64Builder::with_capacity(default_capacity), - datetime_precisions: UInt64Builder::with_capacity(default_capacity), - interval_types: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + // The real `information_schema.columns` table is always the + // full column set; `Basic`/`Describe` are projections specific + // to the `SHOW COLUMNS` / `DESCRIBE` SQL surface, not this table. + schema: Arc::new(columns_schema(ColumnsDetail::Full)), + config, } } } @@ -957,7 +1134,7 @@ impl PartitionStream for InformationSchemaColumns { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Full); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), @@ -970,281 +1147,19 @@ impl PartitionStream for InformationSchemaColumns { } } -/// Builds the `information_schema.COLUMNS` table row by row -/// -/// Columns are based on -struct InformationSchemaColumnsBuilder { - schema: SchemaRef, - catalog_names: StringBuilder, - schema_names: StringBuilder, - table_names: StringBuilder, - column_names: StringBuilder, - ordinal_positions: UInt64Builder, - column_defaults: StringBuilder, - is_nullables: StringBuilder, - data_types: StringBuilder, - character_maximum_lengths: UInt64Builder, - character_octet_lengths: UInt64Builder, - numeric_precisions: UInt64Builder, - numeric_precision_radixes: UInt64Builder, - numeric_scales: UInt64Builder, - datetime_precisions: UInt64Builder, - interval_types: StringBuilder, -} - -impl InformationSchemaColumnsBuilder { - fn add_column( - &mut self, - catalog_name: &str, - schema_name: &str, - table_name: &str, - field_position: usize, - field: &Field, - ) { - use DataType::*; - - // Note: append_value is actually infallible. - self.catalog_names.append_value(catalog_name); - self.schema_names.append_value(schema_name); - self.table_names.append_value(table_name); - - self.column_names.append_value(field.name()); - - self.ordinal_positions.append_value(field_position as u64); - - // DataFusion does not support column default values, so null - self.column_defaults.append_null(); - - // "YES if the column is possibly nullable, NO if it is known not nullable. " - let nullable_str = if field.is_nullable() { "YES" } else { "NO" }; - self.is_nullables.append_value(nullable_str); - - // "System supplied type" --> Use debug format of the datatype - self.data_types.append_value(field.data_type().to_string()); - - // "If data_type identifies a character or bit string type, the - // declared maximum length; null for all other data types or - // if no maximum length was declared." - // - // Arrow has no equivalent of VARCHAR(20), so we leave this as Null - let max_chars = None; - self.character_maximum_lengths.append_option(max_chars); - - // "Maximum length, in bytes, for binary data, character data, - // or text and image data." - let char_len: Option = match field.data_type() { - Utf8 | Binary => Some(i32::MAX as u64), - LargeBinary | LargeUtf8 => Some(i64::MAX as u64), - _ => None, - }; - self.character_octet_lengths.append_option(char_len); - - // numeric_precision: "If data_type identifies a numeric type, this column - // contains the (declared or implicit) precision of the type - // for this column. The precision indicates the number of - // significant digits. It can be expressed in decimal (base - // 10) or binary (base 2) terms, as specified in the column - // numeric_precision_radix. For all other data types, this - // column is null." - // - // numeric_radix: If data_type identifies a numeric type, this - // column indicates in which base the values in the columns - // numeric_precision and numeric_scale are expressed. The - // value is either 2 or 10. For all other data types, this - // column is null. - // - // numeric_scale: If data_type identifies an exact numeric - // type, this column contains the (declared or implicit) scale - // of the type for this column. The scale indicates the number - // of significant digits to the right of the decimal point. It - // can be expressed in decimal (base 10) or binary (base 2) - // terms, as specified in the column - // numeric_precision_radix. For all other data types, this - // column is null. - let (numeric_precision, numeric_radix, numeric_scale) = match field.data_type() { - Int8 | UInt8 => (Some(8), Some(2), None), - Int16 | UInt16 => (Some(16), Some(2), None), - Int32 | UInt32 => (Some(32), Some(2), None), - // From max value of 65504 as explained on - // https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Exponent_encoding - Float16 => (Some(15), Some(2), None), - // Numbers from postgres `real` type - Float32 => (Some(24), Some(2), None), - // Numbers from postgres `double` type - Float64 => (Some(24), Some(2), None), - Decimal128(precision, scale) => { - (Some(*precision as u64), Some(10), Some(*scale as u64)) - } - _ => (None, None, None), - }; - - self.numeric_precisions.append_option(numeric_precision); - self.numeric_precision_radixes.append_option(numeric_radix); - self.numeric_scales.append_option(numeric_scale); - - self.datetime_precisions.append_option(None); - self.interval_types.append_null(); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_names.finish()), - Arc::new(self.schema_names.finish()), - Arc::new(self.table_names.finish()), - Arc::new(self.column_names.finish()), - Arc::new(self.ordinal_positions.finish()), - Arc::new(self.column_defaults.finish()), - Arc::new(self.is_nullables.finish()), - Arc::new(self.data_types.finish()), - Arc::new(self.character_maximum_lengths.finish()), - Arc::new(self.character_octet_lengths.finish()), - Arc::new(self.numeric_precisions.finish()), - Arc::new(self.numeric_precision_radixes.finish()), - Arc::new(self.numeric_scales.finish()), - Arc::new(self.datetime_precisions.finish()), - Arc::new(self.interval_types.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemata { schema: SchemaRef, config: InformationSchemaConfig, } -/// The Arrow schema of [`information_schema.schemata`] rows. -/// -/// Useful for downstream catalog implementations that want to declare a -/// `TableProvider` for `schemata` before populating any rows via -/// [`InformationSchemataBuilder`]. -/// -/// Columns and nullability match -/// . -/// -/// [`information_schema.schemata`]: https://www.postgresql.org/docs/current/infoschema-schemata.html -pub fn schemata_schema() -> SchemaRef { - Arc::new(Schema::new(vec![ - Field::new("catalog_name", DataType::Utf8, false), - Field::new("schema_name", DataType::Utf8, false), - Field::new("schema_owner", DataType::Utf8, true), - Field::new("default_character_set_catalog", DataType::Utf8, true), - Field::new("default_character_set_schema", DataType::Utf8, true), - Field::new("default_character_set_name", DataType::Utf8, true), - Field::new("sql_path", DataType::Utf8, true), - ])) -} - impl InformationSchemata { fn new(config: InformationSchemaConfig) -> Self { Self { - schema: schemata_schema(), + schema: Arc::new(schemata_schema()), config, } } - - fn builder(&self) -> InformationSchemataBuilder { - InformationSchemataBuilder { - schema: Arc::clone(&self.schema), - catalog_name: StringBuilder::new(), - schema_name: StringBuilder::new(), - schema_owner: StringBuilder::new(), - default_character_set_catalog: StringBuilder::new(), - default_character_set_schema: StringBuilder::new(), - default_character_set_name: StringBuilder::new(), - sql_path: StringBuilder::new(), - } - } -} - -/// Builder that produces [`RecordBatch`] values matching the schema of -/// `information_schema.schemata` (see [`schemata_schema`]). -/// -/// Intended for downstream catalog implementations that need to emit -/// `schemata` rows from their own metadata source rather than going -/// through DataFusion's `InformationSchemaProvider`, which enumerates -/// schemas synchronously via `CatalogProviderList` and so is unsuitable -/// for catalog backends that resolve asynchronously. -#[derive(Debug)] -pub struct InformationSchemataBuilder { - schema: SchemaRef, - catalog_name: StringBuilder, - schema_name: StringBuilder, - schema_owner: StringBuilder, - default_character_set_catalog: StringBuilder, - default_character_set_schema: StringBuilder, - default_character_set_name: StringBuilder, - sql_path: StringBuilder, -} - -impl Default for InformationSchemataBuilder { - fn default() -> Self { - Self::new() - } -} - -impl InformationSchemataBuilder { - /// Construct an empty builder. - pub fn new() -> Self { - Self { - schema: schemata_schema(), - catalog_name: StringBuilder::new(), - schema_name: StringBuilder::new(), - schema_owner: StringBuilder::new(), - default_character_set_catalog: StringBuilder::new(), - default_character_set_schema: StringBuilder::new(), - default_character_set_name: StringBuilder::new(), - sql_path: StringBuilder::new(), - } - } - - /// Append one row to the builder. `schema_owner` is the optional SQL - /// schema owner; the three `default_character_set_*` columns and - /// `sql_path` are written as null (DataFusion does not model those - /// concepts; see the PostgreSQL docs link on [`schemata_schema`]). - pub fn add_schemata( - &mut self, - catalog_name: &str, - schema_name: &str, - schema_owner: Option<&str>, - ) { - self.catalog_name.append_value(catalog_name); - self.schema_name.append_value(schema_name); - match schema_owner { - Some(owner) => self.schema_owner.append_value(owner), - None => self.schema_owner.append_null(), - } - self.default_character_set_catalog.append_null(); - self.default_character_set_schema.append_null(); - self.default_character_set_name.append_null(); - self.sql_path.append_null(); - } - - /// Finalize the builder into a [`RecordBatch`]. - /// - /// Returns an error only if Arrow buffer construction fails, which - /// the builder's column-count and type invariants make unreachable - /// under normal use. The `Result` return type preserves room to add - /// validation in the future without a breaking API change. - pub fn finish(&mut self) -> Result { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.catalog_name.finish()), - Arc::new(self.schema_name.finish()), - Arc::new(self.schema_owner.finish()), - Arc::new(self.default_character_set_catalog.finish()), - Arc::new(self.default_character_set_schema.finish()), - Arc::new(self.default_character_set_name.finish()), - Arc::new(self.sql_path.finish()), - ], - ) - .map_err(DataFusionError::from) - } } impl PartitionStream for InformationSchemata { @@ -1253,7 +1168,7 @@ impl PartitionStream for InformationSchemata { } fn execute(&self, _ctx: Arc) -> SendableRecordBatchStream { - let mut builder = self.builder(); + let mut builder = InformationSchemataBuilder::new(); let config = self.config.clone(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), @@ -1274,21 +1189,9 @@ struct InformationSchemaDfSettings { impl InformationSchemaDfSettings { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("name", DataType::Utf8, false), - Field::new("value", DataType::Utf8, true), - Field::new("description", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaDfSettingsBuilder { - InformationSchemaDfSettingsBuilder { - names: StringBuilder::new(), - values: StringBuilder::new(), - descriptions: StringBuilder::new(), - schema: Arc::clone(&self.schema), + Self { + schema: Arc::new(df_settings_schema(true)), + config, } } } @@ -1300,7 +1203,7 @@ impl PartitionStream for InformationSchemaDfSettings { fn execute(&self, ctx: Arc) -> SendableRecordBatchStream { let config = self.config.clone(); - let mut builder = self.builder(); + let mut builder = InformationSchemaDfSettingsBuilder::new(true); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), // TODO: Stream this @@ -1310,6 +1213,7 @@ impl PartitionStream for InformationSchemaDfSettings { config.make_df_settings( ctx.session_config().options(), &runtime_env, + None, &mut builder, ); Ok(builder.finish()) @@ -1318,33 +1222,6 @@ impl PartitionStream for InformationSchemaDfSettings { } } -struct InformationSchemaDfSettingsBuilder { - schema: SchemaRef, - names: StringBuilder, - values: StringBuilder, - descriptions: StringBuilder, -} - -impl InformationSchemaDfSettingsBuilder { - fn add_setting(&mut self, entry: ConfigEntry) { - self.names.append_value(entry.key); - self.values.append_option(entry.value); - self.descriptions.append_value(entry.description); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.names.finish()), - Arc::new(self.values.finish()), - Arc::new(self.descriptions.finish()), - ], - ) - .unwrap() - } -} - #[derive(Debug)] struct InformationSchemaRoutines { schema: SchemaRef, @@ -1353,109 +1230,13 @@ struct InformationSchemaRoutines { impl InformationSchemaRoutines { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("specific_catalog", DataType::Utf8, false), - Field::new("specific_schema", DataType::Utf8, false), - Field::new("specific_name", DataType::Utf8, false), - Field::new("routine_catalog", DataType::Utf8, false), - Field::new("routine_schema", DataType::Utf8, false), - Field::new("routine_name", DataType::Utf8, false), - Field::new("routine_type", DataType::Utf8, false), - Field::new("is_deterministic", DataType::Boolean, true), - Field::new("data_type", DataType::Utf8, true), - Field::new("function_type", DataType::Utf8, true), - Field::new("description", DataType::Utf8, true), - Field::new("syntax_example", DataType::Utf8, true), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaRoutinesBuilder { - InformationSchemaRoutinesBuilder { - schema: Arc::clone(&self.schema), - specific_catalog: StringBuilder::new(), - specific_schema: StringBuilder::new(), - specific_name: StringBuilder::new(), - routine_catalog: StringBuilder::new(), - routine_schema: StringBuilder::new(), - routine_name: StringBuilder::new(), - routine_type: StringBuilder::new(), - is_deterministic: BooleanBuilder::new(), - data_type: StringBuilder::new(), - function_type: StringBuilder::new(), - description: StringBuilder::new(), - syntax_example: StringBuilder::new(), + Self { + schema: Arc::new(routines_schema()), + config, } } } -struct InformationSchemaRoutinesBuilder { - schema: SchemaRef, - specific_catalog: StringBuilder, - specific_schema: StringBuilder, - specific_name: StringBuilder, - routine_catalog: StringBuilder, - routine_schema: StringBuilder, - routine_name: StringBuilder, - routine_type: StringBuilder, - is_deterministic: BooleanBuilder, - data_type: StringBuilder, - function_type: StringBuilder, - description: StringBuilder, - syntax_example: StringBuilder, -} - -impl InformationSchemaRoutinesBuilder { - #[expect(clippy::too_many_arguments)] - fn add_routine( - &mut self, - catalog_name: impl AsRef, - schema_name: impl AsRef, - routine_name: impl AsRef, - routine_type: impl AsRef, - is_deterministic: bool, - data_type: Option<&impl AsRef>, - function_type: impl AsRef, - description: Option>, - syntax_example: Option>, - ) { - self.specific_catalog.append_value(catalog_name.as_ref()); - self.specific_schema.append_value(schema_name.as_ref()); - self.specific_name.append_value(routine_name.as_ref()); - self.routine_catalog.append_value(catalog_name.as_ref()); - self.routine_schema.append_value(schema_name.as_ref()); - self.routine_name.append_value(routine_name.as_ref()); - self.routine_type.append_value(routine_type.as_ref()); - self.is_deterministic.append_value(is_deterministic); - self.data_type.append_option(data_type.as_ref()); - self.function_type.append_value(function_type.as_ref()); - self.description.append_option(description); - self.syntax_example.append_option(syntax_example); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.specific_catalog.finish()), - Arc::new(self.specific_schema.finish()), - Arc::new(self.specific_name.finish()), - Arc::new(self.routine_catalog.finish()), - Arc::new(self.routine_schema.finish()), - Arc::new(self.routine_name.finish()), - Arc::new(self.routine_type.finish()), - Arc::new(self.is_deterministic.finish()), - Arc::new(self.data_type.finish()), - Arc::new(self.function_type.finish()), - Arc::new(self.description.finish()), - Arc::new(self.syntax_example.finish()), - ], - ) - .unwrap() - } -} - impl PartitionStream for InformationSchemaRoutines { fn schema(&self) -> &SchemaRef { &self.schema @@ -1463,7 +1244,7 @@ impl PartitionStream for InformationSchemaRoutines { fn execute(&self, ctx: Arc) -> SendableRecordBatchStream { let config = self.config.clone(); - let mut builder = self.builder(); + let mut builder = InformationSchemaRoutinesBuilder::new(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), futures::stream::once(async move { @@ -1488,106 +1269,13 @@ struct InformationSchemaParameters { impl InformationSchemaParameters { fn new(config: InformationSchemaConfig) -> Self { - let schema = Arc::new(Schema::new(vec![ - Field::new("specific_catalog", DataType::Utf8, false), - Field::new("specific_schema", DataType::Utf8, false), - Field::new("specific_name", DataType::Utf8, false), - Field::new("ordinal_position", DataType::UInt64, false), - Field::new("parameter_mode", DataType::Utf8, false), - Field::new("parameter_name", DataType::Utf8, true), - Field::new("data_type", DataType::Utf8, false), - Field::new("parameter_default", DataType::Utf8, true), - Field::new("is_variadic", DataType::Boolean, false), - // `rid` (short for `routine id`) is used to differentiate parameters from different signatures - // (It serves as the group-by key when generating the `SHOW FUNCTIONS` query). - // For example, the following signatures have different `rid` values: - // - `datetrunc(Utf8, Timestamp(Microsecond, Some("+TZ"))) -> Timestamp(Microsecond, Some("+TZ"))` - // - `datetrunc(Utf8View, Timestamp(Nanosecond, None)) -> Timestamp(Nanosecond, None)` - Field::new("rid", DataType::UInt8, false), - ])); - - Self { schema, config } - } - - fn builder(&self) -> InformationSchemaParametersBuilder { - InformationSchemaParametersBuilder { - schema: Arc::clone(&self.schema), - specific_catalog: StringBuilder::new(), - specific_schema: StringBuilder::new(), - specific_name: StringBuilder::new(), - ordinal_position: UInt64Builder::new(), - parameter_mode: StringBuilder::new(), - parameter_name: StringBuilder::new(), - data_type: StringBuilder::new(), - parameter_default: StringBuilder::new(), - is_variadic: BooleanBuilder::new(), - rid: UInt8Builder::new(), + Self { + schema: Arc::new(parameters_schema()), + config, } } } -struct InformationSchemaParametersBuilder { - schema: SchemaRef, - specific_catalog: StringBuilder, - specific_schema: StringBuilder, - specific_name: StringBuilder, - ordinal_position: UInt64Builder, - parameter_mode: StringBuilder, - parameter_name: StringBuilder, - data_type: StringBuilder, - parameter_default: StringBuilder, - is_variadic: BooleanBuilder, - rid: UInt8Builder, -} - -impl InformationSchemaParametersBuilder { - #[expect(clippy::too_many_arguments)] - fn add_parameter( - &mut self, - specific_catalog: impl AsRef, - specific_schema: impl AsRef, - specific_name: impl AsRef, - ordinal_position: u64, - parameter_mode: impl AsRef, - parameter_name: Option<&(impl AsRef + ?Sized)>, - data_type: impl AsRef, - parameter_default: Option>, - is_variadic: bool, - rid: u8, - ) { - self.specific_catalog - .append_value(specific_catalog.as_ref()); - self.specific_schema.append_value(specific_schema.as_ref()); - self.specific_name.append_value(specific_name.as_ref()); - self.ordinal_position.append_value(ordinal_position); - self.parameter_mode.append_value(parameter_mode.as_ref()); - self.parameter_name.append_option(parameter_name.as_ref()); - self.data_type.append_value(data_type.as_ref()); - self.parameter_default.append_option(parameter_default); - self.is_variadic.append_value(is_variadic); - self.rid.append_value(rid); - } - - fn finish(&mut self) -> RecordBatch { - RecordBatch::try_new( - Arc::clone(&self.schema), - vec![ - Arc::new(self.specific_catalog.finish()), - Arc::new(self.specific_schema.finish()), - Arc::new(self.specific_name.finish()), - Arc::new(self.ordinal_position.finish()), - Arc::new(self.parameter_mode.finish()), - Arc::new(self.parameter_name.finish()), - Arc::new(self.data_type.finish()), - Arc::new(self.parameter_default.finish()), - Arc::new(self.is_variadic.finish()), - Arc::new(self.rid.finish()), - ], - ) - .unwrap() - } -} - impl PartitionStream for InformationSchemaParameters { fn schema(&self) -> &SchemaRef { &self.schema @@ -1595,7 +1283,7 @@ impl PartitionStream for InformationSchemaParameters { fn execute(&self, ctx: Arc) -> SendableRecordBatchStream { let config = self.config.clone(); - let mut builder = self.builder(); + let mut builder = InformationSchemaParametersBuilder::new(); Box::pin(RecordBatchStreamAdapter::new( Arc::clone(&self.schema), futures::stream::once(async move { @@ -1621,74 +1309,26 @@ mod tests { use datafusion_common::ScalarValue; use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl}; - #[test] - fn schemata_builder_emits_canonical_schema_and_rows() { - // Construct via `Default` so the test exercises both `new()` (via - // the `Default` impl) and the public column-layout contract. - let mut builder = InformationSchemataBuilder::default(); - builder.add_schemata("cat", "schema_one", Some("alice")); - builder.add_schemata("cat", "schema_two", None); - let batch = builder.finish().expect("finish should not fail"); - - assert_eq!(batch.schema(), schemata_schema()); - assert_eq!(batch.num_rows(), 2); - - let col = |name: &str| { - batch - .column_by_name(name) - .unwrap_or_else(|| panic!("missing column {name}")) - }; - let string_col = |name: &str| { - col(name) - .as_any() - .downcast_ref::() - .unwrap_or_else(|| panic!("{name} should be a StringArray")) - }; - - let catalog = string_col("catalog_name"); - assert_eq!(catalog.value(0), "cat"); - assert_eq!(catalog.value(1), "cat"); - - let schema = string_col("schema_name"); - assert_eq!(schema.value(0), "schema_one"); - assert_eq!(schema.value(1), "schema_two"); - - let owner = string_col("schema_owner"); - assert_eq!(owner.value(0), "alice"); - assert!(owner.is_null(1)); - - // The three character-set columns and sql_path are unconditionally - // null — they exist for SQL-standard column-layout compatibility. - for name in [ - "default_character_set_catalog", - "default_character_set_schema", - "default_character_set_name", - "sql_path", - ] { - let c = string_col(name); - assert!(c.is_null(0), "{name} row 0 should be null"); - assert!(c.is_null(1), "{name} row 1 should be null"); - } - } - #[tokio::test] async fn make_tables_uses_table_type() { let config = InformationSchemaConfig { system_catalog_name: None, + information_schema: true, catalog_list: Arc::new(Fixture), table_functions: HashMap::new(), }; - let mut builder = InformationSchemaTablesBuilder { - catalog_names: StringBuilder::new(), - schema_names: StringBuilder::new(), - table_names: StringBuilder::new(), - table_types: StringBuilder::new(), - schema: Arc::new(Schema::empty()), - }; + let mut builder = InformationSchemaTablesBuilder::new(); assert!(config.make_tables(&mut builder).await.is_ok()); - assert_eq!("BASE TABLE", builder.table_types.finish().value(0)); + let batch = builder.finish(); + let table_types = batch + .column_by_name("table_type") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!("BASE TABLE", table_types.value(0)); } // UDF diff --git a/datafusion/common/src/information_schema.rs b/datafusion/common/src/information_schema.rs new file mode 100644 index 0000000000000..d2820032cf3e2 --- /dev/null +++ b/datafusion/common/src/information_schema.rs @@ -0,0 +1,1066 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Schema and [`RecordBatch`] builders for the `information_schema` virtual +//! tables. See `datafusion_catalog::information_schema` for the session +//! integration. + +use crate::Result; +use crate::TableType; +use crate::config::ConfigEntry; +use arrow::array::ArrayRef; +use arrow::array::builder::{BooleanBuilder, StringBuilder, UInt8Builder, UInt64Builder}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatch; +use std::sync::Arc; + +/// Returns the Arrow schema of `information_schema.tables` rows. +/// +/// Columns are based on . +pub fn tables_schema() -> Schema { + Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("table_type", DataType::Utf8, false), + ]) +} + +/// Builds `information_schema.tables` rows. +#[derive(Debug)] +pub struct InformationSchemaTablesBuilder { + schema: SchemaRef, + catalog_names: StringBuilder, + schema_names: StringBuilder, + table_names: StringBuilder, + table_types: StringBuilder, +} + +impl Default for InformationSchemaTablesBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaTablesBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(tables_schema()), + catalog_names: StringBuilder::new(), + schema_names: StringBuilder::new(), + table_names: StringBuilder::new(), + table_types: StringBuilder::new(), + } + } + + /// Append one row. + pub fn add_table( + &mut self, + catalog_name: impl AsRef, + schema_name: impl AsRef, + table_name: impl AsRef, + table_type: TableType, + ) { + // Note: append_value is actually infallible. + self.catalog_names.append_value(catalog_name.as_ref()); + self.schema_names.append_value(schema_name.as_ref()); + self.table_names.append_value(table_name.as_ref()); + self.table_types.append_value(match table_type { + TableType::Base => "BASE TABLE", + TableType::View => "VIEW", + TableType::Temporary => "LOCAL TEMPORARY", + }); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.catalog_names.finish()), + Arc::new(self.schema_names.finish()), + Arc::new(self.table_names.finish()), + Arc::new(self.table_types.finish()), + ], + ) + .unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.views` rows. +/// +/// Columns are based on . +pub fn views_schema() -> Schema { + Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("definition", DataType::Utf8, true), + ]) +} + +/// Builds `information_schema.views` rows. +#[derive(Debug)] +pub struct InformationSchemaViewBuilder { + schema: SchemaRef, + catalog_names: StringBuilder, + schema_names: StringBuilder, + table_names: StringBuilder, + definitions: StringBuilder, +} + +impl Default for InformationSchemaViewBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaViewBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(views_schema()), + catalog_names: StringBuilder::new(), + schema_names: StringBuilder::new(), + table_names: StringBuilder::new(), + definitions: StringBuilder::new(), + } + } + + /// Append one row. `definition` is the view's `CREATE VIEW` (or + /// `CREATE EXTERNAL TABLE`) statement, if known. + pub fn add_view( + &mut self, + catalog_name: impl AsRef, + schema_name: impl AsRef, + table_name: impl AsRef, + definition: Option<&(impl AsRef + ?Sized)>, + ) { + // Note: append_value is actually infallible. + self.catalog_names.append_value(catalog_name.as_ref()); + self.schema_names.append_value(schema_name.as_ref()); + self.table_names.append_value(table_name.as_ref()); + self.definitions.append_option(definition.as_ref()); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.catalog_names.finish()), + Arc::new(self.schema_names.finish()), + Arc::new(self.table_names.finish()), + Arc::new(self.definitions.finish()), + ], + ) + .unwrap() + } +} + +/// How detailed `SHOW COLUMNS` / `DESCRIBE` output should be. +/// +/// All three levels are projections of the same underlying column metadata +/// (see [`InformationSchemaColumnsBuilder`]); which columns are actually kept +/// is decided entirely by this module, so neither `datafusion-expr` (which +/// types the `SHOW COLUMNS` / `DESCRIBE` logical plan) nor `datafusion-core` +/// (which executes it) need their own copy of that projection logic. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ColumnsDetail { + /// `DESCRIBE
` / `DESCRIBE `: `column_name`, `data_type`, + /// `is_nullable`. + Describe, + /// `SHOW COLUMNS FROM
` (no `FULL`/`EXTENDED`): adds + /// `table_catalog`, `table_schema`, `table_name`. + Basic, + /// `SHOW FULL COLUMNS` / `SHOW EXTENDED COLUMNS FROM
` (treated + /// the same), and the real `information_schema.columns` table itself: + /// every column. + Full, +} + +/// Returns the Arrow schema of `information_schema.columns` rows at the +/// given [`ColumnsDetail`] level. +/// +/// The `Full` columns are based on +/// . +pub fn columns_schema(detail: ColumnsDetail) -> Schema { + match detail { + ColumnsDetail::Describe => Schema::new(vec![ + Field::new("column_name", DataType::Utf8, false), + Field::new("data_type", DataType::Utf8, false), + Field::new("is_nullable", DataType::Utf8, false), + ]), + ColumnsDetail::Basic => Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("column_name", DataType::Utf8, false), + Field::new("data_type", DataType::Utf8, false), + Field::new("is_nullable", DataType::Utf8, false), + ]), + ColumnsDetail::Full => Schema::new(vec![ + Field::new("table_catalog", DataType::Utf8, false), + Field::new("table_schema", DataType::Utf8, false), + Field::new("table_name", DataType::Utf8, false), + Field::new("column_name", DataType::Utf8, false), + Field::new("ordinal_position", DataType::UInt64, false), + Field::new("column_default", DataType::Utf8, true), + Field::new("is_nullable", DataType::Utf8, false), + Field::new("data_type", DataType::Utf8, false), + Field::new("character_maximum_length", DataType::UInt64, true), + Field::new("character_octet_length", DataType::UInt64, true), + Field::new("numeric_precision", DataType::UInt64, true), + Field::new("numeric_precision_radix", DataType::UInt64, true), + Field::new("numeric_scale", DataType::UInt64, true), + Field::new("datetime_precision", DataType::UInt64, true), + Field::new("interval_type", DataType::Utf8, true), + ]), + } +} + +/// Builds `information_schema.columns` rows at a given [`ColumnsDetail`] +/// level. +/// +/// Callers always report every column via [`Self::add_column`] regardless of +/// `detail` (the same call works for `SHOW COLUMNS`, `SHOW FULL COLUMNS` and +/// `DESCRIBE` alike); the builder itself only retains the Arrow arrays its +/// `detail` level actually needs. +#[derive(Debug)] +pub struct InformationSchemaColumnsBuilder { + schema: SchemaRef, + detail: ColumnsDetail, + catalog_names: Option, + schema_names: Option, + table_names: Option, + column_names: StringBuilder, + ordinal_positions: Option, + column_defaults: Option, + is_nullables: StringBuilder, + data_types: StringBuilder, + character_maximum_lengths: Option, + character_octet_lengths: Option, + numeric_precisions: Option, + numeric_precision_radixes: Option, + numeric_scales: Option, + datetime_precisions: Option, + interval_types: Option, +} + +impl InformationSchemaColumnsBuilder { + /// Construct an empty builder that will only retain the columns needed + /// for `detail`. + pub fn new(detail: ColumnsDetail) -> Self { + // StringBuilder requires providing an initial capacity, so + // pick 10 here arbitrarily as this is not performance + // critical code and the number of tables is unavailable here. + let default_capacity = 10; + + let with_table_qualifiers = + matches!(detail, ColumnsDetail::Basic | ColumnsDetail::Full); + let full = detail == ColumnsDetail::Full; + + Self { + schema: Arc::new(columns_schema(detail)), + detail, + catalog_names: with_table_qualifiers.then(StringBuilder::new), + schema_names: with_table_qualifiers.then(StringBuilder::new), + table_names: with_table_qualifiers.then(StringBuilder::new), + column_names: StringBuilder::new(), + ordinal_positions: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + column_defaults: full.then(StringBuilder::new), + is_nullables: StringBuilder::new(), + data_types: StringBuilder::new(), + character_maximum_lengths: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + character_octet_lengths: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + numeric_precisions: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + numeric_precision_radixes: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + numeric_scales: full.then(|| UInt64Builder::with_capacity(default_capacity)), + datetime_precisions: full + .then(|| UInt64Builder::with_capacity(default_capacity)), + interval_types: full.then(StringBuilder::new), + } + } + + /// Append one row, derived from the given field's position and Arrow + /// type. Always pass the full information for the column; values that + /// aren't relevant at this builder's `detail` level are simply not kept. + pub fn add_column( + &mut self, + catalog_name: &str, + schema_name: &str, + table_name: &str, + field_position: usize, + field: &Field, + ) { + use DataType::*; + + // Note: append_value is actually infallible. + if let Some(b) = &mut self.catalog_names { + b.append_value(catalog_name); + } + if let Some(b) = &mut self.schema_names { + b.append_value(schema_name); + } + if let Some(b) = &mut self.table_names { + b.append_value(table_name); + } + + self.column_names.append_value(field.name()); + + if let Some(b) = &mut self.ordinal_positions { + b.append_value(field_position as u64); + } + + // DataFusion does not support column default values, so null + if let Some(b) = &mut self.column_defaults { + b.append_null(); + } + + // "YES if the column is possibly nullable, NO if it is known not nullable. " + let nullable_str = if field.is_nullable() { "YES" } else { "NO" }; + self.is_nullables.append_value(nullable_str); + + // "System supplied type" --> Use debug format of the datatype + self.data_types.append_value(field.data_type().to_string()); + + // "If data_type identifies a character or bit string type, the + // declared maximum length; null for all other data types or + // if no maximum length was declared." + // + // Arrow has no equivalent of VARCHAR(20), so we leave this as Null + if let Some(b) = &mut self.character_maximum_lengths { + b.append_option(None); + } + + // "Maximum length, in bytes, for binary data, character data, + // or text and image data." + if let Some(b) = &mut self.character_octet_lengths { + let char_len: Option = match field.data_type() { + Utf8 | Binary => Some(i32::MAX as u64), + LargeBinary | LargeUtf8 => Some(i64::MAX as u64), + _ => None, + }; + b.append_option(char_len); + } + + // numeric_precision: "If data_type identifies a numeric type, this column + // contains the (declared or implicit) precision of the type + // for this column. The precision indicates the number of + // significant digits. It can be expressed in decimal (base + // 10) or binary (base 2) terms, as specified in the column + // numeric_precision_radix. For all other data types, this + // column is null." + // + // numeric_radix: If data_type identifies a numeric type, this + // column indicates in which base the values in the columns + // numeric_precision and numeric_scale are expressed. The + // value is either 2 or 10. For all other data types, this + // column is null. + // + // numeric_scale: If data_type identifies an exact numeric + // type, this column contains the (declared or implicit) scale + // of the type for this column. The scale indicates the number + // of significant digits to the right of the decimal point. It + // can be expressed in decimal (base 10) or binary (base 2) + // terms, as specified in the column + // numeric_precision_radix. For all other data types, this + // column is null. + if let (Some(precisions), Some(radixes), Some(scales)) = ( + &mut self.numeric_precisions, + &mut self.numeric_precision_radixes, + &mut self.numeric_scales, + ) { + let (numeric_precision, numeric_radix, numeric_scale) = + match field.data_type() { + Int8 | UInt8 => (Some(8), Some(2), None), + Int16 | UInt16 => (Some(16), Some(2), None), + Int32 | UInt32 => (Some(32), Some(2), None), + // From max value of 65504 as explained on + // https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Exponent_encoding + Float16 => (Some(15), Some(2), None), + // Numbers from postgres `real` type + Float32 => (Some(24), Some(2), None), + // Numbers from postgres `double` type + Float64 => (Some(24), Some(2), None), + Decimal128(precision, scale) => { + (Some(*precision as u64), Some(10), Some(*scale as u64)) + } + _ => (None, None, None), + }; + + precisions.append_option(numeric_precision); + radixes.append_option(numeric_radix); + scales.append_option(numeric_scale); + } + + if let Some(b) = &mut self.datetime_precisions { + b.append_option(None); + } + if let Some(b) = &mut self.interval_types { + b.append_null(); + } + } + + /// Finalize the builder into a [`RecordBatch`], in the column order + /// matching `columns_schema(self.detail)`. + pub fn finish(&mut self) -> RecordBatch { + let mut columns: Vec = Vec::new(); + if self.detail != ColumnsDetail::Describe { + columns.push(Arc::new(self.catalog_names.as_mut().unwrap().finish())); + columns.push(Arc::new(self.schema_names.as_mut().unwrap().finish())); + columns.push(Arc::new(self.table_names.as_mut().unwrap().finish())); + } + columns.push(Arc::new(self.column_names.finish())); + if self.detail == ColumnsDetail::Full { + columns.push(Arc::new(self.ordinal_positions.as_mut().unwrap().finish())); + columns.push(Arc::new(self.column_defaults.as_mut().unwrap().finish())); + } + if self.detail == ColumnsDetail::Full { + columns.push(Arc::new(self.is_nullables.finish())); + columns.push(Arc::new(self.data_types.finish())); + } else { + // `Describe`/`Basic` list `data_type` before `is_nullable`, + // unlike the SQL-standard `information_schema.columns` order + // used by `Full`. + columns.push(Arc::new(self.data_types.finish())); + columns.push(Arc::new(self.is_nullables.finish())); + } + if self.detail == ColumnsDetail::Full { + columns.push(Arc::new( + self.character_maximum_lengths.as_mut().unwrap().finish(), + )); + columns.push(Arc::new( + self.character_octet_lengths.as_mut().unwrap().finish(), + )); + columns.push(Arc::new(self.numeric_precisions.as_mut().unwrap().finish())); + columns.push(Arc::new( + self.numeric_precision_radixes.as_mut().unwrap().finish(), + )); + columns.push(Arc::new(self.numeric_scales.as_mut().unwrap().finish())); + columns.push(Arc::new( + self.datetime_precisions.as_mut().unwrap().finish(), + )); + columns.push(Arc::new(self.interval_types.as_mut().unwrap().finish())); + } + RecordBatch::try_new(Arc::clone(&self.schema), columns).unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.schemata` rows. +/// +/// Columns and nullability match +/// . +pub fn schemata_schema() -> Schema { + Schema::new(vec![ + Field::new("catalog_name", DataType::Utf8, false), + Field::new("schema_name", DataType::Utf8, false), + Field::new("schema_owner", DataType::Utf8, true), + Field::new("default_character_set_catalog", DataType::Utf8, true), + Field::new("default_character_set_schema", DataType::Utf8, true), + Field::new("default_character_set_name", DataType::Utf8, true), + Field::new("sql_path", DataType::Utf8, true), + ]) +} + +/// Builds `information_schema.schemata` rows. +#[derive(Debug)] +pub struct InformationSchemataBuilder { + schema: SchemaRef, + catalog_name: StringBuilder, + schema_name: StringBuilder, + schema_owner: StringBuilder, + default_character_set_catalog: StringBuilder, + default_character_set_schema: StringBuilder, + default_character_set_name: StringBuilder, + sql_path: StringBuilder, +} + +impl Default for InformationSchemataBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemataBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(schemata_schema()), + catalog_name: StringBuilder::new(), + schema_name: StringBuilder::new(), + schema_owner: StringBuilder::new(), + default_character_set_catalog: StringBuilder::new(), + default_character_set_schema: StringBuilder::new(), + default_character_set_name: StringBuilder::new(), + sql_path: StringBuilder::new(), + } + } + + /// Append one row to the builder. `schema_owner` is the optional SQL + /// schema owner; the three `default_character_set_*` columns and + /// `sql_path` are written as null (DataFusion does not model those + /// concepts; see the PostgreSQL docs link on [`schemata_schema`]). + pub fn add_schemata( + &mut self, + catalog_name: &str, + schema_name: &str, + schema_owner: Option<&str>, + ) { + self.catalog_name.append_value(catalog_name); + self.schema_name.append_value(schema_name); + match schema_owner { + Some(owner) => self.schema_owner.append_value(owner), + None => self.schema_owner.append_null(), + } + self.default_character_set_catalog.append_null(); + self.default_character_set_schema.append_null(); + self.default_character_set_name.append_null(); + self.sql_path.append_null(); + } + + /// Finalize the builder into a [`RecordBatch`]. + /// + /// Returns an error only if Arrow buffer construction fails, which + /// the builder's column-count and type invariants make unreachable + /// under normal use. The `Result` return type preserves room to add + /// validation in the future without a breaking API change. + pub fn finish(&mut self) -> Result { + Ok(RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.catalog_name.finish()), + Arc::new(self.schema_name.finish()), + Arc::new(self.schema_owner.finish()), + Arc::new(self.default_character_set_catalog.finish()), + Arc::new(self.default_character_set_schema.finish()), + Arc::new(self.default_character_set_name.finish()), + Arc::new(self.sql_path.finish()), + ], + )?) + } +} + +/// Returns the Arrow schema of `information_schema.df_settings` rows. +/// +/// `SHOW ` / `SHOW ALL` without `VERBOSE` only keep `name`/`value`; +/// `verbose` (and the real `information_schema.df_settings` table, which is +/// always verbose) also includes `description`. +pub fn df_settings_schema(verbose: bool) -> Schema { + let mut fields = vec![ + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Utf8, true), + ]; + if verbose { + fields.push(Field::new("description", DataType::Utf8, true)); + } + Schema::new(fields) +} + +/// Builds `information_schema.df_settings` rows, retaining only the columns +/// needed at the given verbosity. +#[derive(Debug)] +pub struct InformationSchemaDfSettingsBuilder { + schema: SchemaRef, + names: StringBuilder, + values: StringBuilder, + descriptions: Option, +} + +impl InformationSchemaDfSettingsBuilder { + /// Construct an empty builder that will only retain `description` when + /// `verbose` is `true`. + pub fn new(verbose: bool) -> Self { + Self { + schema: Arc::new(df_settings_schema(verbose)), + names: StringBuilder::new(), + values: StringBuilder::new(), + descriptions: verbose.then(StringBuilder::new), + } + } + + /// Append one row. + pub fn add_setting(&mut self, entry: ConfigEntry) { + self.names.append_value(entry.key); + self.values.append_option(entry.value); + if let Some(descriptions) = &mut self.descriptions { + descriptions.append_value(entry.description); + } + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + let mut columns: Vec = vec![ + Arc::new(self.names.finish()), + Arc::new(self.values.finish()), + ]; + if let Some(descriptions) = &mut self.descriptions { + columns.push(Arc::new(descriptions.finish())); + } + RecordBatch::try_new(Arc::clone(&self.schema), columns).unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.routines` rows. +pub fn routines_schema() -> Schema { + Schema::new(vec![ + Field::new("specific_catalog", DataType::Utf8, false), + Field::new("specific_schema", DataType::Utf8, false), + Field::new("specific_name", DataType::Utf8, false), + Field::new("routine_catalog", DataType::Utf8, false), + Field::new("routine_schema", DataType::Utf8, false), + Field::new("routine_name", DataType::Utf8, false), + Field::new("routine_type", DataType::Utf8, false), + Field::new("is_deterministic", DataType::Boolean, true), + Field::new("data_type", DataType::Utf8, true), + Field::new("function_type", DataType::Utf8, true), + Field::new("description", DataType::Utf8, true), + Field::new("syntax_example", DataType::Utf8, true), + ]) +} + +/// Builds `information_schema.routines` rows. +#[derive(Debug)] +pub struct InformationSchemaRoutinesBuilder { + schema: SchemaRef, + specific_catalog: StringBuilder, + specific_schema: StringBuilder, + specific_name: StringBuilder, + routine_catalog: StringBuilder, + routine_schema: StringBuilder, + routine_name: StringBuilder, + routine_type: StringBuilder, + is_deterministic: BooleanBuilder, + data_type: StringBuilder, + function_type: StringBuilder, + description: StringBuilder, + syntax_example: StringBuilder, +} + +impl Default for InformationSchemaRoutinesBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaRoutinesBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(routines_schema()), + specific_catalog: StringBuilder::new(), + specific_schema: StringBuilder::new(), + specific_name: StringBuilder::new(), + routine_catalog: StringBuilder::new(), + routine_schema: StringBuilder::new(), + routine_name: StringBuilder::new(), + routine_type: StringBuilder::new(), + is_deterministic: BooleanBuilder::new(), + data_type: StringBuilder::new(), + function_type: StringBuilder::new(), + description: StringBuilder::new(), + syntax_example: StringBuilder::new(), + } + } + + /// Append one row. + #[expect(clippy::too_many_arguments)] + pub fn add_routine( + &mut self, + catalog_name: impl AsRef, + schema_name: impl AsRef, + routine_name: impl AsRef, + routine_type: impl AsRef, + is_deterministic: bool, + data_type: Option<&impl AsRef>, + function_type: impl AsRef, + description: Option>, + syntax_example: Option>, + ) { + self.specific_catalog.append_value(catalog_name.as_ref()); + self.specific_schema.append_value(schema_name.as_ref()); + self.specific_name.append_value(routine_name.as_ref()); + self.routine_catalog.append_value(catalog_name.as_ref()); + self.routine_schema.append_value(schema_name.as_ref()); + self.routine_name.append_value(routine_name.as_ref()); + self.routine_type.append_value(routine_type.as_ref()); + self.is_deterministic.append_value(is_deterministic); + self.data_type.append_option(data_type.as_ref()); + self.function_type.append_value(function_type.as_ref()); + self.description.append_option(description); + self.syntax_example.append_option(syntax_example); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.specific_catalog.finish()), + Arc::new(self.specific_schema.finish()), + Arc::new(self.specific_name.finish()), + Arc::new(self.routine_catalog.finish()), + Arc::new(self.routine_schema.finish()), + Arc::new(self.routine_name.finish()), + Arc::new(self.routine_type.finish()), + Arc::new(self.is_deterministic.finish()), + Arc::new(self.data_type.finish()), + Arc::new(self.function_type.finish()), + Arc::new(self.description.finish()), + Arc::new(self.syntax_example.finish()), + ], + ) + .unwrap() + } +} + +/// Returns the Arrow schema of `information_schema.parameters` rows. +pub fn parameters_schema() -> Schema { + Schema::new(vec![ + Field::new("specific_catalog", DataType::Utf8, false), + Field::new("specific_schema", DataType::Utf8, false), + Field::new("specific_name", DataType::Utf8, false), + Field::new("ordinal_position", DataType::UInt64, false), + Field::new("parameter_mode", DataType::Utf8, false), + Field::new("parameter_name", DataType::Utf8, true), + Field::new("data_type", DataType::Utf8, false), + Field::new("parameter_default", DataType::Utf8, true), + Field::new("is_variadic", DataType::Boolean, false), + // `rid` (short for `routine id`) is used to differentiate parameters from different signatures + // (It serves as the group-by key when generating the `SHOW FUNCTIONS` query). + // For example, the following signatures have different `rid` values: + // - `datetrunc(Utf8, Timestamp(Microsecond, Some("+TZ"))) -> Timestamp(Microsecond, Some("+TZ"))` + // - `datetrunc(Utf8View, Timestamp(Nanosecond, None)) -> Timestamp(Nanosecond, None)` + Field::new("rid", DataType::UInt8, false), + ]) +} + +/// Builds `information_schema.parameters` rows. +#[derive(Debug)] +pub struct InformationSchemaParametersBuilder { + schema: SchemaRef, + specific_catalog: StringBuilder, + specific_schema: StringBuilder, + specific_name: StringBuilder, + ordinal_position: UInt64Builder, + parameter_mode: StringBuilder, + parameter_name: StringBuilder, + data_type: StringBuilder, + parameter_default: StringBuilder, + is_variadic: BooleanBuilder, + rid: UInt8Builder, +} + +impl Default for InformationSchemaParametersBuilder { + fn default() -> Self { + Self::new() + } +} + +impl InformationSchemaParametersBuilder { + /// Construct an empty builder. + pub fn new() -> Self { + Self { + schema: Arc::new(parameters_schema()), + specific_catalog: StringBuilder::new(), + specific_schema: StringBuilder::new(), + specific_name: StringBuilder::new(), + ordinal_position: UInt64Builder::new(), + parameter_mode: StringBuilder::new(), + parameter_name: StringBuilder::new(), + data_type: StringBuilder::new(), + parameter_default: StringBuilder::new(), + is_variadic: BooleanBuilder::new(), + rid: UInt8Builder::new(), + } + } + + /// Append one row. + #[expect(clippy::too_many_arguments)] + pub fn add_parameter( + &mut self, + specific_catalog: impl AsRef, + specific_schema: impl AsRef, + specific_name: impl AsRef, + ordinal_position: u64, + parameter_mode: impl AsRef, + parameter_name: Option<&(impl AsRef + ?Sized)>, + data_type: impl AsRef, + parameter_default: Option>, + is_variadic: bool, + rid: u8, + ) { + self.specific_catalog + .append_value(specific_catalog.as_ref()); + self.specific_schema.append_value(specific_schema.as_ref()); + self.specific_name.append_value(specific_name.as_ref()); + self.ordinal_position.append_value(ordinal_position); + self.parameter_mode.append_value(parameter_mode.as_ref()); + self.parameter_name.append_option(parameter_name.as_ref()); + self.data_type.append_value(data_type.as_ref()); + self.parameter_default.append_option(parameter_default); + self.is_variadic.append_value(is_variadic); + self.rid.append_value(rid); + } + + /// Finalize the builder into a [`RecordBatch`]. + pub fn finish(&mut self) -> RecordBatch { + RecordBatch::try_new( + Arc::clone(&self.schema), + vec![ + Arc::new(self.specific_catalog.finish()), + Arc::new(self.specific_schema.finish()), + Arc::new(self.specific_name.finish()), + Arc::new(self.ordinal_position.finish()), + Arc::new(self.parameter_mode.finish()), + Arc::new(self.parameter_name.finish()), + Arc::new(self.data_type.finish()), + Arc::new(self.parameter_default.finish()), + Arc::new(self.is_variadic.finish()), + Arc::new(self.rid.finish()), + ], + ) + .unwrap() + } +} + +/// Returns the Arrow schema of `SHOW FUNCTIONS [LIKE ]`. +pub fn show_functions_schema() -> Schema { + Schema::new(vec![ + Field::new("function_name", DataType::Utf8, false), + Field::new("return_type", DataType::Utf8, true), + Field::new("parameters", DataType::new_list(DataType::Utf8, true), true), + Field::new( + "parameter_types", + DataType::new_list(DataType::Utf8, true), + true, + ), + Field::new("function_type", DataType::Utf8, true), + Field::new("description", DataType::Utf8, true), + Field::new("syntax_example", DataType::Utf8, true), + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{Array, StringArray}; + + #[test] + fn schemata_builder_emits_canonical_schema_and_rows() { + // Construct via `Default` so the test exercises both `new()` (via + // the `Default` impl) and the public column-layout contract. + let mut builder = InformationSchemataBuilder::default(); + builder.add_schemata("cat", "schema_one", Some("alice")); + builder.add_schemata("cat", "schema_two", None); + let batch = builder.finish().expect("finish should not fail"); + + assert_eq!(batch.schema().as_ref(), &schemata_schema()); + assert_eq!(batch.num_rows(), 2); + + let col = |name: &str| { + batch + .column_by_name(name) + .unwrap_or_else(|| panic!("missing column {name}")) + }; + let string_col = |name: &str| { + col(name) + .as_any() + .downcast_ref::() + .unwrap_or_else(|| panic!("{name} should be a StringArray")) + }; + + let catalog = string_col("catalog_name"); + assert_eq!(catalog.value(0), "cat"); + assert_eq!(catalog.value(1), "cat"); + + let schema = string_col("schema_name"); + assert_eq!(schema.value(0), "schema_one"); + assert_eq!(schema.value(1), "schema_two"); + + let owner = string_col("schema_owner"); + assert_eq!(owner.value(0), "alice"); + assert!(owner.is_null(1)); + + // The three character-set columns and sql_path are unconditionally + // null — they exist for SQL-standard column-layout compatibility. + for name in [ + "default_character_set_catalog", + "default_character_set_schema", + "default_character_set_name", + "sql_path", + ] { + let c = string_col(name); + assert!(c.is_null(0), "{name} row 0 should be null"); + assert!(c.is_null(1), "{name} row 1 should be null"); + } + } + + #[test] + fn tables_builder_renders_table_type_as_sql_string() { + let mut builder = InformationSchemaTablesBuilder::new(); + builder.add_table("cat", "schema", "t", TableType::Base); + let batch = builder.finish(); + + assert_eq!(batch.schema().as_ref(), &tables_schema()); + let table_types = batch + .column_by_name("table_type") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(table_types.value(0), "BASE TABLE"); + } + + #[test] + fn columns_builder_describe_retains_only_describe_columns() { + let field = Field::new("c", DataType::Int32, true); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Describe); + builder.add_column("cat", "schema", "t", 0, &field); + let batch = builder.finish(); + + assert_eq!( + batch.schema().as_ref(), + &columns_schema(ColumnsDetail::Describe) + ); + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(), + vec!["column_name", "data_type", "is_nullable"] + ); + assert_eq!( + batch + .column_by_name("column_name") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "c" + ); + } + + #[test] + fn columns_builder_basic_adds_table_qualifiers_but_not_full_metadata() { + let field = Field::new("c", DataType::Int32, true); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Basic); + builder.add_column("cat", "schema", "t", 0, &field); + let batch = builder.finish(); + + assert_eq!( + batch.schema().as_ref(), + &columns_schema(ColumnsDetail::Basic) + ); + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(), + vec![ + "table_catalog", + "table_schema", + "table_name", + "column_name", + "data_type", + "is_nullable", + ] + ); + assert_eq!( + batch + .column_by_name("table_catalog") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + "cat" + ); + } + + #[test] + fn columns_builder_full_retains_every_column_in_standard_order() { + let field = Field::new("c", DataType::Decimal128(10, 2), false); + let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Full); + builder.add_column("cat", "schema", "t", 3, &field); + let batch = builder.finish(); + + assert_eq!( + batch.schema().as_ref(), + &columns_schema(ColumnsDetail::Full) + ); + assert_eq!(batch.num_columns(), 15); + // `Full` keeps the SQL-standard order (`is_nullable` before + // `data_type`), unlike `Describe`/`Basic`. + assert_eq!( + batch + .schema() + .fields() + .iter() + .map(|f| f.name().as_str()) + .collect::>(), + vec![ + "table_catalog", + "table_schema", + "table_name", + "column_name", + "ordinal_position", + "column_default", + "is_nullable", + "data_type", + "character_maximum_length", + "character_octet_length", + "numeric_precision", + "numeric_precision_radix", + "numeric_scale", + "datetime_precision", + "interval_type", + ] + ); + assert_eq!( + batch + .column_by_name("ordinal_position") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 3 + ); + assert_eq!( + batch + .column_by_name("numeric_precision") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(0), + 10 + ); + } +} diff --git a/datafusion/common/src/lib.rs b/datafusion/common/src/lib.rs index 2eebfe4963057..d07cb85b19cce 100644 --- a/datafusion/common/src/lib.rs +++ b/datafusion/common/src/lib.rs @@ -33,6 +33,7 @@ mod param_value; mod partitioning; mod schema_reference; mod table_reference; +mod table_type; mod unnest; pub mod alias; @@ -49,6 +50,7 @@ pub mod file_options; pub mod format; pub mod hash_utils; pub mod heap_size; +pub mod information_schema; pub mod instant; pub mod metadata; pub mod nested_struct; @@ -99,6 +101,7 @@ pub use schema_reference::SchemaReference; pub use spans::{Location, Span, Spans}; pub use stats::{ColumnStatistics, Statistics}; pub use table_reference::{ResolvedTableReference, TableReference}; +pub use table_type::TableType; pub use unnest::{NullHandling, RecursionUnnestOption, UnnestOptions}; pub use utils::project_schema; diff --git a/datafusion/common/src/table_type.rs b/datafusion/common/src/table_type.rs new file mode 100644 index 0000000000000..144baae689213 --- /dev/null +++ b/datafusion/common/src/table_type.rs @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// Indicates the type of this table for metadata/catalog purposes. +/// +/// Lives in `datafusion-common` (and is re-exported as `datafusion_expr::TableType`) +/// rather than in `datafusion-expr`, so that lower-level crates that cannot +/// depend on `datafusion-expr` — such as the shared `information_schema` +/// builders in [`crate::information_schema`] — can still use it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TableType { + /// An ordinary physical table. + Base, + /// A non-materialized table that itself uses a query internally to provide data. + View, + /// A transient table. + Temporary, +} + +impl std::fmt::Display for TableType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TableType::Base => write!(f, "Base"), + TableType::View => write!(f, "View"), + TableType::Temporary => write!(f, "Temporary"), + } + } +} diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 6db66e0e266fa..23d1584928f98 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -27,6 +27,7 @@ use crate::datasource::physical_plan::{FileOutputMode, FileSinkConfig}; use crate::datasource::{DefaultTableSource, source_as_provider}; use crate::error::{DataFusionError, Result}; use crate::execution::context::ExecutionProps; +use crate::execution::session_state::SessionState; use crate::logical_expr::utils::generate_sort_key; use crate::logical_expr::{ Aggregate, EmptyRelation, Join, Projection, Sort, TableScan, Unnest, Values, Window, @@ -60,11 +61,12 @@ use crate::physical_plan::{ }; use crate::schema_equivalence::schema_satisfied_by; -use arrow::array::{ArrayRef, RecordBatch, UInt64Array, builder::StringBuilder}; +use arrow::array::{ArrayRef, RecordBatch, UInt64Array}; use arrow::compute::SortOptions; -use arrow::datatypes::Schema; +use arrow::datatypes::{DataType, Schema}; use arrow_schema::Field; use datafusion_catalog::ScanArgs; +use datafusion_catalog::information_schema::InformationSchemaProvider; use datafusion_common::Column; use datafusion_common::HashMap as DFHashMap; use datafusion_common::config::ConfigOptions; @@ -93,9 +95,9 @@ use datafusion_expr::physical_planning_context::{ }; use datafusion_expr::utils::{expr_to_columns, split_conjunction}; use datafusion_expr::{ - Analyze, BinaryExpr, DescribeTable, DmlStatement, Explain, ExplainFormat, Extension, - FetchType, Filter, JoinType, Operator, RecursiveQuery, SkipType, StringifiedPlan, - WindowFrame, WindowFrameBound, WriteOp, + Analyze, BinaryExpr, DmlStatement, Explain, ExplainFormat, Extension, FetchType, + Filter, JoinType, Operator, RecursiveQuery, Show, ShowKind, SkipType, + StringifiedPlan, WindowFrame, WindowFrameBound, WriteOp, }; use datafusion_physical_expr::aggregate::{ AggregateFunctionExpr, LoweredAggregate, LoweredAggregateBuilder, @@ -830,12 +832,73 @@ impl DefaultPhysicalPlanner { produce_one_row: true, schema, }) => Arc::new(PlaceholderRowExec::new(Arc::clone(schema.inner()))), - LogicalPlan::DescribeTable(DescribeTable { - schema, - output_schema, + LogicalPlan::Show(Show { + kind: ShowKind::Tables, + .. + }) => InformationSchemaProvider::from(session_state).show_tables()?, + LogicalPlan::Show(Show { + kind: ShowKind::Describe { schema }, + .. + }) => { + InformationSchemaProvider::from(session_state).describe_schema(schema)? + } + LogicalPlan::Show(Show { + kind: ShowKind::Columns { table_ref, detail }, + .. + }) => { + let catalog_options = &session_state.config_options().catalog; + let resolved = table_ref.clone().resolve( + &catalog_options.default_catalog, + &catalog_options.default_schema, + ); + InformationSchemaProvider::from(session_state) + .show_columns(&resolved, *detail) + .await? + } + LogicalPlan::Show(Show { + kind: ShowKind::CreateTable { table_ref }, + .. + }) => { + let catalog_options = &session_state.config_options().catalog; + let resolved = table_ref.clone().resolve( + &catalog_options.default_catalog, + &catalog_options.default_schema, + ); + + InformationSchemaProvider::from(session_state) + .show_create_table(&resolved) + .await? + } + LogicalPlan::Show(Show { + kind: ShowKind::Variables { name, verbose }, + .. + }) => InformationSchemaProvider::from(session_state).show_variables( + session_state.config_options(), + session_state.runtime_env(), + name.as_deref(), + *verbose, + )?, + LogicalPlan::Show(Show { + kind: ShowKind::Functions { filter }, + .. }) => { - let output_schema = Arc::clone(output_schema.inner()); - self.plan_describe(&Arc::clone(schema), output_schema)? + // Table functions (UDTFs) are only reachable through the + // concrete `SessionState` (the `Session` trait has no + // accessor for them); a custom `Session` implementor will + // simply see no `TABLE`-typed routines in `SHOW FUNCTIONS`. + let table_functions = session_state + .as_any() + .downcast_ref::() + .map(|state| state.table_functions().clone()) + .unwrap_or_default(); + InformationSchemaProvider::from(session_state) + .with_table_functions(table_functions) + .show_functions( + session_state.scalar_functions(), + session_state.aggregate_functions(), + session_state.window_functions(), + filter.as_deref(), + )? } // 1 Child @@ -859,7 +922,7 @@ impl DefaultPhysicalPlanner { // the column name rather than column name + explicit data type. let table_partition_cols = partition_by .iter() - .map(|s| (s.to_string(), arrow::datatypes::DataType::Null)) + .map(|s| (s.to_string(), DataType::Null)) .collect::>(); let keep_partition_by_columns = match source_option_tuples @@ -2720,7 +2783,7 @@ fn extract_dml_filters( // Plans without filter information LogicalPlan::EmptyRelation(_) | LogicalPlan::Values(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Explain(_) | LogicalPlan::Analyze(_) | LogicalPlan::Distinct(_) @@ -3406,43 +3469,6 @@ impl DefaultPhysicalPlanner { Ok(new_plan) } - // return an record_batch which describes a table's schema. - fn plan_describe( - &self, - table_schema: &Arc, - output_schema: Arc, - ) -> Result> { - let mut column_names = StringBuilder::new(); - let mut data_types = StringBuilder::new(); - let mut is_nullables = StringBuilder::new(); - for field in table_schema.fields() { - column_names.append_value(field.name()); - - // "System supplied type" --> Use debug format of the datatype - let data_type = field.data_type(); - data_types.append_value(format!("{data_type}")); - - // "YES if the column is possibly nullable, NO if it is known not nullable. " - let nullable_str = if field.is_nullable() { "YES" } else { "NO" }; - is_nullables.append_value(nullable_str); - } - - let record_batch = RecordBatch::try_new( - output_schema, - vec![ - Arc::new(column_names.finish()), - Arc::new(data_types.finish()), - Arc::new(is_nullables.finish()), - ], - )?; - - let schema = record_batch.schema(); - let partitions = vec![vec![record_batch]]; - let projection = None; - let mem_exec = MemorySourceConfig::try_new_exec(&partitions, schema, projection)?; - Ok(mem_exec) - } - /// Build physical plans for scalar subqueries and assign each an ordinal /// `SubqueryIndex`. Returns the links (plan + index) and a map from logical /// `Subquery` to its index. diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index aa81df22554c2..1b5f580e5f021 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -33,8 +33,8 @@ use crate::expr_rewriter::{ use crate::logical_plan::{ Aggregate, Analyze, AsOfJoin, AsOfMatch, Distinct, DistinctOn, EmptyRelation, Explain, Filter, Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, - PlanType, Prepare, Projection, Repartition, Sort, SubqueryAlias, TableScanBuilder, - Union, Unnest, Values, Window, + PlanType, Prepare, Projection, Repartition, Show, ShowKind, Sort, SubqueryAlias, + TableScanBuilder, Union, Unnest, Values, Window, }; use crate::select_expr::SelectExpr; use crate::utils::{ @@ -54,6 +54,7 @@ use arrow::compute::can_cast_types; use arrow::datatypes::{DataType, Field, FieldRef, Fields, Schema, SchemaRef}; use datafusion_common::display::ToStringifiedPlan; use datafusion_common::file_options::file_type::FileType; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::metadata::FieldMetadata; use datafusion_common::{ Column, Constraints, DFSchema, DFSchemaRef, FunctionalDependencies, NullEquality, @@ -173,6 +174,72 @@ impl LogicalPlanBuilder { })) } + /// Create a `DESCRIBE ` plan, describing the schema of a query (or + /// other statement) that isn't backed by any catalog table. + pub fn describe(schema: Arc) -> Result { + Self::show( + ShowKind::Describe { schema }, + datafusion_common::information_schema::columns_schema( + ColumnsDetail::Describe, + ), + ) + } + + /// Create a `SHOW TABLES` plan. + pub fn show_tables() -> Result { + Self::show( + ShowKind::Tables, + datafusion_common::information_schema::tables_schema(), + ) + } + + /// Create a `DESCRIBE
` / `SHOW COLUMNS FROM
` plan. + /// `SHOW FULL COLUMNS` / `SHOW EXTENDED COLUMNS` are treated the same and + /// should pass [`ColumnsDetail::Full`]; `DESCRIBE
` should pass + /// [`ColumnsDetail::Describe`]. + pub fn show_columns( + table_ref: TableReference, + detail: ColumnsDetail, + ) -> Result { + Self::show( + ShowKind::Columns { table_ref, detail }, + datafusion_common::information_schema::columns_schema(detail), + ) + } + + /// Create a `SHOW CREATE TABLE
` plan. + pub fn show_create_table(table_ref: TableReference) -> Result { + Self::show( + ShowKind::CreateTable { table_ref }, + datafusion_common::information_schema::views_schema(), + ) + } + + /// Create a `SHOW ` / `SHOW ALL` plan. `name` is `None` for + /// `SHOW ALL`. + pub fn show_variables(name: Option, verbose: bool) -> Result { + Self::show( + ShowKind::Variables { name, verbose }, + datafusion_common::information_schema::df_settings_schema(verbose), + ) + } + + /// Create a `SHOW FUNCTIONS [LIKE ]` plan. + pub fn show_functions(filter: Option) -> Result { + Self::show( + ShowKind::Functions { filter }, + datafusion_common::information_schema::show_functions_schema(), + ) + } + + /// Shared helper for the `SHOW`-class factory methods above. + fn show(kind: ShowKind, output_schema: Schema) -> Result { + Ok(Self::new(LogicalPlan::Show(Show { + kind, + output_schema: Arc::new(DFSchema::try_from(output_schema)?), + }))) + } + /// Convert a regular plan into a recursive query. /// `is_distinct` indicates whether the recursive term should be de-duplicated (`UNION`) after each iteration or not (`UNION ALL`). pub fn to_recursive_query( diff --git a/datafusion/expr/src/logical_plan/display.rs b/datafusion/expr/src/logical_plan/display.rs index 2dcaefe9a2af5..fee9aec977b5f 100644 --- a/datafusion/expr/src/logical_plan/display.rs +++ b/datafusion/expr/src/logical_plan/display.rs @@ -21,8 +21,8 @@ use std::collections::HashMap; use std::fmt; use crate::{ - Aggregate, AsOfJoin, DescribeTable, Distinct, DistinctOn, DmlStatement, Expr, Filter, - Join, Limit, LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition, + Aggregate, AsOfJoin, Distinct, DistinctOn, DmlStatement, Expr, Filter, Join, Limit, + LogicalPlan, Partitioning, Projection, RecursiveQuery, Repartition, Show, ShowKind, Sort, Subquery, SubqueryAlias, TableProviderFilterPushDown, TableScan, Unnest, Values, Window, expr_vec_fmt, }; @@ -30,6 +30,7 @@ use crate::{ use crate::dml::CopyTo; use arrow::datatypes::Schema; use datafusion_common::display::GraphvizBuilder; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::tree_node::{TreeNodeRecursion, TreeNodeVisitor}; use datafusion_common::{Column, DataFusionError, internal_datafusion_err}; use serde::Serialize; @@ -691,11 +692,59 @@ impl<'a, 'b> PgJsonVisitor<'a, 'b> { "Detail": format!("{:?}", e.node) ) } - LogicalPlan::DescribeTable(DescribeTable { .. }) => { - pg_fields!( - "Node Type": "DescribeTable" - ) - } + LogicalPlan::Show(Show { kind, .. }) => match kind { + ShowKind::Describe { .. } => { + pg_fields!( + "Node Type": "Describe" + ) + } + ShowKind::Tables => { + pg_fields!( + "Node Type": "ShowTables" + ) + } + ShowKind::Columns { + table_ref, + detail: ColumnsDetail::Describe, + } => { + pg_fields!( + "Node Type": "DescribeTable", + "Relation Name": table_ref.table() + ) + } + ShowKind::Columns { table_ref, detail } => { + pg_fields!( + "Node Type": "ShowColumns", + "Relation Name": table_ref.table(), + "Detail": format!("{detail:?}") + ) + } + ShowKind::CreateTable { table_ref } => { + pg_fields!( + "Node Type": "ShowCreateTable", + "Relation Name": table_ref.table() + ) + } + ShowKind::Variables { name, verbose } => { + let mut object = pg_fields!( + "Node Type": "ShowVariables", + "Verbose": verbose + ); + if let Some(name) = name { + object.push("Variable", serde_json::Value::String(name.clone())); + } + object + } + ShowKind::Functions { filter } => { + let mut object = pg_fields!( + "Node Type": "ShowFunctions" + ); + if let Some(filter) = filter { + object.push("Filter", serde_json::Value::String(filter.clone())); + } + object + } + }, LogicalPlan::Unnest(Unnest { input: plan, list_type_columns: list_col_indices, diff --git a/datafusion/expr/src/logical_plan/mod.rs b/datafusion/expr/src/logical_plan/mod.rs index 81483dae7fb9c..4945d4d03654e 100644 --- a/datafusion/expr/src/logical_plan/mod.rs +++ b/datafusion/expr/src/logical_plan/mod.rs @@ -43,10 +43,10 @@ pub use dml::{ WriteOp, }; pub use plan::{ - Aggregate, Analyze, AsOfJoin, AsOfMatch, ColumnUnnestList, DescribeTable, Distinct, - DistinctOn, EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter, - Join, JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, - Projection, RangePartitioning, RecursiveQuery, Repartition, SkipType, Sort, + Aggregate, Analyze, AsOfJoin, AsOfMatch, ColumnUnnestList, Distinct, DistinctOn, + EmptyRelation, Explain, ExplainOption, Extension, FetchType, Filter, Join, + JoinConstraint, JoinType, Limit, LogicalPlan, Partitioning, PlanType, Projection, + RangePartitioning, RecursiveQuery, Repartition, Show, ShowKind, SkipType, Sort, StringifiedPlan, Subquery, SubqueryAlias, TableScan, TableScanBuilder, ToStringifiedPlan, Union, Unnest, Values, Window, projection_schema, }; diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index a2fee99ec2a64..01fa7637756ec 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -59,6 +59,7 @@ use arrow::compute::SortOptions; use arrow::datatypes::{DataType, Field, FieldRef, Metadata, Schema, SchemaRef}; use datafusion_common::cse::{NormalizeEq, Normalizeable}; use datafusion_common::format::{ExplainAnalyzeCategories, ExplainFormat, MetricType}; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::metadata::check_metadata_with_storage_equal; use datafusion_common::tree_node::{ Transformed, TreeNode, TreeNodeContainer, TreeNodeRecursion, @@ -68,7 +69,8 @@ use datafusion_common::{ FunctionalDependence, FunctionalDependencies, NullEquality, ParamValues, Result, ScalarValue, Spans, SplitPoint, TableReference, UnnestOptions, aggregate_functional_dependencies, assert_eq_or_internal_err, assert_or_internal_err, - internal_err, plan_datafusion_err, plan_err, validate_range_split_points, + information_schema, internal_err, plan_datafusion_err, plan_err, + validate_range_split_points, }; use itertools::Itertools as _; @@ -291,9 +293,10 @@ pub enum LogicalPlan { Ddl(DdlStatement), /// `COPY TO` for writing plan results to files Copy(CopyTo), - /// Describe the schema of the table. This is used to implement the - /// SQL `DESCRIBE` command from MySQL. - DescribeTable(DescribeTable), + /// A `SHOW`-class SQL statement: `SHOW TABLES`, `SHOW COLUMNS`, + /// `SHOW FUNCTIONS`, `SHOW `, `SHOW CREATE TABLE`, and + /// `DESCRIBE` (MySQL's alias for `SHOW COLUMNS FROM`). + Show(Show), /// Unnest a column that contains a nested list type such as an /// ARRAY. This is used to implement SQL `UNNEST` Unnest(Unnest), @@ -359,9 +362,7 @@ impl LogicalPlan { LogicalPlan::Analyze(analyze) => &analyze.schema, LogicalPlan::Extension(extension) => extension.node.schema(), LogicalPlan::Union(Union { schema, .. }) => schema, - LogicalPlan::DescribeTable(DescribeTable { output_schema, .. }) => { - output_schema - } + LogicalPlan::Show(Show { output_schema, .. }) => output_schema, LogicalPlan::Dml(DmlStatement { output_schema, .. }) => output_schema, LogicalPlan::Copy(CopyTo { output_schema, .. }) => output_schema, LogicalPlan::Ddl(ddl) => ddl.schema(), @@ -398,11 +399,7 @@ impl LogicalPlan { /// Returns the (fixed) output schema for `DESCRIBE` plans pub fn describe_schema() -> Schema { - Schema::new(vec![ - Field::new("column_name", DataType::Utf8, false), - Field::new("data_type", DataType::Utf8, false), - Field::new("is_nullable", DataType::Utf8, false), - ]) + information_schema::columns_schema(ColumnsDetail::Describe) } /// Returns all expressions (non-recursively) evaluated by the current @@ -498,7 +495,7 @@ impl LogicalPlan { LogicalPlan::TableScan { .. } | LogicalPlan::EmptyRelation { .. } | LogicalPlan::Values { .. } - | LogicalPlan::DescribeTable(_) => vec![], + | LogicalPlan::Show(_) => vec![], } } @@ -627,7 +624,7 @@ impl LogicalPlan { | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) | LogicalPlan::Ddl(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Unnest(_) => Ok(None), } } @@ -806,7 +803,7 @@ impl LogicalPlan { LogicalPlan::TableScan(_) => Ok(self), LogicalPlan::EmptyRelation(_) => Ok(self), LogicalPlan::Statement(_) => Ok(self), - LogicalPlan::DescribeTable(_) => Ok(self), + LogicalPlan::Show(_) => Ok(self), LogicalPlan::Unnest(Unnest { input, exec_columns, @@ -1276,7 +1273,7 @@ impl LogicalPlan { LogicalPlan::EmptyRelation(_) | LogicalPlan::Ddl(_) | LogicalPlan::Statement(_) - | LogicalPlan::DescribeTable(_) => { + | LogicalPlan::Show(_) => { // All of these plan types have no inputs / exprs so should not be called self.assert_no_expressions(expr)?; self.assert_no_inputs(inputs)?; @@ -1540,7 +1537,7 @@ impl LogicalPlan { | LogicalPlan::Analyze(_) | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Statement(_) | LogicalPlan::Extension(_) => None, } @@ -1617,7 +1614,7 @@ impl LogicalPlan { | LogicalPlan::Analyze(_) | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Statement(_) | LogicalPlan::Extension(_) => 0, } @@ -1656,7 +1653,7 @@ impl LogicalPlan { LogicalPlan::Dml(_) => Ok(None), LogicalPlan::Ddl(_) => Ok(None), LogicalPlan::Copy(_) => Ok(None), - LogicalPlan::DescribeTable(_) => Ok(None), + LogicalPlan::Show(_) => Ok(None), LogicalPlan::Unnest(_) => Ok(None), LogicalPlan::RecursiveQuery(_) => Ok(None), } @@ -1695,7 +1692,7 @@ impl LogicalPlan { LogicalPlan::Dml(_) => Ok(None), LogicalPlan::Ddl(_) => Ok(None), LogicalPlan::Copy(_) => Ok(None), - LogicalPlan::DescribeTable(_) => Ok(None), + LogicalPlan::Show(_) => Ok(None), LogicalPlan::Unnest(_) => Ok(None), LogicalPlan::RecursiveQuery(_) => Ok(None), } @@ -2433,9 +2430,18 @@ impl LogicalPlan { LogicalPlan::Analyze { .. } => write!(f, "Analyze"), LogicalPlan::Union(_) => write!(f, "Union"), LogicalPlan::Extension(e) => e.node.fmt_for_explain(f), - LogicalPlan::DescribeTable(DescribeTable { .. }) => { - write!(f, "DescribeTable") - } + LogicalPlan::Show(Show { kind, .. }) => match kind { + ShowKind::Describe { .. } + | ShowKind::Columns { + detail: ColumnsDetail::Describe, + .. + } => write!(f, "DescribeTable"), + ShowKind::Tables => write!(f, "ShowTables"), + ShowKind::Columns { .. } => write!(f, "ShowColumns"), + ShowKind::CreateTable { .. } => write!(f, "ShowCreateTable"), + ShowKind::Variables { .. } => write!(f, "ShowVariables"), + ShowKind::Functions { .. } => write!(f, "ShowFunctions"), + }, LogicalPlan::Unnest(Unnest { input: plan, list_type_columns: list_col_indices, @@ -3676,9 +3682,10 @@ impl PartialOrd for Union { } } -/// Describe the schema of table +/// A `SHOW`-class SQL statement: `SHOW TABLES`, `SHOW COLUMNS`, `SHOW +/// FUNCTIONS`, `SHOW `, `SHOW CREATE TABLE`, and `DESCRIBE`. /// -/// # Example output: +/// # Example output (`DESCRIBE`): /// /// ```sql /// > describe traces; @@ -3699,16 +3706,60 @@ impl PartialOrd for Union { /// +--------------------+-----------------------------+-------------+ /// ``` #[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct DescribeTable { - /// Table schema - pub schema: Arc, - /// schema of describe table output +pub struct Show { + /// Which `SHOW`-class statement this is, and any parameters it carries + pub kind: ShowKind, + /// schema of the output produced by this statement pub output_schema: DFSchemaRef, } +/// The specific `SHOW`-class statement carried by a [`Show`] plan. +/// +/// Note that none of these variants carry the `information_schema` content +/// itself: that is resolved later, by the physical planner, which already +/// depends on the catalog APIs needed to do so. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ShowKind { + /// `DESCRIBE `: describe the schema of a query (or other + /// statement) that isn't backed by a catalog table. + Describe { + /// The schema to describe + schema: Arc, + }, + /// `SHOW TABLES` + Tables, + /// `SHOW COLUMNS FROM table` / `SHOW FULL COLUMNS FROM table` / `SHOW + /// EXTENDED COLUMNS FROM table` (`FULL` and `EXTENDED` are treated the + /// same), and `DESCRIBE table` (which is just `SHOW COLUMNS` at + /// [`ColumnsDetail::Describe`] detail). + Columns { + /// Table to describe + table_ref: TableReference, + /// How detailed the output should be + detail: ColumnsDetail, + }, + /// `SHOW CREATE TABLE table`. + CreateTable { + /// Table to describe + table_ref: TableReference, + }, + /// `SHOW ` / `SHOW ALL`. + Variables { + /// The name of the variable to show, or `None` if all variables should be shown. + name: Option, + /// Whether to include the `description` column (`SHOW ... VERBOSE`) + verbose: bool, + }, + /// `SHOW FUNCTIONS [LIKE ]` + Functions { + /// The `LIKE` pattern to filter function names by, if any + filter: Option, + }, +} + // Manual implementation of `PartialOrd`, returning none since there are no comparable types in -// `DescribeTable`. This allows `LogicalPlan` to derive `PartialOrd`. -impl PartialOrd for DescribeTable { +// `Show`. This allows `LogicalPlan` to derive `PartialOrd`. +impl PartialOrd for Show { fn partial_cmp(&self, _other: &Self) -> Option { // There is no relevant comparison for schemas None @@ -6833,21 +6884,25 @@ mod tests { schema: Arc::new(DFSchema::empty()), }); - let describe_table = LogicalPlan::DescribeTable(DescribeTable { - schema: Arc::new(Schema::new(vec![Field::new( - "foo", - DataType::Int32, - false, - )])), + let describe_table = LogicalPlan::Show(Show { + kind: ShowKind::Describe { + schema: Arc::new(Schema::new(vec![Field::new( + "foo", + DataType::Int32, + false, + )])), + }, output_schema: DFSchemaRef::new(DFSchema::empty()), }); - let describe_table_clone = LogicalPlan::DescribeTable(DescribeTable { - schema: Arc::new(Schema::new(vec![Field::new( - "foo", - DataType::Int32, - false, - )])), + let describe_table_clone = LogicalPlan::Show(Show { + kind: ShowKind::Describe { + schema: Arc::new(Schema::new(vec![Field::new( + "foo", + DataType::Int32, + false, + )])), + }, output_schema: DFSchemaRef::new(DFSchema::empty()), }); diff --git a/datafusion/expr/src/logical_plan/tree_node.rs b/datafusion/expr/src/logical_plan/tree_node.rs index 9e961a1f1127a..e567686a163db 100644 --- a/datafusion/expr/src/logical_plan/tree_node.rs +++ b/datafusion/expr/src/logical_plan/tree_node.rs @@ -389,7 +389,7 @@ impl TreeNode for LogicalPlan { LogicalPlan::TableScan { .. } | LogicalPlan::EmptyRelation { .. } | LogicalPlan::Values { .. } - | LogicalPlan::DescribeTable(_) => Transformed::no(self), + | LogicalPlan::Show(_) => Transformed::no(self), }) } } @@ -525,7 +525,7 @@ impl LogicalPlan { | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) => Ok(TreeNodeRecursion::Continue), + | LogicalPlan::Show(_) => Ok(TreeNodeRecursion::Continue), } } @@ -812,7 +812,7 @@ impl LogicalPlan { | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) => Transformed::no(self), + | LogicalPlan::Show(_) => Transformed::no(self), }) } diff --git a/datafusion/expr/src/table_source.rs b/datafusion/expr/src/table_source.rs index f870b5e7ea949..a9a92c0901acd 100644 --- a/datafusion/expr/src/table_source.rs +++ b/datafusion/expr/src/table_source.rs @@ -20,6 +20,7 @@ use crate::{Expr, LogicalPlan}; use arrow::datatypes::SchemaRef; +pub use datafusion_common::TableType; use datafusion_common::{Constraints, Result}; use std::{any::Any, borrow::Cow}; @@ -50,27 +51,6 @@ pub enum TableProviderFilterPushDown { Exact, } -/// Indicates the type of this table for metadata/catalog purposes. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TableType { - /// An ordinary physical table. - Base, - /// A non-materialized table that itself uses a query internally to provide data. - View, - /// A transient table. - Temporary, -} - -impl std::fmt::Display for TableType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - TableType::Base => write!(f, "Base"), - TableType::View => write!(f, "View"), - TableType::Temporary => write!(f, "Temporary"), - } - } -} - /// Planning time information about a table. /// /// This trait is used during logical query planning and optimizations, and diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index e0d43d7c15244..b044b68235391 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -580,7 +580,7 @@ impl OptimizerRule for CommonSubexprEliminate { | LogicalPlan::Explain(_) | LogicalPlan::Analyze(_) | LogicalPlan::Statement(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Distinct(_) | LogicalPlan::Extension(_) | LogicalPlan::Dml(_) diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs index 0a1db9b2128c6..b8a9ca55e5070 100644 --- a/datafusion/optimizer/src/decorrelate.rs +++ b/datafusion/optimizer/src/decorrelate.rs @@ -359,7 +359,7 @@ impl TreeNodeRewriter for PullUpCorrelatedExpr { | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) => { + | LogicalPlan::Show(_) => { if holds_outer_reference(&plan) { Ok(self.stop_pull_up(plan)) } else { diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index c913a4ce961cc..86b4758ba4bfb 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -364,9 +364,7 @@ fn optimize_projections( .collect() } } - LogicalPlan::EmptyRelation(_) - | LogicalPlan::Values(_) - | LogicalPlan::DescribeTable(_) => { + LogicalPlan::EmptyRelation(_) | LogicalPlan::Values(_) | LogicalPlan::Show(_) => { // These operators have no inputs, so stop the optimization process. return Ok(Transformed::no(plan)); } diff --git a/datafusion/optimizer/src/optimizer.rs b/datafusion/optimizer/src/optimizer.rs index 94583d545dec0..70701ebfa370d 100644 --- a/datafusion/optimizer/src/optimizer.rs +++ b/datafusion/optimizer/src/optimizer.rs @@ -471,7 +471,7 @@ fn map_children_mut Result>( LogicalPlan::TableScan { .. } | LogicalPlan::EmptyRelation { .. } | LogicalPlan::Values { .. } - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::Ddl(DdlStatement::CreateExternalTable(_)) | LogicalPlan::Ddl(DdlStatement::CreateCatalogSchema(_)) | LogicalPlan::Ddl(DdlStatement::CreateCatalog(_)) diff --git a/datafusion/proto/src/logical_plan/mod.rs b/datafusion/proto/src/logical_plan/mod.rs index d86b27ff3c4e3..4972d27e9c393 100644 --- a/datafusion/proto/src/logical_plan/mod.rs +++ b/datafusion/proto/src/logical_plan/mod.rs @@ -2408,8 +2408,8 @@ impl AsLogicalPlan for LogicalPlanNode { ))), }) } - LogicalPlan::DescribeTable(_) => Err(proto_error( - "LogicalPlan serde is not yet implemented for DescribeTable", + LogicalPlan::Show(_) => Err(proto_error( + "LogicalPlan serde is not yet implemented for Show", )), LogicalPlan::RecursiveQuery(recursive) => { let static_term = LogicalPlanNode::try_from_logical_plan( diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 609738a33988f..bd82210b8828f 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -16,23 +16,21 @@ // under the License. use std::collections::{BTreeMap, HashMap, HashSet}; -use std::ops::Deref; use std::path::Path; use std::str::FromStr; use std::sync::Arc; use crate::parser::{ - CopyToSource, CopyToStatement, CreateExternalCatalog, CreateExternalTable, DFParser, + CopyToSource, CopyToStatement, CreateExternalCatalog, CreateExternalTable, ExplainStatement, LexOrdering, ResetStatement, Statement as DFStatement, }; -use crate::planner::{ - ContextProvider, PlannerContext, SqlToRel, object_name_to_qualifier, -}; +use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use crate::utils::normalize_ident; use arrow::datatypes::{Field, FieldRef, Fields, Metadata}; use datafusion_common::error::_plan_err; use datafusion_common::format::ExplainStatementOptions; +use datafusion_common::information_schema::ColumnsDetail; use datafusion_common::parsers::CompressionTypeVariant; use datafusion_common::{ Column, Constraint, Constraints, DFSchema, DFSchemaRef, DataFusionError, Result, @@ -52,9 +50,9 @@ use datafusion_expr::{ CreateExternalCatalog as PlanCreateExternalCatalog, CreateExternalTable as PlanCreateExternalTable, CreateFunction, CreateFunctionBody, CreateIndex as PlanCreateIndex, CreateMemoryTable, CreateView, Deallocate, - DescribeTable, DmlStatement, DropCatalog, DropCatalogSchema, DropFunction, DropTable, - DropView, EmptyRelation, Execute, Explain, ExplainFormat, Expr, ExprSchemable, - Filter, LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, + DmlStatement, DropCatalog, DropCatalogSchema, DropFunction, DropTable, DropView, + EmptyRelation, Execute, Explain, ExplainFormat, Expr, ExprSchemable, Filter, + LogicalPlan, LogicalPlanBuilder, OperateFunctionArg, PlanType, Prepare, ResetVariable, SetVariable, SortExpr, Statement as PlanStatement, ToStringifiedPlan, TransactionAccessMode, TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart, Volatility, WriteOp, cast, @@ -1676,31 +1674,16 @@ impl SqlToRel<'_, S> { /// Generate a logical plan from a "SHOW TABLES" query fn show_tables_to_plan(&self) -> Result { - let Some(tables_table_ref) = self.resolve_info_table("tables") else { - return plan_err!( - "SHOW TABLES is not supported unless information_schema is enabled" - ); - }; - - let query = format!("SELECT * FROM {tables_table_ref};"); - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 + LogicalPlanBuilder::show_tables()?.build() } fn describe_table_to_plan(&self, table_name: ObjectName) -> Result { let table_ref = self.object_name_to_table_reference(table_name)?; - let table_source = self.context_provider.get_table_source(table_ref)?; - - let schema = table_source.schema(); - - let output_schema = DFSchema::try_from(LogicalPlan::describe_schema()).unwrap(); + // Do a table lookup to verify the table exists + let _ = self.context_provider.get_table_source(table_ref.clone())?; - Ok(LogicalPlan::DescribeTable(DescribeTable { - schema, - output_schema: Arc::new(output_schema), - })) + LogicalPlanBuilder::show_columns(table_ref, ColumnsDetail::Describe)?.build() } fn describe_query_to_plan(&self, query: Query) -> Result { @@ -1708,12 +1691,7 @@ impl SqlToRel<'_, S> { let schema = Arc::new(plan.schema().as_arrow().clone()); - let output_schema = DFSchema::try_from(LogicalPlan::describe_schema()).unwrap(); - - Ok(LogicalPlan::DescribeTable(DescribeTable { - schema, - output_schema: Arc::new(output_schema), - })) + LogicalPlanBuilder::describe(schema)?.build() } fn copy_to_plan(&self, statement: CopyToStatement) -> Result { @@ -2213,59 +2191,25 @@ impl SqlToRel<'_, S> { } fn show_variable_to_plan(&self, variable: &[Ident]) -> Result { - let Some(df_settings_table_ref) = self.resolve_info_table("df_settings") else { - return plan_err!( - "SHOW [VARIABLE] is not supported unless information_schema is enabled" - ); - }; - let verbose = variable .last() .map(|s| ident_to_string(s) == "verbose") .unwrap_or(false); let mut variable_vec = variable.to_vec(); - let mut columns: String = "name, value".to_owned(); if verbose { - columns = format!("{columns}, description"); variable_vec = variable_vec.split_at(variable_vec.len() - 1).0.to_vec(); } let variable = object_name_to_string(&ObjectName::from(variable_vec)); - let base_query = format!("SELECT {columns} FROM {df_settings_table_ref}"); - let query = if variable == "all" { - // Add an ORDER BY so the output comes out in a consistent order - format!("{base_query} ORDER BY name") - } else if variable == "timezone" || variable == "time.zone" { - // we could introduce alias in OptionDefinition if this string matching thing grows - format!("{base_query} WHERE name = 'datafusion.execution.time_zone'") + let name = if variable == "all" { + None } else { - // These values are what are used to make the information_schema table, so we just - // check here, before actually planning or executing the query, if it would produce no - // results, and error preemptively if it would (for a better UX) - let is_valid_variable = self - .context_provider - .options() - .entries() - .iter() - .any(|opt| opt.key == variable); - - // Check if it's a runtime variable - let is_runtime_variable = variable.starts_with("datafusion.runtime."); - - if !is_valid_variable && !is_runtime_variable { - return plan_err!( - "'{variable}' is not a variable which can be viewed with 'SHOW'" - ); - } - - format!("{base_query} WHERE name = '{variable}'") + // Whether this is actually a known variable is checked later at execution timne. + Some(variable) }; - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - - self.statement_to_plan(rewrite.pop_front().unwrap()) + LogicalPlanBuilder::show_variables(name, verbose)?.build() } fn set_statement_to_plan(&self, statement: Set) -> Result { @@ -2984,210 +2928,54 @@ impl SqlToRel<'_, S> { full: bool, sql_table_name: ObjectName, ) -> Result { - // Figure out the where clause - let where_clause = object_name_to_qualifier( - &sql_table_name, - self.options.enable_ident_normalization, - )?; - - let Some(columns_table_ref) = self.resolve_info_table("columns") else { - return plan_err!( - "SHOW COLUMNS is not supported unless information_schema is enabled" - ); - }; + let table_ref = self.object_name_to_table_reference(sql_table_name)?; // Do a table lookup to verify the table exists - let table_ref = self.object_name_to_table_reference(sql_table_name)?; - let _ = self.context_provider.get_table_source(table_ref)?; + let _ = self.context_provider.get_table_source(table_ref.clone())?; // Treat both FULL and EXTENDED as the same - let select_list = if full || extended { - "*" + let detail = if full || extended { + ColumnsDetail::Full } else { - "table_catalog, table_schema, table_name, column_name, data_type, is_nullable" + ColumnsDetail::Basic }; - let query = - format!("SELECT {select_list} FROM {columns_table_ref} WHERE {where_clause}"); - - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 + LogicalPlanBuilder::show_columns(table_ref, detail)?.build() } - /// Rewrite `SHOW FUNCTIONS` to another SQL query - /// The query is based on the `information_schema.routines` and `information_schema.parameters` tables + /// `SHOW FUNCTIONS [LIKE ]`. /// /// The output columns: /// - function_name: The name of function /// - return_type: The return type of the function /// - parameters: The name of parameters (ordered by the ordinal position) /// - parameter_types: The type of parameters (ordered by the ordinal position) + /// - function_type: SCALAR / AGGREGATE / WINDOW / TABLE /// - description: The description of the function (the description defined in the document) /// - syntax_example: The syntax_example of the function (the syntax_example defined in the document) fn show_functions_to_plan( &self, filter: Option, ) -> Result { - let where_clause = if let Some(filter) = filter { - match filter { - ShowStatementFilter::Like(like) => { - format!("WHERE p.function_name like '{like}'") - } - _ => return plan_err!("Unsupported SHOW FUNCTIONS filter"), - } - } else { - "".to_string() + let filter = match filter { + Some(ShowStatementFilter::Like(like)) => Some(like), + Some(_) => return plan_err!("Unsupported SHOW FUNCTIONS filter"), + None => None, }; - // Scalar / aggregate / window functions are resolved by joining - // parameters (IN rows aggregated per OUT row) with routines. - // Table functions (UDTFs) don't have parameter rows, so they are - // sourced directly from routines via a UNION branch. Restricting - // the JOIN to non-TABLE routines prevents same-named scalar+UDTF - // pairs (e.g. `generate_series`) from cross-joining. - let where_clause = where_clause.replace("p.function_name", "sc.function_name"); - let query = format!( - r#" -SELECT DISTINCT - sc.function_name, - sc.return_type, - sc.parameters, - sc.parameter_types, - sc.function_type, - sc.description, - sc.syntax_example -FROM ( - SELECT - p.function_name, - p.return_type, - p.parameters, - p.parameter_types, - r.function_type function_type, - r.description description, - r.syntax_example syntax_example - FROM ( - SELECT - o.specific_name function_name, - o.data_type return_type, - array_agg(i.parameter_name ORDER BY i.ordinal_position ASC) parameters, - array_agg(i.data_type ORDER BY i.ordinal_position ASC) parameter_types - FROM ( - SELECT - specific_catalog, - specific_schema, - specific_name, - ordinal_position, - parameter_name, - data_type, - rid - FROM - information_schema.parameters - WHERE - parameter_mode = 'OUT' - ) o - LEFT JOIN - ( - SELECT - specific_catalog, - specific_schema, - specific_name, - ordinal_position, - parameter_name, - data_type, - rid - FROM - information_schema.parameters - WHERE - parameter_mode = 'IN' - ) i - ON i.specific_catalog = o.specific_catalog - AND i.specific_schema = o.specific_schema - AND i.specific_name = o.specific_name - AND i.rid = o.rid - GROUP BY 1, 2, o.rid - ) as p - JOIN information_schema.routines r - ON p.function_name = r.routine_name - AND r.function_type <> 'TABLE' - - UNION ALL - - SELECT - routine_name function_name, - data_type return_type, - array_agg(NULL) FILTER (WHERE FALSE) parameters, - array_agg(NULL) FILTER (WHERE FALSE) parameter_types, - function_type, - description, - syntax_example - FROM information_schema.routines - WHERE function_type = 'TABLE' - GROUP BY routine_name, data_type, function_type, description, syntax_example -) sc -{where_clause} - "# - ); - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 + LogicalPlanBuilder::show_functions(filter)?.build() } fn show_create_table_to_plan( &self, sql_table_name: ObjectName, ) -> Result { - let Some(tables_table_ref) = self.resolve_info_table("views") else { - return plan_err!( - "SHOW CREATE TABLE is not supported unless information_schema is enabled" - ); - }; - - // Figure out the where clause - let where_clause = object_name_to_qualifier( - &sql_table_name, - self.options.enable_ident_normalization, - )?; - - // Do a table lookup to verify the table exists let table_ref = self.object_name_to_table_reference(sql_table_name)?; - let _ = self.context_provider.get_table_source(table_ref)?; - - let query = format!( - "SELECT table_catalog, table_schema, table_name, definition FROM {tables_table_ref} WHERE {where_clause}" - ); - let mut rewrite = DFParser::parse_sql(&query)?; - assert_eq!(rewrite.len(), 1); - self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1 - } - - /// Return true if there is a table provider available for "schema.table" - fn resolve_info_table(&self, table: &str) -> Option { - let table_reference = if let Some(system_catalog) = - &self.context_provider.options().catalog.system_catalog - { - TableReference::Full { - catalog: system_catalog.deref().into(), - schema: "information_schema".into(), - table: table.into(), - } - } else { - TableReference::Partial { - schema: "information_schema".into(), - table: table.into(), - } - }; + // Do a table lookup to verify the table exists + let _ = self.context_provider.get_table_source(table_ref.clone())?; - if self - .context_provider - .get_table_source(table_reference.clone()) - .is_ok() - { - Some(table_reference) - } else { - None - } + LogicalPlanBuilder::show_create_table(table_ref)?.build() } fn validate_transaction_kind( diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index 83ced616e0993..d177bfbbb45af 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -204,7 +204,7 @@ impl Unparser<'_> { | LogicalPlan::Analyze(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) + | LogicalPlan::Show(_) | LogicalPlan::RecursiveQuery(_) | LogicalPlan::Unnest(_) => not_impl_err!("Unsupported plan: {plan:?}"), } diff --git a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs index a8a18ed3fd290..7a3c2ffe4e4be 100644 --- a/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs +++ b/datafusion/sqllogictest/src/engines/datafusion_substrait_roundtrip_engine/runner.rs @@ -173,8 +173,8 @@ async fn run_query_substrait_round_trip( | LogicalPlan::Explain(_) | LogicalPlan::Dml(_) | LogicalPlan::Copy(_) - | LogicalPlan::DescribeTable(_) - | LogicalPlan::Statement(_) => logical_plan, + | LogicalPlan::Show(_) + | LogicalPlan::Statement(_) => logical_plan(), // For any other plan, convert to Substrait logical_plan => { let plan = to_substrait_plan(logical_plan, &state)?; diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index e9d7e861fbb5c..fe84367617530 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -23,8 +23,12 @@ SELECT * from information_schema.tables statement error DataFusion error: Error during planning: table 'datafusion.information_schema.schemata' not found SELECT * from information_schema.schemata -statement error DataFusion error: Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled -show all +# SHOW resolves its content directly (like DESCRIBE/SHOW COLUMNS), so it +# works regardless of whether information_schema is enabled. +query TT +SHOW datafusion.catalog.information_schema +---- +datafusion.catalog.information_schema false # Turn it on @@ -656,17 +660,22 @@ datafusion information_schema views VIEW query TT EXPLAIN SHOW TABLES ---- -logical_plan TableScan: information_schema.tables projection=[table_catalog, table_schema, table_name, table_type] +logical_plan ShowTables physical_plan StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, table_type] # information_schema_show_tables_no_information_schema +# +# SHOW TABLES reuses the information_schema.tables machinery directly, so it +# works regardless of whether information_schema is enabled; it just won't +# list the information_schema tables themselves when disabled. statement ok set datafusion.catalog.information_schema = false; -statement error Error during planning: SHOW TABLES is not supported unless information_schema is enabled +query TTTT rowsort SHOW TABLES +---- statement ok set datafusion.catalog.information_schema = true; @@ -704,12 +713,17 @@ SHOW EXTENDED COLUMNS FROM t; datafusion public t i 0 NULL NO Int32 NULL NULL 32 2 NULL NULL NULL # information_schema_show_columns_no_information_schema +# +# SHOW COLUMNS resolves the table directly (like DESCRIBE), so it works +# regardless of whether information_schema is enabled. statement ok set datafusion.catalog.information_schema = false; -statement error Error during planning: SHOW COLUMNS is not supported unless information_schema is enabled +query TTTTTT SHOW COLUMNS FROM t +---- +datafusion public t i Int32 NO statement ok set datafusion.catalog.information_schema = true; @@ -740,17 +754,26 @@ SHOW SOMETHING_UNKNOWN; statement ok DROP TABLE t; -# show_unsupported_when_information_schema_off +# show_variable_when_information_schema_off +# +# SHOW resolves its content directly, so it still works (and still rejects +# unknown variable names) regardless of whether information_schema is +# enabled. statement ok set datafusion.catalog.information_schema = false; -statement error Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled +statement error DataFusion error: Error during planning: 'something' is not a variable which can be viewed with 'SHOW' SHOW SOMETHING -statement error Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled +statement error DataFusion error: Error during planning: 'something' is not a variable which can be viewed with 'SHOW' SHOW SOMETHING VERBOSE +query TT +SHOW datafusion.catalog.information_schema +---- +datafusion.catalog.information_schema false + statement ok set datafusion.catalog.information_schema = true; diff --git a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt index d4048a8b6b327..73f04a6a16d76 100644 --- a/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt +++ b/datafusion/sqllogictest/test_files/information_schema_multiple_catalogs.slt @@ -19,8 +19,12 @@ statement error DataFusion error: Error during planning: table 'datafusion.information_schema.tables' not found SELECT * from information_schema.tables -statement error DataFusion error: Error during planning: SHOW \[VARIABLE\] is not supported unless information_schema is enabled -show all +# SHOW resolves its content directly (like DESCRIBE/SHOW COLUMNS), so it +# works regardless of whether information_schema is enabled. +query TT +SHOW datafusion.catalog.information_schema +---- +datafusion.catalog.information_schema false # Turn it on @@ -254,22 +258,18 @@ my_catalog my_schema t1 column1 my_catalog my_schema t2 column1 my_other_catalog my_other_schema t3 column1 +# There are no other views besides the information schema ones so the result set should be empty query TTT rowsort SELECT table_catalog, table_schema, table_name FROM my_catalog.information_schema.views WHERE table_schema <> 'information_schema'; ---- -my_catalog my_schema t1 -my_catalog my_schema t2 query TTT rowsort SELECT table_catalog, table_schema, table_name FROM system.information_schema.views WHERE table_schema <> 'information_schema'; ---- -my_catalog my_schema t1 -my_catalog my_schema t2 -my_other_catalog my_other_schema t3 # SHOW statements use the system catalog's information schema query TTTT rowsort diff --git a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs index 15f59919a2a95..2c4057c656caa 100644 --- a/datafusion/substrait/src/logical_plan/producer/rel/mod.rs +++ b/datafusion/substrait/src/logical_plan/producer/rel/mod.rs @@ -70,9 +70,7 @@ pub fn to_substrait_rel( LogicalPlan::Dml(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::Ddl(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::Copy(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, - LogicalPlan::DescribeTable(plan) => { - not_impl_err!("Unsupported plan type: {plan:?}")? - } + LogicalPlan::Show(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::Unnest(plan) => not_impl_err!("Unsupported plan type: {plan:?}")?, LogicalPlan::RecursiveQuery(plan) => { not_impl_err!("Unsupported plan type: {plan:?}")? From 92d8abfc1abb4d348a5eb73aa8bc7854ce484b2c Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Fri, 23 Oct 2026 15:03:30 +0200 Subject: [PATCH 16/16] Documentation typo --- datafusion/sql/src/statement.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index bd82210b8828f..ca5535dd5cee0 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -2205,7 +2205,7 @@ impl SqlToRel<'_, S> { let name = if variable == "all" { None } else { - // Whether this is actually a known variable is checked later at execution timne. + // No need to validate if the variable exists here; that is handled at execution time. Some(variable) };