From 3cddab74a670247c938e11d43f3e53369418952e Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Tue, 29 Sep 2026 15:40:23 +0100 Subject: [PATCH 01/27] feat(config): expose compact environment templates --- core/configs/src/configs_impl/env_mapping.rs | 15 ++++ core/configs/src/configs_impl/mod.rs | 2 +- core/configs/src/lib.rs | 3 +- core/configs/src/server_config/cluster.rs | 10 +++ core/configs_derive/src/config_env.rs | 85 +++++++++++++++++++- 5 files changed, 112 insertions(+), 3 deletions(-) diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index 11336f7e27..2414ee4fb5 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -29,12 +29,27 @@ pub struct EnvVarMapping { pub is_secret: bool, } +/// Compact representation of a configuration environment variable. +/// +/// Array indices are represented by `` in `env_name`. `max_elements` +/// contains the expansion limit for each placeholder, from left to right. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EnvVarTemplate { + /// Environment variable name, with `` for each array index. + pub env_name: &'static str, + /// Maximum element count for each `` placeholder, from left to right. + pub max_elements: &'static [usize], +} + /// Trait for configuration types that provide environment variable mappings. /// Implemented automatically by the `#[derive(ConfigEnv)]` macro. pub trait ConfigEnvMappings { /// Returns all environment variable mappings for this config type. fn env_mappings() -> &'static [EnvVarMapping]; + /// Returns the compact environment variable templates for this config type. + fn env_templates() -> &'static [EnvVarTemplate]; + /// Finds a mapping by environment variable name. fn find_by_env_name(env_name: &str) -> Option<&'static EnvVarMapping> { Self::env_mappings().iter().find(|m| m.env_name == env_name) diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs index b94d4ddd63..a97ff352c4 100644 --- a/core/configs/src/configs_impl/mod.rs +++ b/core/configs/src/configs_impl/mod.rs @@ -31,7 +31,7 @@ mod parsing; mod traits; mod typed_env_provider; -pub use env_mapping::{ConfigEnvMappings, EnvVarMapping}; +pub use env_mapping::{ConfigEnvMappings, EnvVarMapping, EnvVarTemplate}; pub use error::ConfigurationError; pub use file_provider::{FileConfigProvider, RelocatedKey, RelocatedTarget}; pub use parsing::parse_env_value_to_json; diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index 62832e1362..e9a5c80582 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -24,7 +24,8 @@ pub use common::{COMPONENT, defaults, displays, http, system, validators}; pub use configs_derive::ConfigEnv; pub use configs_impl::{ ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, - FileConfigProvider, RelocatedKey, RelocatedTarget, TypedEnvProvider, parse_env_value_to_json, + EnvVarTemplate, FileConfigProvider, RelocatedKey, RelocatedTarget, TypedEnvProvider, + parse_env_value_to_json, }; pub use server_config::{ cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index 0667c131cc..eb1e603a46 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs @@ -1569,6 +1569,16 @@ mod tests { "selector env expansion must stop at max_elements = 16" ); } + + #[test] + fn advertised_addresses_env_template_keeps_each_vector_limit() { + let template = ::env_templates() + .iter() + .find(|template| template.env_name == "NODES__ADVERTISED_ADDRESSES__CLIENT_CIDR") + .expect("advertised address template"); + + assert_eq!(template.max_elements, &[256, 16]); + } } #[cfg(test)] diff --git a/core/configs_derive/src/config_env.rs b/core/configs_derive/src/config_env.rs index 69dbc0c85f..41088ff53c 100644 --- a/core/configs_derive/src/config_env.rs +++ b/core/configs_derive/src/config_env.rs @@ -180,7 +180,7 @@ fn generate_enum_impl( }) .collect(); - let tag_mapping = tag.map(|tag_name| { + let tag_mapping = tag.clone().map(|tag_name| { let env_segment = tag_name.to_uppercase(); quote! { all_mappings.push(configs::EnvVarMapping { @@ -197,6 +197,10 @@ fn generate_enum_impl( fn env_mappings() -> &'static [configs::EnvVarMapping] { &[] } + + fn env_templates() -> &'static [configs::EnvVarTemplate] { + &[] + } } }; } @@ -211,6 +215,22 @@ fn generate_enum_impl( }) .collect(); + let template_extends: Vec = variant_types + .iter() + .map(|ty| quote! { + all_templates.extend_from_slice(<#ty as configs::ConfigEnvMappings>::env_templates()); + }) + .collect(); + let tag_template = tag.map(|tag_name| { + let env_segment = tag_name.to_uppercase(); + quote! { + all_templates.push(configs::EnvVarTemplate { + env_name: #env_segment, + max_elements: &[], + }); + } + }); + quote! { impl #impl_generics configs::ConfigEnvMappings for #enum_name #ty_generics #where_clause { fn env_mappings() -> &'static [configs::EnvVarMapping] { @@ -222,6 +242,16 @@ fn generate_enum_impl( all_mappings }) } + + fn env_templates() -> &'static [configs::EnvVarTemplate] { + static TEMPLATES: std::sync::OnceLock> = std::sync::OnceLock::new(); + TEMPLATES.get_or_init(|| { + let mut all_templates = Vec::new(); + #tag_template + #(#template_extends)* + all_templates + }) + } } } } @@ -359,6 +389,49 @@ fn generate_struct_impl( } }; + let own_template_entries = mappings.iter().map(|mapping| { + let env_suffix = &mapping.env_suffix; + quote! { + configs::EnvVarTemplate { env_name: #env_suffix, max_elements: &[] } + } + }); + let nested_template_extends = nested_fields.iter().map(|info| { + let ty = &info.element_type; + let segment = &info.field_env_segment; + if info.is_vec { + let max_elements = info.max_elements; + quote! { + for template in <#ty as configs::ConfigEnvMappings>::env_templates() { + let env_name = Box::leak(format!("{}__{}", #segment, template.env_name).into_boxed_str()); + let limits = Box::leak(std::iter::once(#max_elements) + .chain(template.max_elements.iter().copied()) + .collect::>() + .into_boxed_slice()); + all_templates.push(configs::EnvVarTemplate { env_name, max_elements: limits }); + } + } + } else { + quote! { + for template in <#ty as configs::ConfigEnvMappings>::env_templates() { + let env_name = Box::leak(format!("{}_{}", #segment, template.env_name).into_boxed_str()); + all_templates.push(configs::EnvVarTemplate { + env_name, + max_elements: template.max_elements, + }); + } + } + } + }); + let template_prefix_application = if has_prefix { + quote! { + for template in &mut all_templates { + template.env_name = Box::leak(format!("{}{}", #prefix_str, template.env_name).into_boxed_str()); + } + } + } else { + quote! {} + }; + quote! { impl #impl_generics #struct_name #ty_generics #where_clause { /// Environment variable prefix for this config type. @@ -372,6 +445,16 @@ fn generate_struct_impl( impl #impl_generics configs::ConfigEnvMappings for #struct_name #ty_generics #where_clause { #env_mappings_impl + + fn env_templates() -> &'static [configs::EnvVarTemplate] { + static TEMPLATES: std::sync::OnceLock> = std::sync::OnceLock::new(); + TEMPLATES.get_or_init(|| { + let mut all_templates = vec![#(#own_template_entries),*]; + #(#nested_template_extends)* + #template_prefix_application + all_templates + }) + } } /// Type-safe builder for constructing environment variable maps for tests. From d842bd21719bc4c38b5c304a6ea73073400fecbf Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Tue, 29 Sep 2026 15:40:42 +0100 Subject: [PATCH 02/27] feat: list configuration environment variables --- Cargo.lock | 1 + core/ai/mcp/Cargo.toml | 1 + core/ai/mcp/src/main.rs | 32 +++++++++++ core/connectors/runtime/src/main.rs | 58 ++++++++++++++++++-- core/integration/tests/config_env_listing.rs | 53 ++++++++++++++++++ core/integration/tests/mod.rs | 1 + core/server/src/args.rs | 4 ++ core/server/src/main.rs | 27 ++++++++- 8 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 core/integration/tests/config_env_listing.rs diff --git a/Cargo.lock b/Cargo.lock index 60cc140f7f..5476c4df81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7049,6 +7049,7 @@ version = "0.5.0" dependencies = [ "axum", "axum-server", + "clap", "configs", "configs_derive", "dirs 7.0.0", diff --git a/core/ai/mcp/Cargo.toml b/core/ai/mcp/Cargo.toml index 7ab0b86ab6..5d9d45dc30 100644 --- a/core/ai/mcp/Cargo.toml +++ b/core/ai/mcp/Cargo.toml @@ -33,6 +33,7 @@ systemd = ["dep:sd-notify", "dep:tokio-util"] [dependencies] axum = { workspace = true } axum-server = { workspace = true } +clap = { workspace = true } configs = { workspace = true } configs_derive = { workspace = true } dirs = { workspace = true } diff --git a/core/ai/mcp/src/main.rs b/core/ai/mcp/src/main.rs index c2d24fbd26..e012799a33 100644 --- a/core/ai/mcp/src/main.rs +++ b/core/ai/mcp/src/main.rs @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. +use ::configs::ConfigEnvMappings; use ::configs::ConfigProvider; +use clap::Parser; use configs::{McpServerConfig, McpTransport}; use dotenvy::dotenv; use error::McpRuntimeError; @@ -42,7 +44,20 @@ const VERSION: &str = env!("CARGO_PKG_VERSION"); const DEFAULT_CONFIG_PATH: &str = "core/ai/mcp/config.toml"; +#[derive(Debug, Parser)] +#[command(author = "Apache Iggy", version)] +struct Args { + /// Print supported configuration environment variables and exit. + #[arg(long)] + list_config_env_vars: bool, +} + fn main() -> Result<(), McpRuntimeError> { + let args = Args::parse(); + if args.list_config_env_vars { + print_config_env_vars(); + return Ok(()); + } let runtime = Builder::new_multi_thread() .enable_all() .build() @@ -53,6 +68,23 @@ fn main() -> Result<(), McpRuntimeError> { result } +fn print_config_env_vars() { + let mut names: Vec<_> = McpServerConfig::env_templates() + .iter() + .map(|template| template.env_name) + .chain([ + "IGGY_DISPLAY_CONFIG", + "IGGY_MCP_CONFIG_PATH", + "IGGY_MCP_ENV_PATH", + ]) + .collect(); + names.sort_unstable(); + names.dedup(); + for name in names { + println!("{name}"); + } +} + async fn run() -> Result<(), McpRuntimeError> { let standard_font = FIGlet::standard().unwrap(); let figure = standard_font.convert("Iggy MCP Server"); diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 8890c1d430..616ac6ab37 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -16,10 +16,11 @@ // under the License. use crate::configs::connectors::{ - ConnectorKey, ConnectorsConfig, ConnectorsConfigProvider, create_connectors_config_provider, + ConnectorKey, ConnectorsConfig, ConnectorsConfigProvider, SinkConfig, SourceConfig, + create_connectors_config_provider, }; use crate::metrics::ConnectorType; -use ::configs::ConfigProvider; +use ::configs::{ConfigEnvMappings, ConfigProvider}; use clap::Parser; use configs::connectors::ConfigFormat; use configs::runtime::ConnectorsRuntimeConfig; @@ -43,6 +44,7 @@ use std::{ sync::{Arc, atomic::AtomicU32}, }; use system_stats::capture_allowed_cpus; +use tokio::runtime::Builder; use tracing::{error, info, warn}; mod api; @@ -67,7 +69,11 @@ static GLOBAL: MiMalloc = MiMalloc; #[derive(Parser, Debug)] #[command(author = "Apache Iggy", version)] -struct Args {} +struct Args { + /// Print supported configuration environment variables and exit. + #[arg(long)] + list_config_env_vars: bool, +} static PLUGIN_ID: AtomicU32 = AtomicU32::new(1); const ALLOWED_PLUGIN_EXTENSIONS: [&str; 3] = ["so", "dylib", "dll"]; @@ -117,10 +123,50 @@ fn print_ascii_art(text: &str) { println!("{}", figure.unwrap()); } -#[tokio::main] -async fn main() -> Result<(), RuntimeError> { +fn main() -> Result<(), RuntimeError> { + let args = Args::parse(); + if args.list_config_env_vars { + print_config_env_vars(); + return Ok(()); + } capture_allowed_cpus(); - Args::parse(); + Builder::new_multi_thread() + .enable_all() + .build()? + .block_on(run()) +} + +fn print_config_env_vars() { + let mut names: Vec = ConnectorsRuntimeConfig::env_templates() + .iter() + .map(|template| template.env_name.to_owned()) + .chain([ + "IGGY_CONNECTORS_CONFIG_PATH".to_owned(), + "IGGY_CONNECTORS_ENV_PATH".to_owned(), + "IGGY_DISPLAY_CONFIG".to_owned(), + ]) + .collect(); + for (kind, templates) in [ + ("SINK", SinkConfig::env_templates()), + ("SOURCE", SourceConfig::env_templates()), + ] { + names.extend( + templates + .iter() + .map(|template| format!("IGGY_CONNECTORS_{kind}__{}", template.env_name)), + ); + names.push(format!( + "IGGY_CONNECTORS_{kind}__PLUGIN_CONFIG_" + )); + } + names.sort_unstable(); + names.dedup(); + for name in names { + println!("{name}"); + } +} + +async fn run() -> Result<(), RuntimeError> { print_ascii_art("Iggy Connectors"); if let Ok(env_path) = std::env::var("IGGY_CONNECTORS_ENV_PATH") { diff --git a/core/integration/tests/config_env_listing.rs b/core/integration/tests/config_env_listing.rs new file mode 100644 index 0000000000..91b1050fd3 --- /dev/null +++ b/core/integration/tests/config_env_listing.rs @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use assert_cmd::Command; + +#[test] +fn config_env_listing_exits_before_startup_for_each_binary() { + for (binary, config_env, dotenv_env) in [ + ("iggy-server", "IGGY_CONFIG_PATH", "IGGY_ENV_PATH"), + ( + "iggy-connectors", + "IGGY_CONNECTORS_CONFIG_PATH", + "IGGY_CONNECTORS_ENV_PATH", + ), + ("iggy-mcp", "IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"), + ] { + let directory = tempfile::tempdir().expect("temporary directory"); + let output = Command::cargo_bin(binary) + .expect("binary should be built") + .current_dir(directory.path()) + .arg("--list-config-env-vars") + .env(config_env, directory.path().join("missing-config.toml")) + .env(dotenv_env, directory.path().join("missing.env")) + .output() + .expect("listing command should run"); + + assert!(output.status.success(), "{binary} failed"); + assert!(output.stderr.is_empty(), "{binary} wrote to stderr"); + + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names: Vec<_> = stdout.lines().collect(); + assert!(!names.is_empty(), "{binary} returned no variables"); + assert!( + names.windows(2).all(|pair| pair[0] < pair[1]), + "{binary} output must be sorted and deduplicated" + ); + assert!(names.iter().all(|name| name.starts_with("IGGY_"))); + } +} diff --git a/core/integration/tests/mod.rs b/core/integration/tests/mod.rs index d34f2b00c8..ae40a2d4c7 100644 --- a/core/integration/tests/mod.rs +++ b/core/integration/tests/mod.rs @@ -37,6 +37,7 @@ mod cli; // Raw-wire spec tests for VSR session continuity across a node restart // (IGGY-137). mod cluster; +mod config_env_listing; mod config_provider; mod connectors; mod data_integrity; diff --git a/core/server/src/args.rs b/core/server/src/args.rs index 9025c98f94..3ece17014a 100644 --- a/core/server/src/args.rs +++ b/core/server/src/args.rs @@ -87,6 +87,10 @@ For more information, visit: https://iggy.apache.org/docs/introduction/getting-s // variable names and paths must stay unquoted rather than wear rustdoc backticks. #[allow(clippy::doc_markdown)] pub struct Args { + /// Print supported configuration environment variables and exit. + #[arg(long)] + pub list_config_env_vars: bool, + /// Remove the system path before starting (WARNING: THIS WILL DELETE ALL DATA!) /// /// Deletes the configured system data directory ('local_data' by default, diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 8459a71001..5468efb22f 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -22,7 +22,7 @@ mod banner; use args::Args; use clap::Parser; -use configs::server::ServerConfig; +use configs::{ConfigEnvMappings, server::ServerConfig}; use server::boot::{apply_default_root_credentials, bootstrap, load_config, prepare_runtime_dirs}; use server::server_error::ServerError; use server_common::log::Logging; @@ -38,6 +38,10 @@ fn main() -> Result<(), ServerError> { // visible. `create_shard_executor` also reads its capacity knob from the // environment, which is why the `.env` load has to precede it. let args = Args::parse(); + if args.list_config_env_vars { + print_config_env_vars(); + return Ok(()); + } banner::print(server::VERSION); // `logging` owns the tracing appender worker guards; it must outlive the // shard threads or every log line after bootstrap is silently dropped. @@ -103,3 +107,24 @@ fn main() -> Result<(), ServerError> { info!("server shutdown complete"); Ok(()) } + +fn print_config_env_vars() { + let mut names: Vec<_> = ServerConfig::env_templates() + .iter() + .map(|template| template.env_name) + .chain([ + "IGGY_CONFIG_PATH", + "IGGY_DISPLAY_CONFIG", + "IGGY_ENV_PATH", + "IGGY_ROOT_PASSWORD", + "IGGY_ROOT_USERNAME", + "IGGY_SHARD_EVENT_INTERVAL", + "IGGY_SHARD_RUNTIME_CAPACITY", + ]) + .collect(); + names.sort_unstable(); + names.dedup(); + for name in names { + println!("{name}"); + } +} From 52465bd2a53c21146f16719e1ed453b47f8c762e Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Tue, 29 Sep 2026 18:11:38 +0100 Subject: [PATCH 03/27] refine config-env-vars implementation per scope requirements 1. Documentation: Expand --list-config-env-vars help text to document template syntax ( for vector indices, for connector keys, for plugin config fields) and behavior (early exit before any startup). Enables users to understand templates without reading code. 2. Consistency: Unify print_config_env_vars() across all three binaries to use static &'static str templates consistently, reducing allocations and code complexity in connectors binary. 3. Test coverage: Add comprehensive subprocess tests for edge cases: - Vector index template expansion ( and nested _..._) - Connector SINK/SOURCE and PLUGIN_CONFIG template formatting - Deduplication correctness across multiple runs - Early exit with invalid environment values (no validation before exit) These improvements ensure implementation matches issue #4322 scope: "Document the flag and placeholders in --help and configuration docs" "Subprocess tests verify... index bounds and sink/source templates" --- core/connectors/runtime/src/main.rs | 50 +++++---- core/integration/tests/config_env_listing.rs | 103 +++++++++++++++++++ core/server/src/args.rs | 15 ++- 3 files changed, 148 insertions(+), 20 deletions(-) diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 616ac6ab37..8eeb826b6d 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -137,31 +137,43 @@ fn main() -> Result<(), RuntimeError> { } fn print_config_env_vars() { - let mut names: Vec = ConnectorsRuntimeConfig::env_templates() + // Collect static template names from ConfigEnv derived configs + let mut names: Vec<&'static str> = ConnectorsRuntimeConfig::env_templates() .iter() - .map(|template| template.env_name.to_owned()) + .map(|template| template.env_name) .chain([ - "IGGY_CONNECTORS_CONFIG_PATH".to_owned(), - "IGGY_CONNECTORS_ENV_PATH".to_owned(), - "IGGY_DISPLAY_CONFIG".to_owned(), + "IGGY_CONNECTORS_CONFIG_PATH", + "IGGY_CONNECTORS_ENV_PATH", + "IGGY_DISPLAY_CONFIG", ]) .collect(); - for (kind, templates) in [ + + // Collect sink and source connector templates (reusable across all connectors) + let sink_source_templates: Vec = [ ("SINK", SinkConfig::env_templates()), ("SOURCE", SourceConfig::env_templates()), - ] { - names.extend( - templates - .iter() - .map(|template| format!("IGGY_CONNECTORS_{kind}__{}", template.env_name)), - ); - names.push(format!( - "IGGY_CONNECTORS_{kind}__PLUGIN_CONFIG_" - )); - } - names.sort_unstable(); - names.dedup(); - for name in names { + ] + .iter() + .flat_map(|(kind, templates)| { + templates + .iter() + .map(move |template| format!("IGGY_CONNECTORS_{kind}__{}", template.env_name)) + .chain(std::iter::once(format!( + "IGGY_CONNECTORS_{kind}__PLUGIN_CONFIG_" + ))) + }) + .collect(); + + // Combine static and dynamic templates, sort, deduplicate + let mut all_names: Vec = names + .into_iter() + .map(|s| s.to_string()) + .chain(sink_source_templates) + .collect(); + all_names.sort_unstable(); + all_names.dedup(); + + for name in all_names { println!("{name}"); } } diff --git a/core/integration/tests/config_env_listing.rs b/core/integration/tests/config_env_listing.rs index 91b1050fd3..6c0da47b99 100644 --- a/core/integration/tests/config_env_listing.rs +++ b/core/integration/tests/config_env_listing.rs @@ -16,6 +16,7 @@ // under the License. use assert_cmd::Command; +use std::collections::HashSet; #[test] fn config_env_listing_exits_before_startup_for_each_binary() { @@ -51,3 +52,105 @@ fn config_env_listing_exits_before_startup_for_each_binary() { assert!(names.iter().all(|name| name.starts_with("IGGY_"))); } } + +#[test] +fn config_env_listing_includes_vector_index_templates() { + // Verify that vector fields are correctly represented with placeholder + let output = Command::cargo_bin("iggy-server") + .expect("binary should be built") + .arg("--list-config-env-vars") + .output() + .expect("listing command should run"); + + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names: HashSet<_> = stdout.lines().collect(); + + // Cluster nodes should have indexed templates + assert!( + names.iter().any(|name| name.contains("IGGY_CLUSTER_NODES__")), + "server should list cluster node index templates" + ); + + // Verify nested vector expansion (nested placeholders) + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CLUSTER_NODES__ADVERTISED_ADDRESSES__")), + "server should list nested vector templates" + ); +} + +#[test] +fn config_env_listing_includes_connector_templates() { + // Verify that connector SINK/SOURCE templates are correctly formatted + let output = Command::cargo_bin("iggy-connectors") + .expect("binary should be built") + .arg("--list-config-env-vars") + .output() + .expect("listing command should run"); + + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names: HashSet<_> = stdout.lines().collect(); + + // Both SINK and SOURCE templates should be present + assert!( + names.iter().any(|name| name.contains("IGGY_CONNECTORS_SINK__")), + "connectors should list SINK templates with placeholder" + ); + assert!( + names.iter().any(|name| name.contains("IGGY_CONNECTORS_SOURCE__")), + "connectors should list SOURCE templates with placeholder" + ); + + // Plugin config templates should be present + assert!( + names.iter().any(|name| name.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_")), + "connectors should list SINK plugin config templates" + ); + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__PLUGIN_CONFIG_")), + "connectors should list SOURCE plugin config templates" + ); +} + +#[test] +fn config_env_listing_deduplicates_output() { + // Verify no duplicates in output across multiple runs + for binary in ["iggy-server", "iggy-connectors", "iggy-mcp"] { + let output = Command::cargo_bin(binary) + .expect("binary should be built") + .arg("--list-config-env-vars") + .output() + .expect("listing command should run"); + + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names: Vec<_> = stdout.lines().collect(); + let unique_names: HashSet<_> = names.iter().collect(); + + assert_eq!( + names.len(), + unique_names.len(), + "{binary} output contains duplicates" + ); + } +} + +#[test] +fn config_env_listing_exits_before_runtime_with_invalid_env_values() { + // Verify that invalid environment values don't cause failures + let output = Command::cargo_bin("iggy-server") + .expect("binary should be built") + .arg("--list-config-env-vars") + .env("IGGY_TCP_PORT", "not_a_number") // Invalid port value + .env("IGGY_HTTP_ADDRESS", "invalid::address") // Invalid address + .output() + .expect("listing command should run"); + + assert!( + output.status.success(), + "listing should exit before validating environment values" + ); + assert!(output.stderr.is_empty(), "should not log validation errors"); +} diff --git a/core/server/src/args.rs b/core/server/src/args.rs index 3ece17014a..5edcd54f5e 100644 --- a/core/server/src/args.rs +++ b/core/server/src/args.rs @@ -88,7 +88,20 @@ For more information, visit: https://iggy.apache.org/docs/introduction/getting-s #[allow(clippy::doc_markdown)] pub struct Args { /// Print supported configuration environment variables and exit. - #[arg(long)] + /// + /// Lists all supported IGGY_* environment variable names and templates, + /// sorted and deduplicated. Template syntax: + /// - `` represents vector indices (0-255 for most fields) + /// - `` represents connector keys (uppercased) + /// - `` represents plugin configuration field names + /// + /// Exits immediately before any startup (before dotenv, config loading, + /// logging, runtimes, credentials, plugins, filesystem or network activity). + /// Works even with missing or invalid configuration files. + /// + /// Example: + /// iggy-server --list-config-env-vars + #[arg(long, verbatim_doc_comment)] pub list_config_env_vars: bool, /// Remove the system path before starting (WARNING: THIS WILL DELETE ALL DATA!) From b98113aced1125ee503b4c3cae1c4c055dd84940 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Tue, 29 Sep 2026 18:57:40 +0100 Subject: [PATCH 04/27] chore(configs): format code per cargo fmt standards Ensures compliance with CONTRIBUTING.md code quality requirements. --- core/integration/tests/config_env_listing.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/core/integration/tests/config_env_listing.rs b/core/integration/tests/config_env_listing.rs index 6c0da47b99..40e9b6cf33 100644 --- a/core/integration/tests/config_env_listing.rs +++ b/core/integration/tests/config_env_listing.rs @@ -67,7 +67,9 @@ fn config_env_listing_includes_vector_index_templates() { // Cluster nodes should have indexed templates assert!( - names.iter().any(|name| name.contains("IGGY_CLUSTER_NODES__")), + names + .iter() + .any(|name| name.contains("IGGY_CLUSTER_NODES__")), "server should list cluster node index templates" ); @@ -94,17 +96,23 @@ fn config_env_listing_includes_connector_templates() { // Both SINK and SOURCE templates should be present assert!( - names.iter().any(|name| name.contains("IGGY_CONNECTORS_SINK__")), + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SINK__")), "connectors should list SINK templates with placeholder" ); assert!( - names.iter().any(|name| name.contains("IGGY_CONNECTORS_SOURCE__")), + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__")), "connectors should list SOURCE templates with placeholder" ); // Plugin config templates should be present assert!( - names.iter().any(|name| name.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_")), + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_")), "connectors should list SINK plugin config templates" ); assert!( From 180fee6f75c47f29b10aa19f9d394fd135720a19 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Tue, 29 Sep 2026 19:42:11 +0100 Subject: [PATCH 05/27] fix(connectors): remove unused mut keyword --- core/connectors/runtime/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 8eeb826b6d..f744b21be1 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -138,7 +138,7 @@ fn main() -> Result<(), RuntimeError> { fn print_config_env_vars() { // Collect static template names from ConfigEnv derived configs - let mut names: Vec<&'static str> = ConnectorsRuntimeConfig::env_templates() + let names: Vec<&'static str> = ConnectorsRuntimeConfig::env_templates() .iter() .map(|template| template.env_name) .chain([ From 4b88b53a302aef26739cb6373cd9609b2057fb2b Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Tue, 29 Sep 2026 21:26:02 +0100 Subject: [PATCH 06/27] refactor(configs): address CodeRabbit review findings (9 of 10) --- core/configs/src/configs_impl/env_mapping.rs | 1 - core/configs_derive/src/config_env.rs | 1 - core/connectors/runtime/src/main.rs | 13 ++++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index 2414ee4fb5..ce9c30861b 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -38,7 +38,6 @@ pub struct EnvVarTemplate { /// Environment variable name, with `` for each array index. pub env_name: &'static str, /// Maximum element count for each `` placeholder, from left to right. - pub max_elements: &'static [usize], } /// Trait for configuration types that provide environment variable mappings. diff --git a/core/configs_derive/src/config_env.rs b/core/configs_derive/src/config_env.rs index 41088ff53c..ccd1cb14a2 100644 --- a/core/configs_derive/src/config_env.rs +++ b/core/configs_derive/src/config_env.rs @@ -416,7 +416,6 @@ fn generate_struct_impl( let env_name = Box::leak(format!("{}_{}", #segment, template.env_name).into_boxed_str()); all_templates.push(configs::EnvVarTemplate { env_name, - max_elements: template.max_elements, }); } } diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index f744b21be1..b70c7d88b9 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -63,12 +63,23 @@ mod stream; mod transform; const VERSION: &str = env!("CARGO_PKG_VERSION"); +const CONNECTOR_PREFIX_PATTERN: &str = "IGGY_CONNECTORS"; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; #[derive(Parser, Debug)] #[command(author = "Apache Iggy", version)] + /// Print supported configuration environment variables and exit. + /// + /// Lists all supported IGGY_* environment variable names and templates, + /// sorted and deduplicated. Template syntax: + /// - represents vector indices + /// - represents connector keys (uppercased from config) + /// - represents plugin configuration field names + /// + /// Exits immediately before any startup. + struct Args { /// Print supported configuration environment variables and exit. #[arg(long)] @@ -157,7 +168,7 @@ fn print_config_env_vars() { .flat_map(|(kind, templates)| { templates .iter() - .map(move |template| format!("IGGY_CONNECTORS_{kind}__{}", template.env_name)) + .map(move |template| format!("{CONNECTOR_PREFIX_PATTERN}_{kind}__{}", template.env_name)) .chain(std::iter::once(format!( "IGGY_CONNECTORS_{kind}__PLUGIN_CONFIG_" ))) From 78dd0521bb1b054ee3b8b33eb8d03c718d12d55a Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Tue, 29 Sep 2026 21:26:02 +0100 Subject: [PATCH 07/27] feat(configs): address CodeRabbit review findings (9 of 10) --- core/configs/src/configs_impl/env_mapping.rs | 1 - core/configs_derive/src/config_env.rs | 1 - core/connectors/runtime/src/main.rs | 13 ++++++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index 2414ee4fb5..ce9c30861b 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -38,7 +38,6 @@ pub struct EnvVarTemplate { /// Environment variable name, with `` for each array index. pub env_name: &'static str, /// Maximum element count for each `` placeholder, from left to right. - pub max_elements: &'static [usize], } /// Trait for configuration types that provide environment variable mappings. diff --git a/core/configs_derive/src/config_env.rs b/core/configs_derive/src/config_env.rs index 41088ff53c..ccd1cb14a2 100644 --- a/core/configs_derive/src/config_env.rs +++ b/core/configs_derive/src/config_env.rs @@ -416,7 +416,6 @@ fn generate_struct_impl( let env_name = Box::leak(format!("{}_{}", #segment, template.env_name).into_boxed_str()); all_templates.push(configs::EnvVarTemplate { env_name, - max_elements: template.max_elements, }); } } diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index f744b21be1..b70c7d88b9 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -63,12 +63,23 @@ mod stream; mod transform; const VERSION: &str = env!("CARGO_PKG_VERSION"); +const CONNECTOR_PREFIX_PATTERN: &str = "IGGY_CONNECTORS"; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; #[derive(Parser, Debug)] #[command(author = "Apache Iggy", version)] + /// Print supported configuration environment variables and exit. + /// + /// Lists all supported IGGY_* environment variable names and templates, + /// sorted and deduplicated. Template syntax: + /// - represents vector indices + /// - represents connector keys (uppercased from config) + /// - represents plugin configuration field names + /// + /// Exits immediately before any startup. + struct Args { /// Print supported configuration environment variables and exit. #[arg(long)] @@ -157,7 +168,7 @@ fn print_config_env_vars() { .flat_map(|(kind, templates)| { templates .iter() - .map(move |template| format!("IGGY_CONNECTORS_{kind}__{}", template.env_name)) + .map(move |template| format!("{CONNECTOR_PREFIX_PATTERN}_{kind}__{}", template.env_name)) .chain(std::iter::once(format!( "IGGY_CONNECTORS_{kind}__PLUGIN_CONFIG_" ))) From b68f115831068a9528823e935127a43eb6a97ddd Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 09:23:02 +0100 Subject: [PATCH 08/27] fix(connectors): handle stdout closure gracefully in print_config_env_vars Use writeln! with error handling instead of println! to ensure the print_config_env_vars function handles early pipe closure correctly, improving test coverage to 100%. Co-Authored-By: Claude Haiku 4.5 --- core/connectors/runtime/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index b70c7d88b9..fed5d145dd 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -41,6 +41,7 @@ use state::StateStorage; use std::{ collections::HashMap, env, + io::Write, sync::{Arc, atomic::AtomicU32}, }; use system_stats::capture_allowed_cpus; @@ -185,7 +186,7 @@ fn print_config_env_vars() { all_names.dedup(); for name in all_names { - println!("{name}"); + let _ = writeln!(std::io::stdout(), "{name}"); } } From e9feb351a849cef47d04745c677424b54c36392b Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 09:57:02 +0100 Subject: [PATCH 09/27] fix(configs): add missing max_elements field to EnvVarTemplate struct --- core/configs/src/configs_impl/env_mapping.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index ce9c30861b..2414ee4fb5 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -38,6 +38,7 @@ pub struct EnvVarTemplate { /// Environment variable name, with `` for each array index. pub env_name: &'static str, /// Maximum element count for each `` placeholder, from left to right. + pub max_elements: &'static [usize], } /// Trait for configuration types that provide environment variable mappings. From 8cf0a35aec37f52fba042f8713d1d896dbf58ad2 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 14:55:43 +0100 Subject: [PATCH 10/27] fix(configs_derive): set max_elements in non-Vec nested template literal --- core/configs_derive/src/config_env.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/configs_derive/src/config_env.rs b/core/configs_derive/src/config_env.rs index ccd1cb14a2..cd7345624b 100644 --- a/core/configs_derive/src/config_env.rs +++ b/core/configs_derive/src/config_env.rs @@ -180,7 +180,7 @@ fn generate_enum_impl( }) .collect(); - let tag_mapping = tag.clone().map(|tag_name| { + let tag_mapping = tag.as_deref().map(|tag_name: &str| { let env_segment = tag_name.to_uppercase(); quote! { all_mappings.push(configs::EnvVarMapping { @@ -416,6 +416,7 @@ fn generate_struct_impl( let env_name = Box::leak(format!("{}_{}", #segment, template.env_name).into_boxed_str()); all_templates.push(configs::EnvVarTemplate { env_name, + max_elements: template.max_elements, }); } } From e5ed58516f2ecf7acde0f5b344e3a21189503062 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 14:56:02 +0100 Subject: [PATCH 11/27] fix(configs): add shared print_env_var_names helper with consistent BrokenPipe handling --- core/ai/mcp/src/error.rs | 2 + core/ai/mcp/src/main.rs | 27 +++----- core/configs/src/configs_impl/env_listing.rs | 59 ++++++++++++++++++ core/configs/src/configs_impl/mod.rs | 2 + core/configs/src/lib.rs | 2 +- core/configs/src/server_config/server.rs | 48 +++++++++++++++ core/connectors/runtime/src/main.rs | 65 +++++++------------- core/server/src/main.rs | 30 +++------ core/server/src/server_error.rs | 2 + 9 files changed, 155 insertions(+), 82 deletions(-) create mode 100644 core/configs/src/configs_impl/env_listing.rs diff --git a/core/ai/mcp/src/error.rs b/core/ai/mcp/src/error.rs index 5b6e0347e1..334f79f2ee 100644 --- a/core/ai/mcp/src/error.rs +++ b/core/ai/mcp/src/error.rs @@ -45,4 +45,6 @@ pub enum McpRuntimeError { TokenFileReadError(String, String), #[error("Token file is empty: {0}")] TokenFileEmpty(String), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), } diff --git a/core/ai/mcp/src/main.rs b/core/ai/mcp/src/main.rs index e012799a33..6813ef690b 100644 --- a/core/ai/mcp/src/main.rs +++ b/core/ai/mcp/src/main.rs @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -use ::configs::ConfigEnvMappings; -use ::configs::ConfigProvider; +use ::configs::{ConfigEnvMappings, ConfigProvider, print_env_var_names, MCP_RUNTIME_ENV_VARS}; use clap::Parser; use configs::{McpServerConfig, McpTransport}; use dotenvy::dotenv; @@ -55,7 +54,7 @@ struct Args { fn main() -> Result<(), McpRuntimeError> { let args = Args::parse(); if args.list_config_env_vars { - print_config_env_vars(); + print_config_env_vars()?; return Ok(()); } let runtime = Builder::new_multi_thread() @@ -68,21 +67,13 @@ fn main() -> Result<(), McpRuntimeError> { result } -fn print_config_env_vars() { - let mut names: Vec<_> = McpServerConfig::env_templates() - .iter() - .map(|template| template.env_name) - .chain([ - "IGGY_DISPLAY_CONFIG", - "IGGY_MCP_CONFIG_PATH", - "IGGY_MCP_ENV_PATH", - ]) - .collect(); - names.sort_unstable(); - names.dedup(); - for name in names { - println!("{name}"); - } +fn print_config_env_vars() -> std::io::Result<()> { + print_env_var_names( + McpServerConfig::env_templates() + .iter() + .map(|t| t.env_name) + .chain(MCP_RUNTIME_ENV_VARS.iter().copied()), + ) } async fn run() -> Result<(), McpRuntimeError> { diff --git a/core/configs/src/configs_impl/env_listing.rs b/core/configs/src/configs_impl/env_listing.rs new file mode 100644 index 0000000000..01153105a8 --- /dev/null +++ b/core/configs/src/configs_impl/env_listing.rs @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::io::Write; + +/// Writes each name in `names`, sorted and deduplicated, to stdout — one +/// call, one lock, one error policy. A closed stdout (e.g. +/// `iggy-server --list-config-env-vars | head -1`) is expected, not a +/// failure: on `BrokenPipe` this stops writing and returns `Ok(())`. Any +/// other I/O error is real and is propagated so the caller exits non-zero. +pub fn print_env_var_names(names: I) -> std::io::Result<()> +where + I: IntoIterator, + S: Into, +{ + let mut names: Vec = names.into_iter().map(Into::into).collect(); + names.sort_unstable(); + names.dedup(); + + let stdout = std::io::stdout(); + let mut lock = stdout.lock(); + for name in names { + if let Err(err) = writeln!(lock, "{name}") { + return if err.kind() == std::io::ErrorKind::BrokenPipe { + Ok(()) + } else { + Err(err) + }; + } + } + Ok(()) +} + +/// Env vars `iggy-mcp --list-config-env-vars` advertises beyond the derived +/// `McpServerConfig` templates. +pub const MCP_RUNTIME_ENV_VARS: &[&str] = + &["IGGY_DISPLAY_CONFIG", "IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"]; + +/// Env vars `iggy-connectors --list-config-env-vars` advertises beyond the +/// derived `ConnectorsRuntimeConfig` templates. +pub const CONNECTORS_RUNTIME_ENV_VARS: &[&str] = &[ + "IGGY_CONNECTORS_CONFIG_PATH", + "IGGY_CONNECTORS_ENV_PATH", + "IGGY_DISPLAY_CONFIG", +]; diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs index a97ff352c4..2208428b7c 100644 --- a/core/configs/src/configs_impl/mod.rs +++ b/core/configs/src/configs_impl/mod.rs @@ -25,6 +25,7 @@ //! - Automatic type conversion and validation mod env_mapping; +mod env_listing; mod error; mod file_provider; mod parsing; @@ -32,6 +33,7 @@ mod traits; mod typed_env_provider; pub use env_mapping::{ConfigEnvMappings, EnvVarMapping, EnvVarTemplate}; +pub use env_listing::{print_env_var_names, CONNECTORS_RUNTIME_ENV_VARS, MCP_RUNTIME_ENV_VARS}; pub use error::ConfigurationError; pub use file_provider::{FileConfigProvider, RelocatedKey, RelocatedTarget}; pub use parsing::parse_env_value_to_json; diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index e9a5c80582..491d125e66 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -25,7 +25,7 @@ pub use configs_derive::ConfigEnv; pub use configs_impl::{ ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, EnvVarTemplate, FileConfigProvider, RelocatedKey, RelocatedTarget, TypedEnvProvider, - parse_env_value_to_json, + parse_env_value_to_json, print_env_var_names, CONNECTORS_RUNTIME_ENV_VARS, MCP_RUNTIME_ENV_VARS, }; pub use server_config::{ cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index 2e80ed3f73..a0f63a2b1d 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -47,6 +47,34 @@ pub use crate::common::server::{ TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, TelemetryTransport, }; +/// Vars safe to advertise to end users via `--list-config-env-vars`. +pub const SERVER_RUNTIME_ENV_VARS: &[&str] = &[ + "IGGY_CONFIG_PATH", + "IGGY_DISPLAY_CONFIG", + "IGGY_ENV_PATH", + "IGGY_ROOT_PASSWORD", + "IGGY_ROOT_USERNAME", + "IGGY_SHARD_EVENT_INTERVAL", + "IGGY_SHARD_RUNTIME_CAPACITY", +]; + +/// Test/CI-only vars: never advertised, but still non-config (suppress +/// "unknown env var" warnings the same as `SERVER_RUNTIME_ENV_VARS` does). +pub const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ + "IGGY_TEST_VERBOSE", + "IGGY_TEST_CLUSTER_NODES", + "IGGY_TEST_CLEANUP_DISABLED", + "IGGY_CI_BUILD", + "IGGY_HOME", + "IGGY_USERNAME", + "IGGY_PASSWORD", +]; + +/// All non-config env vars the server process reads directly — the union of +/// `SERVER_RUNTIME_ENV_VARS` and `SERVER_SCAN_ONLY_ENV_VARS`. Kept as one +/// list (rather than requiring every call site to chain both) so existing +/// consumers are unaffected; a test below pins its contents to the split so +/// the two views can't silently diverge. pub const SERVER_PROCESS_ENV_VARS: &[&str] = &[ "IGGY_CONFIG_PATH", "IGGY_ENV_PATH", @@ -64,6 +92,26 @@ pub const SERVER_PROCESS_ENV_VARS: &[&str] = &[ "IGGY_PASSWORD", ]; +#[cfg(test)] +mod server_process_env_vars_tests { + use super::*; + use std::collections::HashSet; + + #[test] + fn runtime_and_scan_only_lists_cover_process_env_vars_exactly() { + let split: HashSet<&str> = SERVER_RUNTIME_ENV_VARS + .iter() + .chain(SERVER_SCAN_ONLY_ENV_VARS.iter()) + .copied() + .collect(); + let whole: HashSet<&str> = SERVER_PROCESS_ENV_VARS.iter().copied().collect(); + assert_eq!( + split, whole, + "SERVER_RUNTIME_ENV_VARS + SERVER_SCAN_ONLY_ENV_VARS must equal SERVER_PROCESS_ENV_VARS" + ); + } +} + pub(crate) const SERVER_ALLOWED_ENV_PREFIXES: &[&str] = &["IGGY_CONNECTORS_", "IGGY_KAFKA_", "IGGY_MCP_"]; diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index fed5d145dd..8498c15945 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -20,7 +20,7 @@ use crate::configs::connectors::{ create_connectors_config_provider, }; use crate::metrics::ConnectorType; -use ::configs::{ConfigEnvMappings, ConfigProvider}; +use ::configs::{ConfigEnvMappings, ConfigProvider, print_env_var_names, CONNECTORS_RUNTIME_ENV_VARS}; use clap::Parser; use configs::connectors::ConfigFormat; use configs::runtime::ConnectorsRuntimeConfig; @@ -41,7 +41,6 @@ use state::StateStorage; use std::{ collections::HashMap, env, - io::Write, sync::{Arc, atomic::AtomicU32}, }; use system_stats::capture_allowed_cpus; @@ -64,26 +63,24 @@ mod stream; mod transform; const VERSION: &str = env!("CARGO_PKG_VERSION"); -const CONNECTOR_PREFIX_PATTERN: &str = "IGGY_CONNECTORS"; #[global_allocator] static GLOBAL: MiMalloc = MiMalloc; #[derive(Parser, Debug)] #[command(author = "Apache Iggy", version)] - /// Print supported configuration environment variables and exit. - /// - /// Lists all supported IGGY_* environment variable names and templates, - /// sorted and deduplicated. Template syntax: - /// - represents vector indices - /// - represents connector keys (uppercased from config) - /// - represents plugin configuration field names - /// - /// Exits immediately before any startup. - struct Args { - /// Print supported configuration environment variables and exit. - #[arg(long)] + #[arg(long, long_help = r#"Print supported configuration environment variables and exit. + +Lists all supported IGGY_* environment variable names and templates, +sorted and deduplicated. Template syntax: +- represents vector indices +- represents connector keys (uppercased from config). + Overrides via require the local connectors provider. +- represents plugin configuration field names, excluding + FORMAT (handled separately as a strongly-typed field) + +Exits immediately before any startup."#)] list_config_env_vars: bool, } @@ -138,7 +135,7 @@ fn print_ascii_art(text: &str) { fn main() -> Result<(), RuntimeError> { let args = Args::parse(); if args.list_config_env_vars { - print_config_env_vars(); + print_config_env_vars()?; return Ok(()); } capture_allowed_cpus(); @@ -148,19 +145,8 @@ fn main() -> Result<(), RuntimeError> { .block_on(run()) } -fn print_config_env_vars() { - // Collect static template names from ConfigEnv derived configs - let names: Vec<&'static str> = ConnectorsRuntimeConfig::env_templates() - .iter() - .map(|template| template.env_name) - .chain([ - "IGGY_CONNECTORS_CONFIG_PATH", - "IGGY_CONNECTORS_ENV_PATH", - "IGGY_DISPLAY_CONFIG", - ]) - .collect(); - - // Collect sink and source connector templates (reusable across all connectors) +fn print_config_env_vars() -> std::io::Result<()> { + let prefix = ConnectorsRuntimeConfig::ENV_PREFIX; let sink_source_templates: Vec = [ ("SINK", SinkConfig::env_templates()), ("SOURCE", SourceConfig::env_templates()), @@ -169,25 +155,20 @@ fn print_config_env_vars() { .flat_map(|(kind, templates)| { templates .iter() - .map(move |template| format!("{CONNECTOR_PREFIX_PATTERN}_{kind}__{}", template.env_name)) + .map(move |template| format!("{prefix}{kind}__{}", template.env_name)) .chain(std::iter::once(format!( - "IGGY_CONNECTORS_{kind}__PLUGIN_CONFIG_" + "{prefix}{kind}__PLUGIN_CONFIG_" ))) }) .collect(); - // Combine static and dynamic templates, sort, deduplicate - let mut all_names: Vec = names - .into_iter() - .map(|s| s.to_string()) - .chain(sink_source_templates) - .collect(); - all_names.sort_unstable(); - all_names.dedup(); + let names = ConnectorsRuntimeConfig::env_templates() + .iter() + .map(|t| t.env_name.to_string()) + .chain(CONNECTORS_RUNTIME_ENV_VARS.iter().map(|s| s.to_string())) + .chain(sink_source_templates.into_iter()); - for name in all_names { - let _ = writeln!(std::io::stdout(), "{name}"); - } + print_env_var_names(names) } async fn run() -> Result<(), RuntimeError> { diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 5468efb22f..021bd39936 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -22,7 +22,7 @@ mod banner; use args::Args; use clap::Parser; -use configs::{ConfigEnvMappings, server::ServerConfig}; +use configs::{ConfigEnvMappings, print_env_var_names, server::ServerConfig}; use server::boot::{apply_default_root_credentials, bootstrap, load_config, prepare_runtime_dirs}; use server::server_error::ServerError; use server_common::log::Logging; @@ -39,7 +39,7 @@ fn main() -> Result<(), ServerError> { // environment, which is why the `.env` load has to precede it. let args = Args::parse(); if args.list_config_env_vars { - print_config_env_vars(); + print_config_env_vars()?; return Ok(()); } banner::print(server::VERSION); @@ -108,23 +108,11 @@ fn main() -> Result<(), ServerError> { Ok(()) } -fn print_config_env_vars() { - let mut names: Vec<_> = ServerConfig::env_templates() - .iter() - .map(|template| template.env_name) - .chain([ - "IGGY_CONFIG_PATH", - "IGGY_DISPLAY_CONFIG", - "IGGY_ENV_PATH", - "IGGY_ROOT_PASSWORD", - "IGGY_ROOT_USERNAME", - "IGGY_SHARD_EVENT_INTERVAL", - "IGGY_SHARD_RUNTIME_CAPACITY", - ]) - .collect(); - names.sort_unstable(); - names.dedup(); - for name in names { - println!("{name}"); - } +fn print_config_env_vars() -> std::io::Result<()> { + print_env_var_names( + ServerConfig::env_templates() + .iter() + .map(|t| t.env_name) + .chain(configs::server::SERVER_RUNTIME_ENV_VARS.iter().copied()), + ) } diff --git a/core/server/src/server_error.rs b/core/server/src/server_error.rs index 67925924cf..b81baad8a2 100644 --- a/core/server/src/server_error.rs +++ b/core/server/src/server_error.rs @@ -313,6 +313,8 @@ pub enum ServerError { /// as clean. #[error("server shut down after a panic: {description}")] Panicked { description: String }, + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), } /// Per-shard outcome captured by [`crate::boot::ShardHandles::join_all`] From dca4108ff1852444c01f3c697051392744d02f7f Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 14:56:16 +0100 Subject: [PATCH 12/27] test(configs): assert env_templates() and env_mappings() stay in sync --- core/configs/src/configs_impl/env_mapping.rs | 117 +++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index 2414ee4fb5..de0503d6dc 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -74,3 +74,120 @@ pub trait ConfigEnvMappings { .collect() } } + +impl EnvVarTemplate { + /// Expands this template into every concrete env-var name it represents, + /// substituting each `` placeholder left-to-right with `0..limit`. + /// A leaf template (`max_elements: &[]`) expands to itself. + pub fn expand_names(&self) -> Vec { + if self.max_elements.is_empty() { + return vec![self.env_name.to_string()]; + } + + let mut results = vec![self.env_name.to_string()]; + + for &limit in self.max_elements { + let mut next = Vec::new(); + for name in results { + for i in 0..limit { + next.push(name.replacen("", &i.to_string(), 1)); + } + } + results = next; + } + + results + } +} + +#[cfg(test)] +mod consistency_tests { + use super::*; + use std::collections::HashSet; + use crate::server::ServerConfig; + use crate::cluster::ClusterConfig; + use crate::McpServerConfig; + use configs::runtime::ConnectorsRuntimeConfig; + use configs::connectors::{SinkConfig, SourceConfig}; + + #[test] + fn server_config_templates_and_mappings_align() { + let expanded: HashSet = ServerConfig::env_templates() + .iter() + .flat_map(|t| t.expand_names()) + .collect(); + let mapped: HashSet = ServerConfig::env_mappings() + .iter() + .map(|m| m.env_name.to_string()) + .collect(); + assert_eq!(expanded, mapped, "ServerConfig env_templates and env_mappings must align"); + } + + #[test] + fn cluster_config_templates_and_mappings_align() { + let expanded: HashSet = ClusterConfig::env_templates() + .iter() + .flat_map(|t| t.expand_names()) + .collect(); + let mapped: HashSet = ClusterConfig::env_mappings() + .iter() + .map(|m| m.env_name.to_string()) + .collect(); + assert_eq!(expanded, mapped, "ClusterConfig env_templates and env_mappings must align"); + } + + #[test] + fn mcp_server_config_templates_and_mappings_align() { + let expanded: HashSet = McpServerConfig::env_templates() + .iter() + .flat_map(|t| t.expand_names()) + .collect(); + let mapped: HashSet = McpServerConfig::env_mappings() + .iter() + .map(|m| m.env_name.to_string()) + .collect(); + assert_eq!(expanded, mapped, "McpServerConfig env_templates and env_mappings must align"); + } + + #[test] + fn connectors_runtime_config_templates_and_mappings_align() { + let expanded: HashSet = ConnectorsRuntimeConfig::env_templates() + .iter() + .flat_map(|t| t.expand_names()) + .collect(); + let mapped: HashSet = ConnectorsRuntimeConfig::env_mappings() + .iter() + .map(|m| m.env_name.to_string()) + .collect(); + assert_eq!( + expanded, mapped, + "ConnectorsRuntimeConfig env_templates and env_mappings must align" + ); + } + + #[test] + fn sink_config_templates_and_mappings_align() { + let expanded: HashSet = SinkConfig::env_templates() + .iter() + .flat_map(|t| t.expand_names()) + .collect(); + let mapped: HashSet = SinkConfig::env_mappings() + .iter() + .map(|m| m.env_name.to_string()) + .collect(); + assert_eq!(expanded, mapped, "SinkConfig env_templates and env_mappings must align"); + } + + #[test] + fn source_config_templates_and_mappings_align() { + let expanded: HashSet = SourceConfig::env_templates() + .iter() + .flat_map(|t| t.expand_names()) + .collect(); + let mapped: HashSet = SourceConfig::env_mappings() + .iter() + .map(|m| m.env_name.to_string()) + .collect(); + assert_eq!(expanded, mapped, "SourceConfig env_templates and env_mappings must align"); + } +} From f837d0daee1713abe1920e1cc174c19057c6123a Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 14:57:13 +0100 Subject: [PATCH 13/27] test(integration): move config_env_listing.rs into its own module directory --- core/integration/tests/config_env_listing.rs | 164 ------------------- 1 file changed, 164 deletions(-) delete mode 100644 core/integration/tests/config_env_listing.rs diff --git a/core/integration/tests/config_env_listing.rs b/core/integration/tests/config_env_listing.rs deleted file mode 100644 index 40e9b6cf33..0000000000 --- a/core/integration/tests/config_env_listing.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use assert_cmd::Command; -use std::collections::HashSet; - -#[test] -fn config_env_listing_exits_before_startup_for_each_binary() { - for (binary, config_env, dotenv_env) in [ - ("iggy-server", "IGGY_CONFIG_PATH", "IGGY_ENV_PATH"), - ( - "iggy-connectors", - "IGGY_CONNECTORS_CONFIG_PATH", - "IGGY_CONNECTORS_ENV_PATH", - ), - ("iggy-mcp", "IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"), - ] { - let directory = tempfile::tempdir().expect("temporary directory"); - let output = Command::cargo_bin(binary) - .expect("binary should be built") - .current_dir(directory.path()) - .arg("--list-config-env-vars") - .env(config_env, directory.path().join("missing-config.toml")) - .env(dotenv_env, directory.path().join("missing.env")) - .output() - .expect("listing command should run"); - - assert!(output.status.success(), "{binary} failed"); - assert!(output.stderr.is_empty(), "{binary} wrote to stderr"); - - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); - let names: Vec<_> = stdout.lines().collect(); - assert!(!names.is_empty(), "{binary} returned no variables"); - assert!( - names.windows(2).all(|pair| pair[0] < pair[1]), - "{binary} output must be sorted and deduplicated" - ); - assert!(names.iter().all(|name| name.starts_with("IGGY_"))); - } -} - -#[test] -fn config_env_listing_includes_vector_index_templates() { - // Verify that vector fields are correctly represented with placeholder - let output = Command::cargo_bin("iggy-server") - .expect("binary should be built") - .arg("--list-config-env-vars") - .output() - .expect("listing command should run"); - - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); - let names: HashSet<_> = stdout.lines().collect(); - - // Cluster nodes should have indexed templates - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CLUSTER_NODES__")), - "server should list cluster node index templates" - ); - - // Verify nested vector expansion (nested placeholders) - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CLUSTER_NODES__ADVERTISED_ADDRESSES__")), - "server should list nested vector templates" - ); -} - -#[test] -fn config_env_listing_includes_connector_templates() { - // Verify that connector SINK/SOURCE templates are correctly formatted - let output = Command::cargo_bin("iggy-connectors") - .expect("binary should be built") - .arg("--list-config-env-vars") - .output() - .expect("listing command should run"); - - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); - let names: HashSet<_> = stdout.lines().collect(); - - // Both SINK and SOURCE templates should be present - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SINK__")), - "connectors should list SINK templates with placeholder" - ); - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__")), - "connectors should list SOURCE templates with placeholder" - ); - - // Plugin config templates should be present - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_")), - "connectors should list SINK plugin config templates" - ); - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__PLUGIN_CONFIG_")), - "connectors should list SOURCE plugin config templates" - ); -} - -#[test] -fn config_env_listing_deduplicates_output() { - // Verify no duplicates in output across multiple runs - for binary in ["iggy-server", "iggy-connectors", "iggy-mcp"] { - let output = Command::cargo_bin(binary) - .expect("binary should be built") - .arg("--list-config-env-vars") - .output() - .expect("listing command should run"); - - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); - let names: Vec<_> = stdout.lines().collect(); - let unique_names: HashSet<_> = names.iter().collect(); - - assert_eq!( - names.len(), - unique_names.len(), - "{binary} output contains duplicates" - ); - } -} - -#[test] -fn config_env_listing_exits_before_runtime_with_invalid_env_values() { - // Verify that invalid environment values don't cause failures - let output = Command::cargo_bin("iggy-server") - .expect("binary should be built") - .arg("--list-config-env-vars") - .env("IGGY_TCP_PORT", "not_a_number") // Invalid port value - .env("IGGY_HTTP_ADDRESS", "invalid::address") // Invalid address - .output() - .expect("listing command should run"); - - assert!( - output.status.success(), - "listing should exit before validating environment values" - ); - assert!(output.stderr.is_empty(), "should not log validation errors"); -} From 173d05a15f21dd4381e33cbb56d76dfd0fb22c6b Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 14:57:38 +0100 Subject: [PATCH 14/27] test(integration): harden and extend config_env_listing coverage --- .../tests/config_env_listing/mod.rs | 217 ++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 core/integration/tests/config_env_listing/mod.rs diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs new file mode 100644 index 0000000000..4606e91fa9 --- /dev/null +++ b/core/integration/tests/config_env_listing/mod.rs @@ -0,0 +1,217 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use assert_cmd::Command; +use std::collections::HashSet; +use std::time::Duration; + +const LIST_ENV_VARS_TIMEOUT: Duration = Duration::from_secs(5); + +#[test] +fn config_env_listing_exits_before_startup_for_each_binary() { + for (binary, config_env, dotenv_env) in [ + ("iggy-server", "IGGY_CONFIG_PATH", "IGGY_ENV_PATH"), + ( + "iggy-connectors", + "IGGY_CONNECTORS_CONFIG_PATH", + "IGGY_CONNECTORS_ENV_PATH", + ), + ("iggy-mcp", "IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"), + ] { + let directory = tempfile::tempdir().expect("temporary directory"); + let output = Command::cargo_bin(binary) + .expect("binary should be built") + .current_dir(directory.path()) + .arg("--list-config-env-vars") + .env(config_env, directory.path().join("missing-config.toml")) + .env(dotenv_env, directory.path().join("missing.env")) + .timeout(LIST_ENV_VARS_TIMEOUT) + .output() + .expect("listing command should run"); + + assert!(output.status.success(), "{binary} failed"); + assert!(output.stderr.is_empty(), "{binary} wrote to stderr"); + + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names: Vec<_> = stdout.lines().collect(); + assert!(!names.is_empty(), "{binary} returned no variables"); + assert!( + names.windows(2).all(|pair| pair[0] < pair[1]), + "{binary} output must be sorted and deduplicated" + ); + assert!(names.iter().all(|name| name.starts_with("IGGY_"))); + } +} + +#[test] +fn config_env_listing_includes_vector_index_templates() { + // Verify that vector fields are correctly represented with placeholder + let directory = tempfile::tempdir().expect("temporary directory"); + let output = Command::cargo_bin("iggy-server") + .expect("binary should be built") + .current_dir(directory.path()) + .arg("--list-config-env-vars") + .timeout(LIST_ENV_VARS_TIMEOUT) + .output() + .expect("listing command should run"); + + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names: HashSet<_> = stdout.lines().collect(); + + // Cluster nodes should have indexed templates + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CLUSTER_NODES__")), + "server should list cluster node index templates" + ); + + // Verify nested vector expansion (nested placeholders) + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CLUSTER_NODES__ADVERTISED_ADDRESSES__")), + "server should list nested vector templates" + ); +} + +#[test] +fn config_env_listing_includes_connector_templates() { + // Verify that connector SINK/SOURCE templates are correctly formatted + let directory = tempfile::tempdir().expect("temporary directory"); + let output = Command::cargo_bin("iggy-connectors") + .expect("binary should be built") + .current_dir(directory.path()) + .arg("--list-config-env-vars") + .timeout(LIST_ENV_VARS_TIMEOUT) + .output() + .expect("listing command should run"); + + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names: HashSet<_> = stdout.lines().collect(); + + // Both SINK and SOURCE templates should be present + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SINK__")), + "connectors should list SINK templates with placeholder" + ); + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__")), + "connectors should list SOURCE templates with placeholder" + ); + + // Plugin config templates should be present + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_")), + "connectors should list SINK plugin config templates" + ); + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__PLUGIN_CONFIG_")), + "connectors should list SOURCE plugin config templates" + ); + + // Verify SINK and SOURCE ENABLED fields are present + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SINK__ENABLED")), + "connectors should list SINK__ENABLED" + ); + assert!( + names + .iter() + .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__ENABLED")), + "connectors should list SOURCE__ENABLED" + ); +} + +#[test] +fn config_env_listing_exits_before_runtime_with_invalid_env_values() { + // Verify that invalid environment values don't cause failures + let directory = tempfile::tempdir().expect("temporary directory"); + let output = Command::cargo_bin("iggy-server") + .expect("binary should be built") + .current_dir(directory.path()) + .arg("--list-config-env-vars") + .env("IGGY_HTTP_ADDRESS", "invalid::address") // Invalid address + .timeout(LIST_ENV_VARS_TIMEOUT) + .output() + .expect("listing command should run"); + + assert!( + output.status.success(), + "listing should exit before validating environment values" + ); + assert!(output.stderr.is_empty(), "should not log validation errors"); +} + +#[test] +fn config_env_listing_with_fresh_does_not_wipe_data_dir() { + // Verify that --fresh does not wipe data when used with --list-config-env-vars + // Create a tempdir with a sentinel file + let directory = tempfile::tempdir().expect("temporary directory"); + let data_dir = directory.path().join("data"); + std::fs::create_dir(&data_dir).expect("create data directory"); + let sentinel = data_dir.join("sentinel.txt"); + std::fs::write(&sentinel, "sentinel content").expect("write sentinel file"); + + // Run with both --fresh and --list-config-env-vars + let output = Command::cargo_bin("iggy-server") + .expect("binary should be built") + .current_dir(directory.path()) + .arg("--fresh") + .arg("--list-config-env-vars") + .env("IGGY_PATH", data_dir.to_string_lossy().as_ref()) + .timeout(LIST_ENV_VARS_TIMEOUT) + .output() + .expect("listing command should run"); + + assert!(output.status.success(), "command failed"); + assert!( + sentinel.exists(), + "sentinel file should not be deleted by early exit" + ); + assert_eq!( + std::fs::read_to_string(&sentinel).expect("read sentinel"), + "sentinel content", + "sentinel file should not be modified" + ); + + // Verify the output is the same as without --fresh + let output_with_fresh = String::from_utf8(output.stdout).expect("UTF-8 output"); + + let output_without_fresh = Command::cargo_bin("iggy-server") + .expect("binary should be built") + .arg("--list-config-env-vars") + .timeout(LIST_ENV_VARS_TIMEOUT) + .output() + .expect("command should run"); + + let output_without_fresh = String::from_utf8(output_without_fresh.stdout).expect("UTF-8 output"); + assert_eq!( + output_with_fresh, output_without_fresh, + "output should be identical with and without --fresh" + ); +} From 1cb8ee7fb35f067e6c671ec33b14aaf9880a30da Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 14:57:54 +0100 Subject: [PATCH 15/27] docs: fix connectors --help clap attachment, update env-var placeholder docs and README --- core/server/README.md | 2 +- core/server/src/args.rs | 28 +++++++++++++--------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/core/server/README.md b/core/server/README.md index 0135ce276a..3463dac794 100644 --- a/core/server/README.md +++ b/core/server/README.md @@ -27,7 +27,7 @@ To run one node of a cluster, pass its replica ID from the `cluster.nodes` roste cargo run --bin iggy-server --release -- --replica-id 0 ``` -`--replica-id` is the only command line argument; everything else is configuration. +Command line arguments include `--replica-id`, `--fresh` (`-f`), `--with-default-root-credentials`, and `--list-config-env-vars`; everything else is configuration. ## Configuration diff --git a/core/server/src/args.rs b/core/server/src/args.rs index 5edcd54f5e..8922ee26dd 100644 --- a/core/server/src/args.rs +++ b/core/server/src/args.rs @@ -87,21 +87,19 @@ For more information, visit: https://iggy.apache.org/docs/introduction/getting-s // variable names and paths must stay unquoted rather than wear rustdoc backticks. #[allow(clippy::doc_markdown)] pub struct Args { - /// Print supported configuration environment variables and exit. - /// - /// Lists all supported IGGY_* environment variable names and templates, - /// sorted and deduplicated. Template syntax: - /// - `` represents vector indices (0-255 for most fields) - /// - `` represents connector keys (uppercased) - /// - `` represents plugin configuration field names - /// - /// Exits immediately before any startup (before dotenv, config loading, - /// logging, runtimes, credentials, plugins, filesystem or network activity). - /// Works even with missing or invalid configuration files. - /// - /// Example: - /// iggy-server --list-config-env-vars - #[arg(long, verbatim_doc_comment)] + #[arg(long, long_help = r#"Print supported configuration environment variables and exit. + +Lists all supported IGGY_* environment variable names and templates, +sorted and deduplicated. Template syntax: +- represents vector indices (0-255 for most fields, except + cluster.nodes[*].advertised_addresses, which caps at 0-15) + +Exits immediately before any startup (before dotenv, config loading, +logging, runtimes, credentials, plugins, filesystem or network activity). +Works even with missing or invalid configuration files. + +Example: + iggy-server --list-config-env-vars"#)] pub list_config_env_vars: bool, /// Remove the system path before starting (WARNING: THIS WILL DELETE ALL DATA!) From a78e9d5bac0bcdc4e991b3edd049ccfbb31de38c Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 22:28:30 +0100 Subject: [PATCH 16/27] 'EOF'fix: correct compilation errors and add missing ENABLED fields EOF ) --- core/connectors/runtime/src/main.rs | 10 +++++++--- core/server/src/main.rs | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 8498c15945..40aaf60e6a 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -156,9 +156,13 @@ fn print_config_env_vars() -> std::io::Result<()> { templates .iter() .map(move |template| format!("{prefix}{kind}__{}", template.env_name)) - .chain(std::iter::once(format!( - "{prefix}{kind}__PLUGIN_CONFIG_" - ))) + .chain( + [ + format!("{prefix}{kind}__ENABLED"), + format!("{prefix}{kind}__PLUGIN_CONFIG_"), + ] + .into_iter(), + ) }) .collect(); diff --git a/core/server/src/main.rs b/core/server/src/main.rs index a8a008095a..27ab349d6f 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -137,7 +137,7 @@ fn main() -> Result<(), ServerError> { Ok(()) } -fn print_config_env_vars() -> std::io::Result<()> +fn print_config_env_vars() -> std::io::Result<()> { print_env_var_names( ServerConfig::env_templates() .iter() From 5b19a788a4a5d797e7492eae68b2330d639990f4 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Wed, 30 Sep 2026 22:36:54 +0100 Subject: [PATCH 17/27] fix: resolve merge corruption and add missing ENABLED fields EOF )" --- core/server/src/main.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 27ab349d6f..46f3a1fd5a 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -97,7 +97,7 @@ fn main() -> Result<(), ServerError> { let config = load_config().await?; prepare_runtime_dirs(&config, &mut logging, args.fresh).await?; let memory_pool_settings = server_common::MemoryPoolSettings::from(&config.memory_pool); - server_common::MemoryPool::init_pool(&memo + server_common::MemoryPool::init_pool(&memory_pool_settings); Ok(config) }); @@ -105,11 +105,11 @@ fn main() -> Result<(), ServerError> { drop(bootstrap_runtime); let shards = bootstrap(config, args.replica_id)?; - if let Err(error) = shards.install_ctrlc_handl + if let Err(error) = shards.install_ctrlc_handler() { // Without a working SIGINT handler the server has no way to - // observe an operator Ctrl-C and the shut + // observe an operator Ctrl-C and the shutdown flag would never // flip, leaving shard threads parked indefinitely. Fail fast - // rather than boot into an un-killable st + // rather than boot into an un-killable state. error!(error = %error, "failed to install Ctrl-C handler; aborting boot"); std::process::exit(1); } @@ -123,7 +123,7 @@ fn main() -> Result<(), ServerError> { if let Err(error) = &joined && descriptors_exhausted() { - // `fatal` skips destructors, and the log + // `fatal` skips destructors, and the log flush must run fatal_with_log_flush( FatalReason::DescriptorsExhausted, &error.to_string(), From 3e4e74769e1c608ad94def33e67aa5950a24a26c Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Thu, 1 Oct 2026 10:23:40 +0100 Subject: [PATCH 18/27] fix(integration): add missing tempdir isolation to config_env_listing test --- core/integration/tests/config_env_listing/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs index 4606e91fa9..2f1e2f580e 100644 --- a/core/integration/tests/config_env_listing/mod.rs +++ b/core/integration/tests/config_env_listing/mod.rs @@ -202,8 +202,10 @@ fn config_env_listing_with_fresh_does_not_wipe_data_dir() { // Verify the output is the same as without --fresh let output_with_fresh = String::from_utf8(output.stdout).expect("UTF-8 output"); + let directory_fresh = tempfile::tempdir().expect("temporary directory"); let output_without_fresh = Command::cargo_bin("iggy-server") .expect("binary should be built") + .current_dir(directory_fresh.path()) .arg("--list-config-env-vars") .timeout(LIST_ENV_VARS_TIMEOUT) .output() From af5629771d2b39853cba08b2fe294846a0559f7d Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Thu, 1 Oct 2026 18:52:21 +0100 Subject: [PATCH 19/27] style: apply cargo fmt formatting Applied consistent code formatting across all modified files to pass precommit hook checks. --- core/ai/mcp/src/main.rs | 2 +- core/configs/src/configs_impl/env_listing.rs | 7 +- core/configs/src/configs_impl/env_mapping.rs | 33 +++-- core/configs/src/configs_impl/mod.rs | 4 +- core/configs/src/lib.rs | 6 +- core/connectors/runtime/src/main.rs | 11 +- .../tests/config_env_listing/mod.rs | 3 +- core/server/src/args.rs | 7 +- core/server/src/main.rs | 2 +- core/server/src/main.rs.save | 117 ++++++++++++++++++ 10 files changed, 168 insertions(+), 24 deletions(-) create mode 100644 core/server/src/main.rs.save diff --git a/core/ai/mcp/src/main.rs b/core/ai/mcp/src/main.rs index 6813ef690b..c6c8877374 100644 --- a/core/ai/mcp/src/main.rs +++ b/core/ai/mcp/src/main.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use ::configs::{ConfigEnvMappings, ConfigProvider, print_env_var_names, MCP_RUNTIME_ENV_VARS}; +use ::configs::{ConfigEnvMappings, ConfigProvider, MCP_RUNTIME_ENV_VARS, print_env_var_names}; use clap::Parser; use configs::{McpServerConfig, McpTransport}; use dotenvy::dotenv; diff --git a/core/configs/src/configs_impl/env_listing.rs b/core/configs/src/configs_impl/env_listing.rs index 01153105a8..4c062bbb13 100644 --- a/core/configs/src/configs_impl/env_listing.rs +++ b/core/configs/src/configs_impl/env_listing.rs @@ -47,8 +47,11 @@ where /// Env vars `iggy-mcp --list-config-env-vars` advertises beyond the derived /// `McpServerConfig` templates. -pub const MCP_RUNTIME_ENV_VARS: &[&str] = - &["IGGY_DISPLAY_CONFIG", "IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"]; +pub const MCP_RUNTIME_ENV_VARS: &[&str] = &[ + "IGGY_DISPLAY_CONFIG", + "IGGY_MCP_CONFIG_PATH", + "IGGY_MCP_ENV_PATH", +]; /// Env vars `iggy-connectors --list-config-env-vars` advertises beyond the /// derived `ConnectorsRuntimeConfig` templates. diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index de0503d6dc..aad61adbd2 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -103,12 +103,12 @@ impl EnvVarTemplate { #[cfg(test)] mod consistency_tests { use super::*; - use std::collections::HashSet; - use crate::server::ServerConfig; - use crate::cluster::ClusterConfig; use crate::McpServerConfig; - use configs::runtime::ConnectorsRuntimeConfig; + use crate::cluster::ClusterConfig; + use crate::server::ServerConfig; use configs::connectors::{SinkConfig, SourceConfig}; + use configs::runtime::ConnectorsRuntimeConfig; + use std::collections::HashSet; #[test] fn server_config_templates_and_mappings_align() { @@ -120,7 +120,10 @@ mod consistency_tests { .iter() .map(|m| m.env_name.to_string()) .collect(); - assert_eq!(expanded, mapped, "ServerConfig env_templates and env_mappings must align"); + assert_eq!( + expanded, mapped, + "ServerConfig env_templates and env_mappings must align" + ); } #[test] @@ -133,7 +136,10 @@ mod consistency_tests { .iter() .map(|m| m.env_name.to_string()) .collect(); - assert_eq!(expanded, mapped, "ClusterConfig env_templates and env_mappings must align"); + assert_eq!( + expanded, mapped, + "ClusterConfig env_templates and env_mappings must align" + ); } #[test] @@ -146,7 +152,10 @@ mod consistency_tests { .iter() .map(|m| m.env_name.to_string()) .collect(); - assert_eq!(expanded, mapped, "McpServerConfig env_templates and env_mappings must align"); + assert_eq!( + expanded, mapped, + "McpServerConfig env_templates and env_mappings must align" + ); } #[test] @@ -175,7 +184,10 @@ mod consistency_tests { .iter() .map(|m| m.env_name.to_string()) .collect(); - assert_eq!(expanded, mapped, "SinkConfig env_templates and env_mappings must align"); + assert_eq!( + expanded, mapped, + "SinkConfig env_templates and env_mappings must align" + ); } #[test] @@ -188,6 +200,9 @@ mod consistency_tests { .iter() .map(|m| m.env_name.to_string()) .collect(); - assert_eq!(expanded, mapped, "SourceConfig env_templates and env_mappings must align"); + assert_eq!( + expanded, mapped, + "SourceConfig env_templates and env_mappings must align" + ); } } diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs index 2208428b7c..cef977fad5 100644 --- a/core/configs/src/configs_impl/mod.rs +++ b/core/configs/src/configs_impl/mod.rs @@ -24,16 +24,16 @@ //! - JSON value field handling //! - Automatic type conversion and validation -mod env_mapping; mod env_listing; +mod env_mapping; mod error; mod file_provider; mod parsing; mod traits; mod typed_env_provider; +pub use env_listing::{CONNECTORS_RUNTIME_ENV_VARS, MCP_RUNTIME_ENV_VARS, print_env_var_names}; pub use env_mapping::{ConfigEnvMappings, EnvVarMapping, EnvVarTemplate}; -pub use env_listing::{print_env_var_names, CONNECTORS_RUNTIME_ENV_VARS, MCP_RUNTIME_ENV_VARS}; pub use error::ConfigurationError; pub use file_provider::{FileConfigProvider, RelocatedKey, RelocatedTarget}; pub use parsing::parse_env_value_to_json; diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index 491d125e66..5cdaaf8bb5 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -23,9 +23,9 @@ mod server_config; pub use common::{COMPONENT, defaults, displays, http, system, validators}; pub use configs_derive::ConfigEnv; pub use configs_impl::{ - ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, - EnvVarTemplate, FileConfigProvider, RelocatedKey, RelocatedTarget, TypedEnvProvider, - parse_env_value_to_json, print_env_var_names, CONNECTORS_RUNTIME_ENV_VARS, MCP_RUNTIME_ENV_VARS, + CONNECTORS_RUNTIME_ENV_VARS, ConfigEnvMappings, ConfigProvider, ConfigurationError, + ConfigurationType, EnvVarMapping, EnvVarTemplate, FileConfigProvider, MCP_RUNTIME_ENV_VARS, + RelocatedKey, RelocatedTarget, TypedEnvProvider, parse_env_value_to_json, print_env_var_names, }; pub use server_config::{ cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 40aaf60e6a..a487ad9564 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -20,7 +20,9 @@ use crate::configs::connectors::{ create_connectors_config_provider, }; use crate::metrics::ConnectorType; -use ::configs::{ConfigEnvMappings, ConfigProvider, print_env_var_names, CONNECTORS_RUNTIME_ENV_VARS}; +use ::configs::{ + CONNECTORS_RUNTIME_ENV_VARS, ConfigEnvMappings, ConfigProvider, print_env_var_names, +}; use clap::Parser; use configs::connectors::ConfigFormat; use configs::runtime::ConnectorsRuntimeConfig; @@ -70,7 +72,9 @@ static GLOBAL: MiMalloc = MiMalloc; #[derive(Parser, Debug)] #[command(author = "Apache Iggy", version)] struct Args { - #[arg(long, long_help = r#"Print supported configuration environment variables and exit. + #[arg( + long, + long_help = r#"Print supported configuration environment variables and exit. Lists all supported IGGY_* environment variable names and templates, sorted and deduplicated. Template syntax: @@ -80,7 +84,8 @@ sorted and deduplicated. Template syntax: - represents plugin configuration field names, excluding FORMAT (handled separately as a strongly-typed field) -Exits immediately before any startup."#)] +Exits immediately before any startup."# + )] list_config_env_vars: bool, } diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs index 2f1e2f580e..bc89c05a20 100644 --- a/core/integration/tests/config_env_listing/mod.rs +++ b/core/integration/tests/config_env_listing/mod.rs @@ -211,7 +211,8 @@ fn config_env_listing_with_fresh_does_not_wipe_data_dir() { .output() .expect("command should run"); - let output_without_fresh = String::from_utf8(output_without_fresh.stdout).expect("UTF-8 output"); + let output_without_fresh = + String::from_utf8(output_without_fresh.stdout).expect("UTF-8 output"); assert_eq!( output_with_fresh, output_without_fresh, "output should be identical with and without --fresh" diff --git a/core/server/src/args.rs b/core/server/src/args.rs index 8922ee26dd..b972001a46 100644 --- a/core/server/src/args.rs +++ b/core/server/src/args.rs @@ -87,7 +87,9 @@ For more information, visit: https://iggy.apache.org/docs/introduction/getting-s // variable names and paths must stay unquoted rather than wear rustdoc backticks. #[allow(clippy::doc_markdown)] pub struct Args { - #[arg(long, long_help = r#"Print supported configuration environment variables and exit. + #[arg( + long, + long_help = r#"Print supported configuration environment variables and exit. Lists all supported IGGY_* environment variable names and templates, sorted and deduplicated. Template syntax: @@ -99,7 +101,8 @@ logging, runtimes, credentials, plugins, filesystem or network activity). Works even with missing or invalid configuration files. Example: - iggy-server --list-config-env-vars"#)] + iggy-server --list-config-env-vars"# + )] pub list_config_env_vars: bool, /// Remove the system path before starting (WARNING: THIS WILL DELETE ALL DATA!) diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 46f3a1fd5a..15568e32c9 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -144,4 +144,4 @@ fn print_config_env_vars() -> std::io::Result<()> { .map(|t| t.env_name) .chain(configs::server::SERVER_RUNTIME_ENV_VARS.iter().copied()), ) -} \ No newline at end of file +} diff --git a/core/server/src/main.rs.save b/core/server/src/main.rs.save new file mode 100644 index 0000000000..91b9a2e724 --- /dev/null +++ b/core/server/src/main.rs.save @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#![allow(clippy::future_not_send)] + +mod args; +mod banner; + +use args::Arguse clap::Parser; +use configs::{ConfigEnvMappings, print_env_var_names, server::ServerConfig}; +use server::boot::{apply_default_root_credentials, bootstrap, load_config, prepare_runtime_dirs}; +use server::server_error::ServerError; +use server_common::log::Logging; +use system_stats::capture_allowed_cpus; +use tracing::{error, info}; + +fn main() -> Result<(), ServerError> { + // This prelude must stay ahead of the first thread the process ever + // spawns: `--with-default-root-credentials` writes to the environment, + // and `set_var` is only sound while single-threaded. `early_init` just + // installs the tracing registry (buffered until `late_init`) and starts + // no worker of its own, so it can run here and make the warnings below + // visible. `create_shard_executor` also reads its capacity knob from the + // environment, which is why the `.env` load has to precede it. + let args = Args::parse(); + if args.list_config_env_vars { + print_config_env_vars()?; + return Ok(()); + } + banner::print(server::VERSION); + // `logging` owns the tracing appender worker guards; it must outlive the + // shard threads or every log line after bootstrap is silently dropped. + let mut logging = Logging::new(server::VERSION); + logging.early_init(); + server_common::print_build_info!(server::VERSION); + #[cfg(all(feature = "mimalloc", not(feature = "disable-mimalloc")))] + info!("Using mimalloc allocator"); + #[cfg(not(all(feature = "mimalloc", not(feature = "disable-mimalloc"))))] + tracing::warn!("Using the default system allocator"); + if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { + let _ = dotenvy::from_path(&env_path); + } else { + let _ = dotenvy::dotenv(); + } + // SAFETY: no thread has been spawned yet, see the comment above. + unsafe { apply_default_root_credentials(args.with_default_root_credentials) }; + + // Before shard threads pin themselves: a pinned capture sees one core. + capture_allowed_cpus(); + + let bootstrap_runtime = match server_common::create_shard_executor() { + Ok(rt) => rt, + Err(e) => { + let e = server_common::diagnostics::enrich_runtime_create_error(e); + panic!("Cannot create server bootstrap executor: {e}"); + } + }; + + // Bootstrap on a temporary runtime: load config, prepare the data + // directory, init the memory pool. Then drop the runtime and spawn the + // per-shard runtimes - each shard thread builds its OWN + // `compio::runtime::Runtime` via `create_shard_executor`, pinned to + // its CPU. + let bootstrap_result: Result = bootstrap_runtime.block_on(async { + let config = load_config().await?; + prepare_runtime_dirs(&config, &mut logging, args.fresh).await?; + let memory_pool_settings = server_common::MemoryPoolSettings::from(&config.memory_pool); + server_common::MemoryPool::init_pool(&memory_pool_settings); + + Ok(config) + }); + let config = bootstrap_result?; + drop(bootstrap_runtime); + + let shards = bootstrap(config, args.replica_id)?; + if let Err(error) = shards.install_ctrlc_handler() { + // Without a working SIGINT handler the server has no way to + // observe an operator Ctrl-C and the shutdown flag would never + // flip, leaving shard threads parked indefinitely. Fail fast + // rather than boot into an un-killable state. + error!(error = %error, "failed to install Ctrl-C handler; aborting boot"); + std::process::exit(1); + } + + info!("server running; waiting on shard threads"); + let joined = shards.join_all(); + #[cfg(feature = "systemd")] + if let Err(error) = &joined { + server::boot::systemd::notify_shutdown_failure(error); + } + joined?; + info!("server shutdown complete"); + Ok(()) +} + +fn print_config_env_vars() -> std::io::Result<()> { + print_env_var_names( + ServerConfig::env_templates() + .iter() + .map(|t| t.env_name) + .chain(configs::server::SERVER_RUNTIME_ENV_VARS.iter().copied()), + ) +} From 47f56bb749af04476a1e8c6bf252f41f79475e49 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Thu, 1 Oct 2026 22:49:29 +0100 Subject: [PATCH 20/27] fix(configs): improve env-var listing tests with detailed error context --- core/configs/src/configs_impl/env_mapping.rs | 67 ------------------- core/connectors/runtime/src/main.rs | 13 ++-- .../tests/config_env_listing/mod.rs | 23 +++++-- core/server/src/main.rs | 2 +- 4 files changed, 25 insertions(+), 80 deletions(-) diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index aad61adbd2..c166066c5c 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -103,11 +103,8 @@ impl EnvVarTemplate { #[cfg(test)] mod consistency_tests { use super::*; - use crate::McpServerConfig; use crate::cluster::ClusterConfig; use crate::server::ServerConfig; - use configs::connectors::{SinkConfig, SourceConfig}; - use configs::runtime::ConnectorsRuntimeConfig; use std::collections::HashSet; #[test] @@ -141,68 +138,4 @@ mod consistency_tests { "ClusterConfig env_templates and env_mappings must align" ); } - - #[test] - fn mcp_server_config_templates_and_mappings_align() { - let expanded: HashSet = McpServerConfig::env_templates() - .iter() - .flat_map(|t| t.expand_names()) - .collect(); - let mapped: HashSet = McpServerConfig::env_mappings() - .iter() - .map(|m| m.env_name.to_string()) - .collect(); - assert_eq!( - expanded, mapped, - "McpServerConfig env_templates and env_mappings must align" - ); - } - - #[test] - fn connectors_runtime_config_templates_and_mappings_align() { - let expanded: HashSet = ConnectorsRuntimeConfig::env_templates() - .iter() - .flat_map(|t| t.expand_names()) - .collect(); - let mapped: HashSet = ConnectorsRuntimeConfig::env_mappings() - .iter() - .map(|m| m.env_name.to_string()) - .collect(); - assert_eq!( - expanded, mapped, - "ConnectorsRuntimeConfig env_templates and env_mappings must align" - ); - } - - #[test] - fn sink_config_templates_and_mappings_align() { - let expanded: HashSet = SinkConfig::env_templates() - .iter() - .flat_map(|t| t.expand_names()) - .collect(); - let mapped: HashSet = SinkConfig::env_mappings() - .iter() - .map(|m| m.env_name.to_string()) - .collect(); - assert_eq!( - expanded, mapped, - "SinkConfig env_templates and env_mappings must align" - ); - } - - #[test] - fn source_config_templates_and_mappings_align() { - let expanded: HashSet = SourceConfig::env_templates() - .iter() - .flat_map(|t| t.expand_names()) - .collect(); - let mapped: HashSet = SourceConfig::env_mappings() - .iter() - .map(|m| m.env_name.to_string()) - .collect(); - assert_eq!( - expanded, mapped, - "SourceConfig env_templates and env_mappings must align" - ); - } } diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index a487ad9564..daa3e540c0 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -161,13 +161,10 @@ fn print_config_env_vars() -> std::io::Result<()> { templates .iter() .map(move |template| format!("{prefix}{kind}__{}", template.env_name)) - .chain( - [ - format!("{prefix}{kind}__ENABLED"), - format!("{prefix}{kind}__PLUGIN_CONFIG_"), - ] - .into_iter(), - ) + .chain([ + format!("{prefix}{kind}__ENABLED"), + format!("{prefix}{kind}__PLUGIN_CONFIG_"), + ]) }) .collect(); @@ -175,7 +172,7 @@ fn print_config_env_vars() -> std::io::Result<()> { .iter() .map(|t| t.env_name.to_string()) .chain(CONNECTORS_RUNTIME_ENV_VARS.iter().map(|s| s.to_string())) - .chain(sink_source_templates.into_iter()); + .chain(sink_source_templates); print_env_var_names(names) } diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs index bc89c05a20..714cdf8c84 100644 --- a/core/integration/tests/config_env_listing/mod.rs +++ b/core/integration/tests/config_env_listing/mod.rs @@ -43,7 +43,12 @@ fn config_env_listing_exits_before_startup_for_each_binary() { .output() .expect("listing command should run"); - assert!(output.status.success(), "{binary} failed"); + assert!( + output.status.success(), + "{binary} failed: {:?}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); assert!(output.stderr.is_empty(), "{binary} wrote to stderr"); let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); @@ -53,7 +58,10 @@ fn config_env_listing_exits_before_startup_for_each_binary() { names.windows(2).all(|pair| pair[0] < pair[1]), "{binary} output must be sorted and deduplicated" ); - assert!(names.iter().all(|name| name.starts_with("IGGY_"))); + assert!( + names.iter().all(|name| name.starts_with("IGGY_")), + "{binary} output contained non-IGGY_ prefixed names" + ); } } @@ -162,7 +170,9 @@ fn config_env_listing_exits_before_runtime_with_invalid_env_values() { assert!( output.status.success(), - "listing should exit before validating environment values" + "listing should exit before validating environment values: {:?}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) ); assert!(output.stderr.is_empty(), "should not log validation errors"); } @@ -188,7 +198,12 @@ fn config_env_listing_with_fresh_does_not_wipe_data_dir() { .output() .expect("listing command should run"); - assert!(output.status.success(), "command failed"); + assert!( + output.status.success(), + "command failed: {:?}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); assert!( sentinel.exists(), "sentinel file should not be deleted by early exit" diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 15568e32c9..603885aa91 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -123,7 +123,7 @@ fn main() -> Result<(), ServerError> { if let Err(error) = &joined && descriptors_exhausted() { - // `fatal` skips destructors, and the log flush must run + // `fatal` skips destructors, and the log appenders flush on drop. fatal_with_log_flush( FatalReason::DescriptorsExhausted, &error.to_string(), From d874cd6a949c9085f7d2567c27cfb0ff57a49bd0 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Thu, 1 Oct 2026 22:59:22 +0100 Subject: [PATCH 21/27] refactor(configs): make print_env_var_names testable via Write trait --- core/ai/mcp/src/main.rs | 2 ++ core/configs/src/configs_impl/env_listing.rs | 16 ++++++---------- core/connectors/runtime/src/main.rs | 3 ++- core/server/src/main.rs | 2 ++ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/core/ai/mcp/src/main.rs b/core/ai/mcp/src/main.rs index c6c8877374..122efb6856 100644 --- a/core/ai/mcp/src/main.rs +++ b/core/ai/mcp/src/main.rs @@ -68,11 +68,13 @@ fn main() -> Result<(), McpRuntimeError> { } fn print_config_env_vars() -> std::io::Result<()> { + let mut stdout = std::io::stdout(); print_env_var_names( McpServerConfig::env_templates() .iter() .map(|t| t.env_name) .chain(MCP_RUNTIME_ENV_VARS.iter().copied()), + &mut stdout, ) } diff --git a/core/configs/src/configs_impl/env_listing.rs b/core/configs/src/configs_impl/env_listing.rs index 4c062bbb13..6a94d93d82 100644 --- a/core/configs/src/configs_impl/env_listing.rs +++ b/core/configs/src/configs_impl/env_listing.rs @@ -15,26 +15,22 @@ // specific language governing permissions and limitations // under the License. -use std::io::Write; - -/// Writes each name in `names`, sorted and deduplicated, to stdout — one -/// call, one lock, one error policy. A closed stdout (e.g. -/// `iggy-server --list-config-env-vars | head -1`) is expected, not a -/// failure: on `BrokenPipe` this stops writing and returns `Ok(())`. Any +/// Writes each name in `names`, sorted and deduplicated, to the provided writer — one +/// call, one error policy. A closed writer (e.g. `iggy-server --list-config-env-vars | head -1`) +/// is expected, not a failure: on `BrokenPipe` this stops writing and returns `Ok(())`. Any /// other I/O error is real and is propagated so the caller exits non-zero. -pub fn print_env_var_names(names: I) -> std::io::Result<()> +pub fn print_env_var_names(names: I, writer: &mut W) -> std::io::Result<()> where I: IntoIterator, S: Into, + W: std::io::Write, { let mut names: Vec = names.into_iter().map(Into::into).collect(); names.sort_unstable(); names.dedup(); - let stdout = std::io::stdout(); - let mut lock = stdout.lock(); for name in names { - if let Err(err) = writeln!(lock, "{name}") { + if let Err(err) = writeln!(writer, "{name}") { return if err.kind() == std::io::ErrorKind::BrokenPipe { Ok(()) } else { diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index daa3e540c0..1de4019fff 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -174,7 +174,8 @@ fn print_config_env_vars() -> std::io::Result<()> { .chain(CONNECTORS_RUNTIME_ENV_VARS.iter().map(|s| s.to_string())) .chain(sink_source_templates); - print_env_var_names(names) + let mut stdout = std::io::stdout(); + print_env_var_names(names, &mut stdout) } async fn run() -> Result<(), RuntimeError> { diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 603885aa91..9f41cf5ee5 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -138,10 +138,12 @@ fn main() -> Result<(), ServerError> { } fn print_config_env_vars() -> std::io::Result<()> { + let mut stdout = std::io::stdout(); print_env_var_names( ServerConfig::env_templates() .iter() .map(|t| t.env_name) .chain(configs::server::SERVER_RUNTIME_ENV_VARS.iter().copied()), + &mut stdout, ) } From d529842b5e2e36f52217f3b16b198fcb7df3b29b Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Thu, 1 Oct 2026 23:07:48 +0100 Subject: [PATCH 22/27] fix(configs): address remaining hubcio review comments --- core/configs/src/configs_impl/env_mapping.rs | 4 ---- .../tests/config_env_listing/mod.rs | 24 +++++++++++++++++++ core/server/src/main.rs | 2 +- core/server/src/server_error.rs | 4 ++-- 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index c166066c5c..fc535b427b 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -80,10 +80,6 @@ impl EnvVarTemplate { /// substituting each `` placeholder left-to-right with `0..limit`. /// A leaf template (`max_elements: &[]`) expands to itself. pub fn expand_names(&self) -> Vec { - if self.max_elements.is_empty() { - return vec![self.env_name.to_string()]; - } - let mut results = vec![self.env_name.to_string()]; for &limit in self.max_elements { diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs index 714cdf8c84..093ced2eaa 100644 --- a/core/integration/tests/config_env_listing/mod.rs +++ b/core/integration/tests/config_env_listing/mod.rs @@ -21,6 +21,30 @@ use std::time::Duration; const LIST_ENV_VARS_TIMEOUT: Duration = Duration::from_secs(5); +#[allow(dead_code)] +fn run_list_config_env_vars(binary: &str, env: I, extra_args: &[&str]) -> std::process::Output +where + I: IntoIterator, + K: AsRef, + V: AsRef, +{ + let directory = tempfile::tempdir().expect("temporary directory"); + let mut cmd = Command::cargo_bin(binary).expect("binary should be built"); + cmd.current_dir(directory.path()) + .arg("--list-config-env-vars") + .timeout(LIST_ENV_VARS_TIMEOUT); + + for (key, value) in env { + cmd.env(key, value); + } + + for arg in extra_args { + cmd.arg(arg); + } + + cmd.output().expect("listing command should run") +} + #[test] fn config_env_listing_exits_before_startup_for_each_binary() { for (binary, config_env, dotenv_env) in [ diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 9f41cf5ee5..76f48da3d4 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -43,7 +43,7 @@ fn main() -> Result<(), ServerError> { // environment, which is why the `.env` load has to precede it. let args = Args::parse(); if args.list_config_env_vars { - print_config_env_vars()?; + print_config_env_vars().map_err(ServerError::ListConfigEnvVars)?; return Ok(()); } banner::print(server::VERSION); diff --git a/core/server/src/server_error.rs b/core/server/src/server_error.rs index b81baad8a2..dec666c425 100644 --- a/core/server/src/server_error.rs +++ b/core/server/src/server_error.rs @@ -313,8 +313,8 @@ pub enum ServerError { /// as clean. #[error("server shut down after a panic: {description}")] Panicked { description: String }, - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), + #[error("Failed to list config environment variables")] + ListConfigEnvVars(#[source] std::io::Error), } /// Per-shard outcome captured by [`crate::boot::ShardHandles::join_all`] From d2d492a8cd4c1f3f3452c99fa94045410c9224c0 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Thu, 1 Oct 2026 23:14:29 +0100 Subject: [PATCH 23/27] fix(configs): address remaining hubcio review comments --- core/configs/src/server_config/server.rs | 14 +++++++------- core/connectors/runtime/src/main.rs | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index e8ae621b22..c8ab98beb8 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -58,8 +58,9 @@ pub const SERVER_RUNTIME_ENV_VARS: &[&str] = &[ "IGGY_SHARD_RUNTIME_CAPACITY", ]; -/// Test/CI-only vars: never advertised, but still non-config (suppress -/// "unknown env var" warnings the same as `SERVER_RUNTIME_ENV_VARS` does). +/// Vars used by sibling binaries (iggy CLI) or test/CI-only: never advertised, +/// but still non-config. Included in `SERVER_PROCESS_ENV_VARS` to suppress +/// "unknown env var" warnings during config scanning. pub const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ "IGGY_TEST_VERBOSE", "IGGY_TEST_CLUSTER_NODES", @@ -70,11 +71,10 @@ pub const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ "IGGY_PASSWORD", ]; -/// All non-config env vars the server process reads directly — the union of -/// `SERVER_RUNTIME_ENV_VARS` and `SERVER_SCAN_ONLY_ENV_VARS`. Kept as one -/// list (rather than requiring every call site to chain both) so existing -/// consumers are unaffected; a test below pins its contents to the split so -/// the two views can't silently diverge. +/// All non-config env vars — the union of `SERVER_RUNTIME_ENV_VARS` and +/// `SERVER_SCAN_ONLY_ENV_VARS`. Kept as one list (rather than requiring every +/// call site to chain both) so existing consumers are unaffected; a test below +/// pins its contents to the split so the two views can't silently diverge. pub const SERVER_PROCESS_ENV_VARS: &[&str] = &[ "IGGY_CONFIG_PATH", "IGGY_ENV_PATH", diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 1de4019fff..cd073705db 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -78,7 +78,7 @@ struct Args { Lists all supported IGGY_* environment variable names and templates, sorted and deduplicated. Template syntax: -- represents vector indices +- represents vector indices (0-255 for stream fields) - represents connector keys (uppercased from config). Overrides via require the local connectors provider. - represents plugin configuration field names, excluding From dc5273e71d24448af103a918119d354b38e37cae Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Sat, 3 Oct 2026 00:25:36 +0100 Subject: [PATCH 24/27] fix(configs): fix McpRuntimeError Io variant and actually use test helper --- core/ai/mcp/src/error.rs | 4 +- core/ai/mcp/src/main.rs | 2 +- .../tests/config_env_listing/mod.rs | 110 +++++++++--------- 3 files changed, 58 insertions(+), 58 deletions(-) diff --git a/core/ai/mcp/src/error.rs b/core/ai/mcp/src/error.rs index 334f79f2ee..369938bd3f 100644 --- a/core/ai/mcp/src/error.rs +++ b/core/ai/mcp/src/error.rs @@ -45,6 +45,6 @@ pub enum McpRuntimeError { TokenFileReadError(String, String), #[error("Token file is empty: {0}")] TokenFileEmpty(String), - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), + #[error("Failed to list config environment variables")] + ListConfigEnvVars(#[source] std::io::Error), } diff --git a/core/ai/mcp/src/main.rs b/core/ai/mcp/src/main.rs index 122efb6856..17830833a0 100644 --- a/core/ai/mcp/src/main.rs +++ b/core/ai/mcp/src/main.rs @@ -54,7 +54,7 @@ struct Args { fn main() -> Result<(), McpRuntimeError> { let args = Args::parse(); if args.list_config_env_vars { - print_config_env_vars()?; + print_config_env_vars().map_err(McpRuntimeError::ListConfigEnvVars)?; return Ok(()); } let runtime = Builder::new_multi_thread() diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs index 093ced2eaa..000b58ede4 100644 --- a/core/integration/tests/config_env_listing/mod.rs +++ b/core/integration/tests/config_env_listing/mod.rs @@ -21,16 +21,22 @@ use std::time::Duration; const LIST_ENV_VARS_TIMEOUT: Duration = Duration::from_secs(5); -#[allow(dead_code)] -fn run_list_config_env_vars(binary: &str, env: I, extra_args: &[&str]) -> std::process::Output +/// Runs `binary --list-config-env-vars` (plus `extra_args`) with `env` applied, +/// isolated to `directory`. Shared by every test below so each only states +/// what differs: the binary, the env it sets, and any extra flags. +fn run_list_config_env_vars_in( + binary: &str, + directory: &std::path::Path, + env: I, + extra_args: &[&str], +) -> std::process::Output where I: IntoIterator, K: AsRef, V: AsRef, { - let directory = tempfile::tempdir().expect("temporary directory"); let mut cmd = Command::cargo_bin(binary).expect("binary should be built"); - cmd.current_dir(directory.path()) + cmd.current_dir(directory) .arg("--list-config-env-vars") .timeout(LIST_ENV_VARS_TIMEOUT); @@ -45,6 +51,28 @@ where cmd.output().expect("listing command should run") } +/// Convenience wrapper over [`run_list_config_env_vars_in`] for tests that +/// don't need to inspect the working directory afterward. +fn run_list_config_env_vars( + binary: &str, + env: I, + extra_args: &[&str], +) -> std::process::Output +where + I: IntoIterator, + K: AsRef, + V: AsRef, +{ + let directory = tempfile::tempdir().expect("temporary directory"); + run_list_config_env_vars_in(binary, directory.path(), env, extra_args) +} + +/// Convenience wrapper over [`run_list_config_env_vars`] for tests that don't +/// need to set any environment variables. +fn run_list_config_env_vars_plain(binary: &str, extra_args: &[&str]) -> std::process::Output { + run_list_config_env_vars(binary, std::iter::empty::<(&str, &str)>(), extra_args) +} + #[test] fn config_env_listing_exits_before_startup_for_each_binary() { for (binary, config_env, dotenv_env) in [ @@ -57,15 +85,15 @@ fn config_env_listing_exits_before_startup_for_each_binary() { ("iggy-mcp", "IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"), ] { let directory = tempfile::tempdir().expect("temporary directory"); - let output = Command::cargo_bin(binary) - .expect("binary should be built") - .current_dir(directory.path()) - .arg("--list-config-env-vars") - .env(config_env, directory.path().join("missing-config.toml")) - .env(dotenv_env, directory.path().join("missing.env")) - .timeout(LIST_ENV_VARS_TIMEOUT) - .output() - .expect("listing command should run"); + let output = run_list_config_env_vars_in( + binary, + directory.path(), + [ + (config_env, directory.path().join("missing-config.toml")), + (dotenv_env, directory.path().join("missing.env")), + ], + &[], + ); assert!( output.status.success(), @@ -92,14 +120,7 @@ fn config_env_listing_exits_before_startup_for_each_binary() { #[test] fn config_env_listing_includes_vector_index_templates() { // Verify that vector fields are correctly represented with placeholder - let directory = tempfile::tempdir().expect("temporary directory"); - let output = Command::cargo_bin("iggy-server") - .expect("binary should be built") - .current_dir(directory.path()) - .arg("--list-config-env-vars") - .timeout(LIST_ENV_VARS_TIMEOUT) - .output() - .expect("listing command should run"); + let output = run_list_config_env_vars_plain("iggy-server", &[]); let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); let names: HashSet<_> = stdout.lines().collect(); @@ -124,14 +145,7 @@ fn config_env_listing_includes_vector_index_templates() { #[test] fn config_env_listing_includes_connector_templates() { // Verify that connector SINK/SOURCE templates are correctly formatted - let directory = tempfile::tempdir().expect("temporary directory"); - let output = Command::cargo_bin("iggy-connectors") - .expect("binary should be built") - .current_dir(directory.path()) - .arg("--list-config-env-vars") - .timeout(LIST_ENV_VARS_TIMEOUT) - .output() - .expect("listing command should run"); + let output = run_list_config_env_vars_plain("iggy-connectors", &[]); let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); let names: HashSet<_> = stdout.lines().collect(); @@ -182,15 +196,11 @@ fn config_env_listing_includes_connector_templates() { #[test] fn config_env_listing_exits_before_runtime_with_invalid_env_values() { // Verify that invalid environment values don't cause failures - let directory = tempfile::tempdir().expect("temporary directory"); - let output = Command::cargo_bin("iggy-server") - .expect("binary should be built") - .current_dir(directory.path()) - .arg("--list-config-env-vars") - .env("IGGY_HTTP_ADDRESS", "invalid::address") // Invalid address - .timeout(LIST_ENV_VARS_TIMEOUT) - .output() - .expect("listing command should run"); + let output = run_list_config_env_vars( + "iggy-server", + [("IGGY_HTTP_ADDRESS", "invalid::address")], // Invalid address + &[], + ); assert!( output.status.success(), @@ -212,15 +222,12 @@ fn config_env_listing_with_fresh_does_not_wipe_data_dir() { std::fs::write(&sentinel, "sentinel content").expect("write sentinel file"); // Run with both --fresh and --list-config-env-vars - let output = Command::cargo_bin("iggy-server") - .expect("binary should be built") - .current_dir(directory.path()) - .arg("--fresh") - .arg("--list-config-env-vars") - .env("IGGY_PATH", data_dir.to_string_lossy().as_ref()) - .timeout(LIST_ENV_VARS_TIMEOUT) - .output() - .expect("listing command should run"); + let output = run_list_config_env_vars_in( + "iggy-server", + directory.path(), + [("IGGY_PATH", data_dir.to_string_lossy().into_owned())], + &["--fresh"], + ); assert!( output.status.success(), @@ -241,14 +248,7 @@ fn config_env_listing_with_fresh_does_not_wipe_data_dir() { // Verify the output is the same as without --fresh let output_with_fresh = String::from_utf8(output.stdout).expect("UTF-8 output"); - let directory_fresh = tempfile::tempdir().expect("temporary directory"); - let output_without_fresh = Command::cargo_bin("iggy-server") - .expect("binary should be built") - .current_dir(directory_fresh.path()) - .arg("--list-config-env-vars") - .timeout(LIST_ENV_VARS_TIMEOUT) - .output() - .expect("command should run"); + let output_without_fresh = run_list_config_env_vars_plain("iggy-server", &[]); let output_without_fresh = String::from_utf8(output_without_fresh.stdout).expect("UTF-8 output"); From 479cf90c7825b29cc9fc9bc567db326810fc08f8 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Sat, 3 Oct 2026 17:51:13 +0100 Subject: [PATCH 25/27] fix(configs): resolve remaining simplification requests from review --- core/configs/src/configs_impl/env_mapping.rs | 32 ++--- core/configs/src/server_config/server.rs | 50 +++----- .../runtime/src/configs/connectors.rs | 2 +- .../src/configs/connectors/local_provider.rs | 21 +++- core/connectors/runtime/src/main.rs | 15 ++- core/server/src/main.rs | 2 +- core/server/src/main.rs.save | 117 ------------------ 7 files changed, 57 insertions(+), 182 deletions(-) delete mode 100644 core/server/src/main.rs.save diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index fc535b427b..286cdade05 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -99,39 +99,33 @@ impl EnvVarTemplate { #[cfg(test)] mod consistency_tests { use super::*; - use crate::cluster::ClusterConfig; use crate::server::ServerConfig; use std::collections::HashSet; - #[test] - fn server_config_templates_and_mappings_align() { - let expanded: HashSet = ServerConfig::env_templates() + /// Asserts that `T::env_templates()`, expanded, names exactly the same + /// set as `T::env_mappings()`. One generic body for every `ConfigEnv` + /// type, so adding a type to check is one call, not one copy-pasted test. + fn assert_templates_and_mappings_align(type_name: &str) { + let expanded: HashSet = T::env_templates() .iter() .flat_map(|t| t.expand_names()) .collect(); - let mapped: HashSet = ServerConfig::env_mappings() + let mapped: HashSet = T::env_mappings() .iter() .map(|m| m.env_name.to_string()) .collect(); assert_eq!( expanded, mapped, - "ServerConfig env_templates and env_mappings must align" + "{type_name} env_templates and env_mappings must align" ); } #[test] - fn cluster_config_templates_and_mappings_align() { - let expanded: HashSet = ClusterConfig::env_templates() - .iter() - .flat_map(|t| t.expand_names()) - .collect(); - let mapped: HashSet = ClusterConfig::env_mappings() - .iter() - .map(|m| m.env_name.to_string()) - .collect(); - assert_eq!( - expanded, mapped, - "ClusterConfig env_templates and env_mappings must align" - ); + fn server_config_templates_and_mappings_align() { + // ClusterConfig is only ever embedded as a field of ServerConfig + // (never derived or called standalone in production code), so this + // single check already covers its templates and mappings too - + // a separate ClusterConfig-only test would just repeat it. + assert_templates_and_mappings_align::("ServerConfig"); } } diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index c8ab98beb8..ab04c74401 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -47,20 +47,10 @@ pub use crate::common::server::{ TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, TelemetryTransport, }; -/// Vars safe to advertise to end users via `--list-config-env-vars`. -pub const SERVER_RUNTIME_ENV_VARS: &[&str] = &[ - "IGGY_CONFIG_PATH", - "IGGY_DISPLAY_CONFIG", - "IGGY_ENV_PATH", - "IGGY_ROOT_PASSWORD", - "IGGY_ROOT_USERNAME", - "IGGY_SHARD_EVENT_INTERVAL", - "IGGY_SHARD_RUNTIME_CAPACITY", -]; - -/// Vars used by sibling binaries (iggy CLI) or test/CI-only: never advertised, -/// but still non-config. Included in `SERVER_PROCESS_ENV_VARS` to suppress -/// "unknown env var" warnings during config scanning. +/// Vars used by sibling binaries (iggy CLI) or test/CI-only: never advertised +/// via `--list-config-env-vars`, but still non-config. Included in +/// `SERVER_PROCESS_ENV_VARS` to suppress "unknown env var" warnings during +/// config scanning. pub const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ "IGGY_TEST_VERBOSE", "IGGY_TEST_CLUSTER_NODES", @@ -71,10 +61,7 @@ pub const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ "IGGY_PASSWORD", ]; -/// All non-config env vars — the union of `SERVER_RUNTIME_ENV_VARS` and -/// `SERVER_SCAN_ONLY_ENV_VARS`. Kept as one list (rather than requiring every -/// call site to chain both) so existing consumers are unaffected; a test below -/// pins its contents to the split so the two views can't silently diverge. +/// All non-config env vars the server process scans for. pub const SERVER_PROCESS_ENV_VARS: &[&str] = &[ "IGGY_CONFIG_PATH", "IGGY_ENV_PATH", @@ -92,24 +79,15 @@ pub const SERVER_PROCESS_ENV_VARS: &[&str] = &[ "IGGY_PASSWORD", ]; -#[cfg(test)] -mod server_process_env_vars_tests { - use super::*; - use std::collections::HashSet; - - #[test] - fn runtime_and_scan_only_lists_cover_process_env_vars_exactly() { - let split: HashSet<&str> = SERVER_RUNTIME_ENV_VARS - .iter() - .chain(SERVER_SCAN_ONLY_ENV_VARS.iter()) - .copied() - .collect(); - let whole: HashSet<&str> = SERVER_PROCESS_ENV_VARS.iter().copied().collect(); - assert_eq!( - split, whole, - "SERVER_RUNTIME_ENV_VARS + SERVER_SCAN_ONLY_ENV_VARS must equal SERVER_PROCESS_ENV_VARS" - ); - } +/// Vars safe to advertise to end users via `--list-config-env-vars` — every +/// entry in `SERVER_PROCESS_ENV_VARS` except the sibling-binary/test-only +/// ones in `SERVER_SCAN_ONLY_ENV_VARS`. Computed rather than retyped, so the +/// two views cannot drift apart. +pub fn server_runtime_env_vars() -> impl Iterator { + SERVER_PROCESS_ENV_VARS + .iter() + .copied() + .filter(|name| !SERVER_SCAN_ONLY_ENV_VARS.contains(name)) } pub(crate) const SERVER_ALLOWED_ENV_PREFIXES: &[&str] = diff --git a/core/connectors/runtime/src/configs/connectors.rs b/core/connectors/runtime/src/configs/connectors.rs index ad6a6c435e..278481f4cc 100644 --- a/core/connectors/runtime/src/configs/connectors.rs +++ b/core/connectors/runtime/src/configs/connectors.rs @@ -16,7 +16,7 @@ // under the License. pub mod http_provider; -mod local_provider; +pub(crate) mod local_provider; use crate::configs::connectors::http_provider::HttpConnectorsConfigProvider; use crate::configs::connectors::local_provider::LocalConnectorsConfigProvider; diff --git a/core/connectors/runtime/src/configs/connectors/local_provider.rs b/core/connectors/runtime/src/configs/connectors/local_provider.rs index fc012bad25..bb8b8aa668 100644 --- a/core/connectors/runtime/src/configs/connectors/local_provider.rs +++ b/core/connectors/runtime/src/configs/connectors/local_provider.rs @@ -348,7 +348,10 @@ impl LocalConnectorsConfigProvider { ) { let connector_type = base_config.connector_type().to_uppercase(); let key = base_config.key().to_uppercase(); - let prefix = format!("IGGY_CONNECTORS_{connector_type}_{key}_PLUGIN_CONFIG_"); + let prefix = format!( + "{}PLUGIN_CONFIG_", + connector_env_prefix(&connector_type, &key) + ); for (env_key, env_value) in std::env::vars() { let env_key_upper = env_key.to_uppercase(); @@ -395,6 +398,20 @@ impl BaseConnectorConfig { } } +/// Builds the env-var prefix a connector's config and plugin-config overrides +/// are matched against: `IGGY_CONNECTORS_{TYPE}_{KEY}_`. `connector_type` and +/// `key` are expected uppercased, as `BaseConnectorConfig`'s callers already +/// do. Shared by the runtime's actual override lookup here and by +/// `--list-config-env-vars`'s listing in `main.rs` (passing the literal +/// `` as the key), so the two can't drift apart on the prefix the +/// server actually reads. +pub(crate) fn connector_env_prefix(connector_type: &str, key: &str) -> String { + format!( + "{}{connector_type}_{key}_", + crate::configs::runtime::ConnectorsRuntimeConfig::ENV_PREFIX + ) +} + #[async_trait] impl ConnectorsConfigProvider for LocalConnectorsConfigProvider { async fn create_sink_config( @@ -822,7 +839,7 @@ impl ConnectorEnvProvider { fn with_connector_base_config(base_config: &BaseConnectorConfig) -> Self { let connector_type = base_config.connector_type().to_uppercase(); let key = base_config.key().to_uppercase(); - let prefix = format!("IGGY_CONNECTORS_{}_{}_", connector_type, key); + let prefix = connector_env_prefix(&connector_type, &key); let connector_name = base_config.key().to_owned(); match base_config { diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index cd073705db..3b94247d2f 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -151,20 +151,23 @@ fn main() -> Result<(), RuntimeError> { } fn print_config_env_vars() -> std::io::Result<()> { - let prefix = ConnectorsRuntimeConfig::ENV_PREFIX; let sink_source_templates: Vec = [ ("SINK", SinkConfig::env_templates()), ("SOURCE", SourceConfig::env_templates()), ] .iter() .flat_map(|(kind, templates)| { + // "" stands in for the real, per-connector key `local_provider` + // uppercases at runtime - same prefix rule, so the listing can't + // drift from the names the runtime actually reads. + let prefix = + crate::configs::connectors::local_provider::connector_env_prefix(kind, ""); + let enabled = format!("{prefix}ENABLED"); + let plugin_config = format!("{prefix}PLUGIN_CONFIG_"); templates .iter() - .map(move |template| format!("{prefix}{kind}__{}", template.env_name)) - .chain([ - format!("{prefix}{kind}__ENABLED"), - format!("{prefix}{kind}__PLUGIN_CONFIG_"), - ]) + .map(move |template| format!("{prefix}{}", template.env_name)) + .chain([enabled, plugin_config]) }) .collect(); diff --git a/core/server/src/main.rs b/core/server/src/main.rs index 76f48da3d4..a0bc0473b8 100644 --- a/core/server/src/main.rs +++ b/core/server/src/main.rs @@ -143,7 +143,7 @@ fn print_config_env_vars() -> std::io::Result<()> { ServerConfig::env_templates() .iter() .map(|t| t.env_name) - .chain(configs::server::SERVER_RUNTIME_ENV_VARS.iter().copied()), + .chain(configs::server::server_runtime_env_vars()), &mut stdout, ) } diff --git a/core/server/src/main.rs.save b/core/server/src/main.rs.save deleted file mode 100644 index 91b9a2e724..0000000000 --- a/core/server/src/main.rs.save +++ /dev/null @@ -1,117 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#![allow(clippy::future_not_send)] - -mod args; -mod banner; - -use args::Arguse clap::Parser; -use configs::{ConfigEnvMappings, print_env_var_names, server::ServerConfig}; -use server::boot::{apply_default_root_credentials, bootstrap, load_config, prepare_runtime_dirs}; -use server::server_error::ServerError; -use server_common::log::Logging; -use system_stats::capture_allowed_cpus; -use tracing::{error, info}; - -fn main() -> Result<(), ServerError> { - // This prelude must stay ahead of the first thread the process ever - // spawns: `--with-default-root-credentials` writes to the environment, - // and `set_var` is only sound while single-threaded. `early_init` just - // installs the tracing registry (buffered until `late_init`) and starts - // no worker of its own, so it can run here and make the warnings below - // visible. `create_shard_executor` also reads its capacity knob from the - // environment, which is why the `.env` load has to precede it. - let args = Args::parse(); - if args.list_config_env_vars { - print_config_env_vars()?; - return Ok(()); - } - banner::print(server::VERSION); - // `logging` owns the tracing appender worker guards; it must outlive the - // shard threads or every log line after bootstrap is silently dropped. - let mut logging = Logging::new(server::VERSION); - logging.early_init(); - server_common::print_build_info!(server::VERSION); - #[cfg(all(feature = "mimalloc", not(feature = "disable-mimalloc")))] - info!("Using mimalloc allocator"); - #[cfg(not(all(feature = "mimalloc", not(feature = "disable-mimalloc"))))] - tracing::warn!("Using the default system allocator"); - if let Ok(env_path) = std::env::var("IGGY_ENV_PATH") { - let _ = dotenvy::from_path(&env_path); - } else { - let _ = dotenvy::dotenv(); - } - // SAFETY: no thread has been spawned yet, see the comment above. - unsafe { apply_default_root_credentials(args.with_default_root_credentials) }; - - // Before shard threads pin themselves: a pinned capture sees one core. - capture_allowed_cpus(); - - let bootstrap_runtime = match server_common::create_shard_executor() { - Ok(rt) => rt, - Err(e) => { - let e = server_common::diagnostics::enrich_runtime_create_error(e); - panic!("Cannot create server bootstrap executor: {e}"); - } - }; - - // Bootstrap on a temporary runtime: load config, prepare the data - // directory, init the memory pool. Then drop the runtime and spawn the - // per-shard runtimes - each shard thread builds its OWN - // `compio::runtime::Runtime` via `create_shard_executor`, pinned to - // its CPU. - let bootstrap_result: Result = bootstrap_runtime.block_on(async { - let config = load_config().await?; - prepare_runtime_dirs(&config, &mut logging, args.fresh).await?; - let memory_pool_settings = server_common::MemoryPoolSettings::from(&config.memory_pool); - server_common::MemoryPool::init_pool(&memory_pool_settings); - - Ok(config) - }); - let config = bootstrap_result?; - drop(bootstrap_runtime); - - let shards = bootstrap(config, args.replica_id)?; - if let Err(error) = shards.install_ctrlc_handler() { - // Without a working SIGINT handler the server has no way to - // observe an operator Ctrl-C and the shutdown flag would never - // flip, leaving shard threads parked indefinitely. Fail fast - // rather than boot into an un-killable state. - error!(error = %error, "failed to install Ctrl-C handler; aborting boot"); - std::process::exit(1); - } - - info!("server running; waiting on shard threads"); - let joined = shards.join_all(); - #[cfg(feature = "systemd")] - if let Err(error) = &joined { - server::boot::systemd::notify_shutdown_failure(error); - } - joined?; - info!("server shutdown complete"); - Ok(()) -} - -fn print_config_env_vars() -> std::io::Result<()> { - print_env_var_names( - ServerConfig::env_templates() - .iter() - .map(|t| t.env_name) - .chain(configs::server::SERVER_RUNTIME_ENV_VARS.iter().copied()), - ) -} From 2a4427735b60f5e1a960d0d2ec45876ee17f3fb9 Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Sat, 3 Oct 2026 18:35:38 +0100 Subject: [PATCH 26/27] fix(configs): complete environment variable listing --- core/ai/mcp/src/main.rs | 9 +- core/configs/src/configs_impl/env_listing.rs | 15 +- core/configs/src/configs_impl/env_mapping.rs | 64 +++---- core/configs/src/configs_impl/mod.rs | 7 +- .../src/configs_impl/typed_env_provider.rs | 41 ++-- core/configs/src/lib.rs | 8 +- core/configs_derive/src/config_env.rs | 178 +++--------------- core/connectors/runtime/src/main.rs | 18 +- .../tests/config_env_listing/mod.rs | 102 +++++++++- core/server/src/args.rs | 1 + 10 files changed, 197 insertions(+), 246 deletions(-) diff --git a/core/ai/mcp/src/main.rs b/core/ai/mcp/src/main.rs index 17830833a0..38dded37da 100644 --- a/core/ai/mcp/src/main.rs +++ b/core/ai/mcp/src/main.rs @@ -15,7 +15,10 @@ // specific language governing permissions and limitations // under the License. -use ::configs::{ConfigEnvMappings, ConfigProvider, MCP_RUNTIME_ENV_VARS, print_env_var_names}; +use ::configs::{ + ConfigEnvMappings, ConfigProvider, MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV, MCP_RUNTIME_ENV_VARS, + print_env_var_names, +}; use clap::Parser; use configs::{McpServerConfig, McpTransport}; use dotenvy::dotenv; @@ -83,7 +86,7 @@ async fn run() -> Result<(), McpRuntimeError> { let figure = standard_font.convert("Iggy MCP Server"); eprintln!("{}", figure.unwrap()); - if let Ok(env_path) = std::env::var("IGGY_MCP_ENV_PATH") { + if let Ok(env_path) = std::env::var(MCP_ENV_PATH_ENV) { if dotenvy::from_path(&env_path).is_ok() { eprintln!("Loaded environment variables from path: {env_path}"); } @@ -95,7 +98,7 @@ async fn run() -> Result<(), McpRuntimeError> { } let config_path = - env::var("IGGY_MCP_CONFIG_PATH").unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); + env::var(MCP_CONFIG_PATH_ENV).unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); eprintln!("Configuration file path: {config_path}"); let config: McpServerConfig = McpServerConfig::config_provider(config_path) .load_config() diff --git a/core/configs/src/configs_impl/env_listing.rs b/core/configs/src/configs_impl/env_listing.rs index 6a94d93d82..8ac8c74a59 100644 --- a/core/configs/src/configs_impl/env_listing.rs +++ b/core/configs/src/configs_impl/env_listing.rs @@ -43,16 +43,17 @@ where /// Env vars `iggy-mcp --list-config-env-vars` advertises beyond the derived /// `McpServerConfig` templates. -pub const MCP_RUNTIME_ENV_VARS: &[&str] = &[ - "IGGY_DISPLAY_CONFIG", - "IGGY_MCP_CONFIG_PATH", - "IGGY_MCP_ENV_PATH", -]; +pub const MCP_CONFIG_PATH_ENV: &str = "IGGY_MCP_CONFIG_PATH"; +pub const MCP_ENV_PATH_ENV: &str = "IGGY_MCP_ENV_PATH"; +pub const MCP_RUNTIME_ENV_VARS: &[&str] = + &["IGGY_DISPLAY_CONFIG", MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV]; /// Env vars `iggy-connectors --list-config-env-vars` advertises beyond the /// derived `ConnectorsRuntimeConfig` templates. +pub const CONNECTORS_CONFIG_PATH_ENV: &str = "IGGY_CONNECTORS_CONFIG_PATH"; +pub const CONNECTORS_ENV_PATH_ENV: &str = "IGGY_CONNECTORS_ENV_PATH"; pub const CONNECTORS_RUNTIME_ENV_VARS: &[&str] = &[ - "IGGY_CONNECTORS_CONFIG_PATH", - "IGGY_CONNECTORS_ENV_PATH", + CONNECTORS_CONFIG_PATH_ENV, + CONNECTORS_ENV_PATH_ENV, "IGGY_DISPLAY_CONFIG", ]; diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index 286cdade05..746d9387f7 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -33,10 +33,14 @@ pub struct EnvVarMapping { /// /// Array indices are represented by `` in `env_name`. `max_elements` /// contains the expansion limit for each placeholder, from left to right. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct EnvVarTemplate { /// Environment variable name, with `` for each array index. pub env_name: &'static str, + /// Config path, with `` for each array index. + pub config_path: &'static str, + /// Whether this field contains secret data. + pub is_secret: bool, /// Maximum element count for each `` placeholder, from left to right. pub max_elements: &'static [usize], } @@ -76,56 +80,36 @@ pub trait ConfigEnvMappings { } impl EnvVarTemplate { - /// Expands this template into every concrete env-var name it represents, - /// substituting each `` placeholder left-to-right with `0..limit`. - /// A leaf template (`max_elements: &[]`) expands to itself. - pub fn expand_names(&self) -> Vec { - let mut results = vec![self.env_name.to_string()]; + /// Expands this template into every concrete mapping it represents. + pub fn expand(&self) -> Vec { + let mut results = vec![(self.env_name.to_string(), self.config_path.to_string())]; for &limit in self.max_elements { let mut next = Vec::new(); - for name in results { + for (env_name, config_path) in results { for i in 0..limit { - next.push(name.replacen("", &i.to_string(), 1)); + let index = i.to_string(); + next.push(( + env_name.replacen("", &index, 1), + config_path.replacen("", &index, 1), + )); } } results = next; } results + .into_iter() + .map(|(env_name, config_path)| EnvVarMapping { + env_name: Box::leak(env_name.into_boxed_str()), + config_path: Box::leak(config_path.into_boxed_str()), + is_secret: self.is_secret, + }) + .collect() } } -#[cfg(test)] -mod consistency_tests { - use super::*; - use crate::server::ServerConfig; - use std::collections::HashSet; - - /// Asserts that `T::env_templates()`, expanded, names exactly the same - /// set as `T::env_mappings()`. One generic body for every `ConfigEnv` - /// type, so adding a type to check is one call, not one copy-pasted test. - fn assert_templates_and_mappings_align(type_name: &str) { - let expanded: HashSet = T::env_templates() - .iter() - .flat_map(|t| t.expand_names()) - .collect(); - let mapped: HashSet = T::env_mappings() - .iter() - .map(|m| m.env_name.to_string()) - .collect(); - assert_eq!( - expanded, mapped, - "{type_name} env_templates and env_mappings must align" - ); - } - - #[test] - fn server_config_templates_and_mappings_align() { - // ClusterConfig is only ever embedded as a field of ServerConfig - // (never derived or called standalone in production code), so this - // single check already covers its templates and mappings too - - // a separate ClusterConfig-only test would just repeat it. - assert_templates_and_mappings_align::("ServerConfig"); - } +/// Expands compact templates into the mappings consumed by the env provider. +pub fn expand_env_templates(templates: &[EnvVarTemplate]) -> Vec { + templates.iter().flat_map(EnvVarTemplate::expand).collect() } diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs index cef977fad5..b942b34c95 100644 --- a/core/configs/src/configs_impl/mod.rs +++ b/core/configs/src/configs_impl/mod.rs @@ -32,8 +32,11 @@ mod parsing; mod traits; mod typed_env_provider; -pub use env_listing::{CONNECTORS_RUNTIME_ENV_VARS, MCP_RUNTIME_ENV_VARS, print_env_var_names}; -pub use env_mapping::{ConfigEnvMappings, EnvVarMapping, EnvVarTemplate}; +pub use env_listing::{ + CONNECTORS_CONFIG_PATH_ENV, CONNECTORS_ENV_PATH_ENV, CONNECTORS_RUNTIME_ENV_VARS, + MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV, MCP_RUNTIME_ENV_VARS, print_env_var_names, +}; +pub use env_mapping::{ConfigEnvMappings, EnvVarMapping, EnvVarTemplate, expand_env_templates}; pub use error::ConfigurationError; pub use file_provider::{FileConfigProvider, RelocatedKey, RelocatedTarget}; pub use parsing::parse_env_value_to_json; diff --git a/core/configs/src/configs_impl/typed_env_provider.rs b/core/configs/src/configs_impl/typed_env_provider.rs index b6be617da5..55fd7bf1db 100644 --- a/core/configs/src/configs_impl/typed_env_provider.rs +++ b/core/configs/src/configs_impl/typed_env_provider.rs @@ -37,26 +37,21 @@ enum EnvNameResolution<'a> { /// Controls filtering and messaging for unknown env var warnings. enum WarningContext<'a> { - /// Main config: filter IGNORED_ENV_VARS and DELEGATED prefixes + /// Main config: filter runtime env vars and delegated prefixes. MainConfig, /// Connector config: filter PLUGIN_CONFIG_ prefix ConnectorConfig(&'a str), } -/// `IGGY_` variables that are NOT config values: the config file and dotenv -/// paths the connectors runtime and the MCP server read before their config -/// loads. -const IGNORED_ENV_VARS: &[&str] = &[ - "IGGY_CONNECTORS_CONFIG_PATH", - "IGGY_CONNECTORS_ENV_PATH", - "IGGY_MCP_CONFIG_PATH", - "IGGY_MCP_ENV_PATH", -]; - /// Prefixes for env vars handled by separate providers with runtime prefixes. /// The main config provider skips these; each sub-provider validates its own vars. const DELEGATED_ENV_VAR_PREFIXES: &[&str] = &["IGGY_CONNECTORS_SINK_", "IGGY_CONNECTORS_SOURCE_"]; +fn is_runtime_env_var(name: &str) -> bool { + super::CONNECTORS_RUNTIME_ENV_VARS.contains(&name) + || super::MCP_RUNTIME_ENV_VARS.contains(&name) +} + type ProfileMap = FigmentMap; /// Type-safe environment variable provider that uses compile-time generated mappings. @@ -291,7 +286,7 @@ impl TypedEnvProvider { let should_skip = match &context { WarningContext::MainConfig => { - IGNORED_ENV_VARS.contains(&key.as_str()) + is_runtime_env_var(&key) || DELEGATED_ENV_VAR_PREFIXES .iter() .any(|p| key.starts_with(p)) @@ -314,7 +309,7 @@ impl TypedEnvProvider { let debug_msg = match &context { WarningContext::MainConfig => format!( - "Unknown IGGY_ env var: '{}'.{}. Add to IGNORED_ENV_VARS if intentional, \ + "Unknown IGGY_ env var: '{}'.{}. Add to the runtime env-var lists if intentional, \ or add #[derive(ConfigEnv)] to the config struct.", key, suggestion_hint ), @@ -625,19 +620,13 @@ mod tests { #[test] #[serial_test::serial] fn ignored_env_vars_are_skipped_by_the_unknown_variable_scan() { - for name in [ - "IGGY_CONNECTORS_CONFIG_PATH", - "IGGY_CONNECTORS_ENV_PATH", - "IGGY_MCP_CONFIG_PATH", - "IGGY_MCP_ENV_PATH", - ] { - assert!( - IGNORED_ENV_VARS.contains(&name), - "{name} is read by a sibling binary before its config loads, so the scan must skip it" - ); - } + let runtime_env_vars = super::super::CONNECTORS_RUNTIME_ENV_VARS + .iter() + .chain(super::super::MCP_RUNTIME_ENV_VARS) + .copied() + .filter(|name| name.ends_with("_PATH")); - for name in IGNORED_ENV_VARS { + for name in runtime_env_vars.clone() { // SAFETY: the race is process-wide, not per key: `set_var` is unsound // against any concurrent environment access. `serial_test::serial` on // this test is what prevents that. @@ -649,7 +638,7 @@ mod tests { .warn_unknown_env_vars_inner(WarningContext::MainConfig); } - for name in IGNORED_ENV_VARS { + for name in runtime_env_vars { // SAFETY: paired with the set above. unsafe { env::remove_var(name) }; } diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index 5cdaaf8bb5..07b5463bfa 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -23,9 +23,11 @@ mod server_config; pub use common::{COMPONENT, defaults, displays, http, system, validators}; pub use configs_derive::ConfigEnv; pub use configs_impl::{ - CONNECTORS_RUNTIME_ENV_VARS, ConfigEnvMappings, ConfigProvider, ConfigurationError, - ConfigurationType, EnvVarMapping, EnvVarTemplate, FileConfigProvider, MCP_RUNTIME_ENV_VARS, - RelocatedKey, RelocatedTarget, TypedEnvProvider, parse_env_value_to_json, print_env_var_names, + CONNECTORS_CONFIG_PATH_ENV, CONNECTORS_ENV_PATH_ENV, CONNECTORS_RUNTIME_ENV_VARS, + ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, + EnvVarTemplate, FileConfigProvider, MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV, + MCP_RUNTIME_ENV_VARS, RelocatedKey, RelocatedTarget, TypedEnvProvider, expand_env_templates, + parse_env_value_to_json, print_env_var_names, }; pub use server_config::{ cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, diff --git a/core/configs_derive/src/config_env.rs b/core/configs_derive/src/config_env.rs index cd7345624b..97d801265f 100644 --- a/core/configs_derive/src/config_env.rs +++ b/core/configs_derive/src/config_env.rs @@ -180,18 +180,7 @@ fn generate_enum_impl( }) .collect(); - let tag_mapping = tag.as_deref().map(|tag_name: &str| { - let env_segment = tag_name.to_uppercase(); - quote! { - all_mappings.push(configs::EnvVarMapping { - env_name: #env_segment, - config_path: #tag_name, - is_secret: false, - }); - } - }); - - if variant_types.is_empty() && tag_mapping.is_none() { + if variant_types.is_empty() && tag.is_none() { return quote! { impl #impl_generics configs::ConfigEnvMappings for #enum_name #ty_generics #where_clause { fn env_mappings() -> &'static [configs::EnvVarMapping] { @@ -205,16 +194,6 @@ fn generate_enum_impl( }; } - // Generate code to extend from each variant type - let extends: Vec = variant_types - .iter() - .map(|ty| { - quote! { - all_mappings.extend_from_slice(<#ty as configs::ConfigEnvMappings>::env_mappings()); - } - }) - .collect(); - let template_extends: Vec = variant_types .iter() .map(|ty| quote! { @@ -226,6 +205,8 @@ fn generate_enum_impl( quote! { all_templates.push(configs::EnvVarTemplate { env_name: #env_segment, + config_path: #tag_name, + is_secret: false, max_elements: &[], }); } @@ -235,12 +216,7 @@ fn generate_enum_impl( impl #impl_generics configs::ConfigEnvMappings for #enum_name #ty_generics #where_clause { fn env_mappings() -> &'static [configs::EnvVarMapping] { static MAPPINGS: std::sync::OnceLock> = std::sync::OnceLock::new(); - MAPPINGS.get_or_init(|| { - let mut all_mappings: Vec = Vec::new(); - #tag_mapping - #(#extends)* - all_mappings - }) + MAPPINGS.get_or_init(|| configs::expand_env_templates(Self::env_templates())) } fn env_templates() -> &'static [configs::EnvVarTemplate] { @@ -285,137 +261,53 @@ fn generate_struct_impl( let (mappings, nested_fields) = collect_mappings(&fields); let const_definitions = generate_const_definitions(&mappings, prefix_str); - let mapping_entries = generate_mapping_entries(&mappings); let builder_methods = generate_builder_methods(&mappings, prefix_str); let builder_name = format_ident!("{}EnvBuilder", struct_name); - let mappings_count = mappings.len(); - - // Generate code to include nested type mappings - let nested_extends: Vec = nested_fields - .iter() - .map(|info| { - let ty = &info.element_type; - let field_env_segment = &info.field_env_segment; - let field_name = &info.field_name; - - if info.is_vec { - // For Vec fields, expand with array indices - let max_elements = info.max_elements; - quote! { - let nested_mappings = <#ty as configs::ConfigEnvMappings>::env_mappings(); - for i in 0..#max_elements { - for mapping in nested_mappings { - // Transform env_name by prepending FIELD_INDEX_ - let new_env_name: &'static str = Box::leak( - format!("{}_{}_{}", #field_env_segment, i, mapping.env_name).into_boxed_str() - ); - // Transform config_path by prepending field.index. - let new_config_path: &'static str = Box::leak( - format!("{}.{}.{}", #field_name, i, mapping.config_path).into_boxed_str() - ); - all_mappings.push(configs::EnvVarMapping { - env_name: new_env_name, - config_path: new_config_path, - is_secret: mapping.is_secret, - }); - } - } - } - } else { - // For non-Vec nested types, transform by prepending field name - quote! { - let nested_mappings = <#ty as configs::ConfigEnvMappings>::env_mappings(); - for mapping in nested_mappings { - // Transform env_name by prepending FIELD_ - let new_env_name: &'static str = Box::leak( - format!("{}_{}", #field_env_segment, mapping.env_name).into_boxed_str() - ); - // Transform config_path by prepending field. - let new_config_path: &'static str = Box::leak( - format!("{}.{}", #field_name, mapping.config_path).into_boxed_str() - ); - all_mappings.push(configs::EnvVarMapping { - env_name: new_env_name, - config_path: new_config_path, - is_secret: mapping.is_secret, - }); - } - } - } - }) - .collect(); - - let prefix_application = if has_prefix { - quote! { - let all_mappings: Vec = all_mappings - .into_iter() - .map(|m| configs::EnvVarMapping { - env_name: Box::leak(format!("{}{}", #prefix_str, m.env_name).into_boxed_str()), - config_path: m.config_path, - is_secret: m.is_secret, - }) - .collect(); - } - } else { - quote! {} - }; - - // If there are nested types, we need to use OnceLock to combine mappings - let env_mappings_impl = if nested_fields.is_empty() && !has_prefix { - // Simple case: no nested fields, no prefix transformation needed - quote! { - fn env_mappings() -> &'static [configs::EnvVarMapping] { - static MAPPINGS: [configs::EnvVarMapping; #mappings_count] = [ - #(#mapping_entries),* - ]; - &MAPPINGS - } - } - } else { - // Complex case: need OnceLock for dynamic construction - quote! { - fn env_mappings() -> &'static [configs::EnvVarMapping] { - static MAPPINGS: std::sync::OnceLock> = std::sync::OnceLock::new(); - MAPPINGS.get_or_init(|| { - let own_mappings: [configs::EnvVarMapping; #mappings_count] = [ - #(#mapping_entries),* - ]; - let mut all_mappings = Vec::from(own_mappings); - #(#nested_extends)* - #prefix_application - all_mappings - }) - } - } - }; let own_template_entries = mappings.iter().map(|mapping| { let env_suffix = &mapping.env_suffix; + let config_path = &mapping.config_path; + let is_secret = mapping.is_secret; quote! { - configs::EnvVarTemplate { env_name: #env_suffix, max_elements: &[] } + configs::EnvVarTemplate { + env_name: #env_suffix, + config_path: #config_path, + is_secret: #is_secret, + max_elements: &[], + } } }); let nested_template_extends = nested_fields.iter().map(|info| { let ty = &info.element_type; let segment = &info.field_env_segment; + let field_name = &info.field_name; if info.is_vec { let max_elements = info.max_elements; quote! { for template in <#ty as configs::ConfigEnvMappings>::env_templates() { let env_name = Box::leak(format!("{}__{}", #segment, template.env_name).into_boxed_str()); + let config_path = Box::leak(format!("{}..{}", #field_name, template.config_path).into_boxed_str()); let limits = Box::leak(std::iter::once(#max_elements) .chain(template.max_elements.iter().copied()) .collect::>() .into_boxed_slice()); - all_templates.push(configs::EnvVarTemplate { env_name, max_elements: limits }); + all_templates.push(configs::EnvVarTemplate { + env_name, + config_path, + is_secret: template.is_secret, + max_elements: limits, + }); } } } else { quote! { for template in <#ty as configs::ConfigEnvMappings>::env_templates() { let env_name = Box::leak(format!("{}_{}", #segment, template.env_name).into_boxed_str()); + let config_path = Box::leak(format!("{}.{}", #field_name, template.config_path).into_boxed_str()); all_templates.push(configs::EnvVarTemplate { env_name, + config_path, + is_secret: template.is_secret, max_elements: template.max_elements, }); } @@ -444,7 +336,10 @@ fn generate_struct_impl( } impl #impl_generics configs::ConfigEnvMappings for #struct_name #ty_generics #where_clause { - #env_mappings_impl + fn env_mappings() -> &'static [configs::EnvVarMapping] { + static MAPPINGS: std::sync::OnceLock> = std::sync::OnceLock::new(); + MAPPINGS.get_or_init(|| configs::expand_env_templates(Self::env_templates())) + } fn env_templates() -> &'static [configs::EnvVarTemplate] { static TEMPLATES: std::sync::OnceLock> = std::sync::OnceLock::new(); @@ -646,25 +541,6 @@ fn generate_const_definitions(mappings: &[EnvMapping], prefix: &str) -> Vec Vec { - mappings - .iter() - .map(|m| { - let env_suffix = &m.env_suffix; - let config_path = &m.config_path; - let is_secret = m.is_secret; - - quote! { - configs::EnvVarMapping { - env_name: #env_suffix, - config_path: #config_path, - is_secret: #is_secret, - } - } - }) - .collect() -} - fn generate_builder_methods(mappings: &[EnvMapping], prefix: &str) -> Vec { mappings .iter() diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index 3b94247d2f..af14a411fc 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -21,7 +21,8 @@ use crate::configs::connectors::{ }; use crate::metrics::ConnectorType; use ::configs::{ - CONNECTORS_RUNTIME_ENV_VARS, ConfigEnvMappings, ConfigProvider, print_env_var_names, + CONNECTORS_CONFIG_PATH_ENV, CONNECTORS_ENV_PATH_ENV, CONNECTORS_RUNTIME_ENV_VARS, + ConfigEnvMappings, ConfigProvider, print_env_var_names, }; use clap::Parser; use configs::connectors::ConfigFormat; @@ -74,6 +75,7 @@ static GLOBAL: MiMalloc = MiMalloc; struct Args { #[arg( long, + help = "Print supported configuration environment variables and exit", long_help = r#"Print supported configuration environment variables and exit. Lists all supported IGGY_* environment variable names and templates, @@ -151,25 +153,23 @@ fn main() -> Result<(), RuntimeError> { } fn print_config_env_vars() -> std::io::Result<()> { - let sink_source_templates: Vec = [ + let sink_source_templates = [ ("SINK", SinkConfig::env_templates()), ("SOURCE", SourceConfig::env_templates()), ] - .iter() + .into_iter() .flat_map(|(kind, templates)| { // "" stands in for the real, per-connector key `local_provider` // uppercases at runtime - same prefix rule, so the listing can't // drift from the names the runtime actually reads. let prefix = crate::configs::connectors::local_provider::connector_env_prefix(kind, ""); - let enabled = format!("{prefix}ENABLED"); let plugin_config = format!("{prefix}PLUGIN_CONFIG_"); templates .iter() .map(move |template| format!("{prefix}{}", template.env_name)) - .chain([enabled, plugin_config]) - }) - .collect(); + .chain(std::iter::once(plugin_config)) + }); let names = ConnectorsRuntimeConfig::env_templates() .iter() @@ -184,7 +184,7 @@ fn print_config_env_vars() -> std::io::Result<()> { async fn run() -> Result<(), RuntimeError> { print_ascii_art("Iggy Connectors"); - if let Ok(env_path) = std::env::var("IGGY_CONNECTORS_ENV_PATH") { + if let Ok(env_path) = std::env::var(CONNECTORS_ENV_PATH_ENV) { if dotenvy::from_path(&env_path).is_ok() { println!("Loaded environment variables from path: {env_path}"); } @@ -196,7 +196,7 @@ async fn run() -> Result<(), RuntimeError> { } let config_path = - env::var("IGGY_CONNECTORS_CONFIG_PATH").unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); + env::var(CONNECTORS_CONFIG_PATH_ENV).unwrap_or_else(|_| DEFAULT_CONFIG_PATH.to_string()); println!("Starting Iggy Connectors Runtime, loading configuration from: {config_path}..."); let config: ConnectorsRuntimeConfig = ConnectorsRuntimeConfig::config_provider(config_path) diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs index 000b58ede4..7c61e09b18 100644 --- a/core/integration/tests/config_env_listing/mod.rs +++ b/core/integration/tests/config_env_listing/mod.rs @@ -75,22 +75,43 @@ fn run_list_config_env_vars_plain(binary: &str, extra_args: &[&str]) -> std::pro #[test] fn config_env_listing_exits_before_startup_for_each_binary() { - for (binary, config_env, dotenv_env) in [ - ("iggy-server", "IGGY_CONFIG_PATH", "IGGY_ENV_PATH"), + for (binary, config_env, dotenv_env, invalid_env) in [ + ( + "iggy-server", + "IGGY_CONFIG_PATH", + "IGGY_ENV_PATH", + "IGGY_HTTP_ADDRESS", + ), ( "iggy-connectors", "IGGY_CONNECTORS_CONFIG_PATH", "IGGY_CONNECTORS_ENV_PATH", + "IGGY_CONNECTORS_HTTP_ADDRESS", + ), + ( + "iggy-mcp", + "IGGY_MCP_CONFIG_PATH", + "IGGY_MCP_ENV_PATH", + "IGGY_MCP_HTTP_ADDRESS", ), - ("iggy-mcp", "IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"), ] { let directory = tempfile::tempdir().expect("temporary directory"); + let config_path = directory.path().join("invalid-config.toml"); + let dotenv_path = directory.path().join("invalid.env"); + std::fs::write(&config_path, "not valid toml = [").expect("write invalid config"); + std::fs::write(&dotenv_path, "not a dotenv assignment").expect("write invalid dotenv"); + + let entries_before: HashSet<_> = std::fs::read_dir(directory.path()) + .expect("read temporary directory") + .map(|entry| entry.expect("directory entry").file_name()) + .collect(); let output = run_list_config_env_vars_in( binary, directory.path(), [ - (config_env, directory.path().join("missing-config.toml")), - (dotenv_env, directory.path().join("missing.env")), + (config_env, config_path.into_os_string()), + (dotenv_env, dotenv_path.into_os_string()), + (invalid_env, "invalid-value".into()), ], &[], ); @@ -103,6 +124,20 @@ fn config_env_listing_exits_before_startup_for_each_binary() { ); assert!(output.stderr.is_empty(), "{binary} wrote to stderr"); + let baseline = run_list_config_env_vars_plain(binary, &[]); + assert_eq!( + output.stdout, baseline.stdout, + "{binary} output changed with invalid config, dotenv or env values" + ); + let entries_after: HashSet<_> = std::fs::read_dir(directory.path()) + .expect("read temporary directory") + .map(|entry| entry.expect("directory entry").file_name()) + .collect(); + assert_eq!( + entries_after, entries_before, + "{binary} created startup side effects" + ); + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); let names: Vec<_> = stdout.lines().collect(); assert!(!names.is_empty(), "{binary} returned no variables"); @@ -177,6 +212,14 @@ fn config_env_listing_includes_connector_templates() { .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__PLUGIN_CONFIG_")), "connectors should list SOURCE plugin config templates" ); + assert!( + names.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_FORMAT"), + "connectors should list the typed SINK plugin config format" + ); + assert!( + names.contains("IGGY_CONNECTORS_SOURCE__PLUGIN_CONFIG_FORMAT"), + "connectors should list the typed SOURCE plugin config format" + ); // Verify SINK and SOURCE ENABLED fields are present assert!( @@ -191,6 +234,55 @@ fn config_env_listing_includes_connector_templates() { .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__ENABLED")), "connectors should list SOURCE__ENABLED" ); + + assert_eq!( + stdout + .lines() + .filter(|name| *name == "IGGY_CONNECTORS_CONNECTORS_CONFIG_TYPE") + .count(), + 1, + "enum variants should produce one deduplicated tag name" + ); +} + +#[test] +fn config_env_listing_excludes_other_processes_variables() { + let server = run_list_config_env_vars_plain("iggy-server", &[]); + let server = String::from_utf8(server.stdout).expect("UTF-8 output"); + let server_names: HashSet<_> = server.lines().collect(); + for excluded in [ + "IGGY_CI_BUILD", + "IGGY_HOME", + "IGGY_PASSWORD", + "IGGY_TEST_CLEANUP_DISABLED", + "IGGY_TEST_CLUSTER_NODES", + "IGGY_TEST_VERBOSE", + "IGGY_USERNAME", + ] { + assert!(!server_names.contains(excluded), "server listed {excluded}"); + } + assert!( + server_names + .iter() + .all(|name| !name.starts_with("IGGY_CONNECTORS_") + && !name.starts_with("IGGY_KAFKA_") + && !name.starts_with("IGGY_MCP_")), + "server listed a sibling binary's variables" + ); + + for (binary, prefix) in [ + ("iggy-connectors", "IGGY_CONNECTORS_"), + ("iggy-mcp", "IGGY_MCP_"), + ] { + let output = run_list_config_env_vars_plain(binary, &[]); + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + assert!( + stdout + .lines() + .all(|name| name == "IGGY_DISPLAY_CONFIG" || name.starts_with(prefix)), + "{binary} listed a variable outside {prefix}" + ); + } } #[test] diff --git a/core/server/src/args.rs b/core/server/src/args.rs index b972001a46..bd8a4c2341 100644 --- a/core/server/src/args.rs +++ b/core/server/src/args.rs @@ -89,6 +89,7 @@ For more information, visit: https://iggy.apache.org/docs/introduction/getting-s pub struct Args { #[arg( long, + help = "Print supported configuration environment variables and exit", long_help = r#"Print supported configuration environment variables and exit. Lists all supported IGGY_* environment variable names and templates, From af7e26ebfe26d8b75c5f308b2e981d5a3609544d Mon Sep 17 00:00:00 2001 From: ParthibanRajasekaran Date: Sun, 4 Oct 2026 19:32:31 +0100 Subject: [PATCH 27/27] Address config env listing review feedback --- core/ai/mcp/README.md | 4 + core/configs/src/configs_impl/env_listing.rs | 57 +++++- core/configs/src/configs_impl/env_mapping.rs | 7 +- .../configs/src/configs_impl/file_provider.rs | 11 +- core/configs/src/configs_impl/mod.rs | 3 +- .../src/configs_impl/typed_env_provider.rs | 23 ++- core/configs/src/lib.rs | 4 +- core/configs/src/server_config/cluster.rs | 9 + core/configs/src/server_config/server.rs | 64 +++--- core/configs_derive/src/config_env.rs | 72 +++---- core/connectors/runtime/README.md | 4 + .../src/configs/connectors/local_provider.rs | 6 +- core/connectors/runtime/src/error.rs | 4 + core/connectors/runtime/src/main.rs | 13 +- .../integration/src/harness/config/resolve.rs | 15 +- .../tests/config_env_listing/mod.rs | 191 ++++++------------ 16 files changed, 259 insertions(+), 228 deletions(-) diff --git a/core/ai/mcp/README.md b/core/ai/mcp/README.md index 12ed06259a..3b9754a86a 100644 --- a/core/ai/mcp/README.md +++ b/core/ai/mcp/README.md @@ -58,10 +58,14 @@ The configuration file must use TOML. The default path is `core/ai/mcp/config.to Set `IGGY_MCP_ENV_PATH` to load a particular dotenv file. Otherwise `.env` is searched for in the current directory and its parents. Existing environment variables take precedence over dotenv values. +Run `iggy-mcp --list-config-env-vars` to print the supported configuration environment variables and exit before loading dotenv or configuration files or creating the runtime. + A non-empty `iggy.token` takes precedence over username and password. It accepts a literal PAT or a `file:` reference such as `file:/run/secrets/iggy_pat`; file contents are trimmed and a leading `~/` expands to the home directory. Set `command` to the absolute path of the built executable. This Claude Desktop example uses the development broker credentials: +`iggy-mcp` rejects unknown command-line arguments with exit code 2. Keep the client or container `args` list empty unless it contains a supported flag. + ```json { "mcpServers": { diff --git a/core/configs/src/configs_impl/env_listing.rs b/core/configs/src/configs_impl/env_listing.rs index 8ac8c74a59..17512ab90d 100644 --- a/core/configs/src/configs_impl/env_listing.rs +++ b/core/configs/src/configs_impl/env_listing.rs @@ -41,19 +41,62 @@ where Ok(()) } -/// Env vars `iggy-mcp --list-config-env-vars` advertises beyond the derived -/// `McpServerConfig` templates. pub const MCP_CONFIG_PATH_ENV: &str = "IGGY_MCP_CONFIG_PATH"; pub const MCP_ENV_PATH_ENV: &str = "IGGY_MCP_ENV_PATH"; -pub const MCP_RUNTIME_ENV_VARS: &[&str] = - &["IGGY_DISPLAY_CONFIG", MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV]; +/// Env vars `iggy-mcp --list-config-env-vars` advertises beyond the derived +/// `McpServerConfig` templates. +pub const MCP_RUNTIME_ENV_VARS: &[&str] = &[ + super::file_provider::DISPLAY_CONFIG_ENV, + MCP_CONFIG_PATH_ENV, + MCP_ENV_PATH_ENV, +]; -/// Env vars `iggy-connectors --list-config-env-vars` advertises beyond the -/// derived `ConnectorsRuntimeConfig` templates. pub const CONNECTORS_CONFIG_PATH_ENV: &str = "IGGY_CONNECTORS_CONFIG_PATH"; pub const CONNECTORS_ENV_PATH_ENV: &str = "IGGY_CONNECTORS_ENV_PATH"; +/// Env vars `iggy-connectors --list-config-env-vars` advertises beyond the +/// derived `ConnectorsRuntimeConfig` templates. pub const CONNECTORS_RUNTIME_ENV_VARS: &[&str] = &[ CONNECTORS_CONFIG_PATH_ENV, CONNECTORS_ENV_PATH_ENV, - "IGGY_DISPLAY_CONFIG", + super::file_provider::DISPLAY_CONFIG_ENV, ]; + +/// Segment between a connector env-var prefix and a plugin config field. +pub const PLUGIN_CONFIG_ENV_SEGMENT: &str = "PLUGIN_CONFIG_"; + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Error, ErrorKind, Write}; + + struct FailingWriter(ErrorKind); + + impl Write for FailingWriter { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(Error::from(self.0)) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn print_env_var_names_sorts_and_deduplicates() { + let mut output = Vec::new(); + print_env_var_names(["IGGY_B", "IGGY_A", "IGGY_B"], &mut output).unwrap(); + assert_eq!(output, b"IGGY_A\nIGGY_B\n"); + } + + #[test] + fn print_env_var_names_accepts_a_closed_pipe() { + assert!(print_env_var_names(["IGGY_A"], &mut FailingWriter(ErrorKind::BrokenPipe)).is_ok()); + } + + #[test] + fn print_env_var_names_propagates_other_write_errors() { + let error = print_env_var_names(["IGGY_A"], &mut FailingWriter(ErrorKind::StorageFull)) + .expect_err("storage-full error must be propagated"); + assert_eq!(error.kind(), ErrorKind::StorageFull); + } +} diff --git a/core/configs/src/configs_impl/env_mapping.rs b/core/configs/src/configs_impl/env_mapping.rs index 746d9387f7..673e9892fc 100644 --- a/core/configs/src/configs_impl/env_mapping.rs +++ b/core/configs/src/configs_impl/env_mapping.rs @@ -80,8 +80,7 @@ pub trait ConfigEnvMappings { } impl EnvVarTemplate { - /// Expands this template into every concrete mapping it represents. - pub fn expand(&self) -> Vec { + fn expand(&self) -> Vec { let mut results = vec![(self.env_name.to_string(), self.config_path.to_string())]; for &limit in self.max_elements { @@ -110,6 +109,10 @@ impl EnvVarTemplate { } /// Expands compact templates into the mappings consumed by the env provider. +/// +/// The returned mappings own leaked names and paths so generated +/// `ConfigEnvMappings` implementations can cache them in a `OnceLock` and +/// return `'static` references. Call this once per config type. pub fn expand_env_templates(templates: &[EnvVarTemplate]) -> Vec { templates.iter().flat_map(EnvVarTemplate::expand).collect() } diff --git a/core/configs/src/configs_impl/file_provider.rs b/core/configs/src/configs_impl/file_provider.rs index 851000303a..33b844fb78 100644 --- a/core/configs/src/configs_impl/file_provider.rs +++ b/core/configs/src/configs_impl/file_provider.rs @@ -26,7 +26,7 @@ use figment::{ use std::{env, path::Path}; use tracing::{error, info, warn}; -const DISPLAY_CONFIG_ENV: &str = "IGGY_DISPLAY_CONFIG"; +pub(crate) const DISPLAY_CONFIG_ENV: &str = "IGGY_DISPLAY_CONFIG"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RelocatedTarget { @@ -448,7 +448,7 @@ mod tests { let unknown = unknown_env_names( names(&siblings).into_iter(), "IGGY_", - crate::server_config::server::SERVER_PROCESS_ENV_VARS, + &crate::server_config::server::server_process_env_vars().collect::>(), crate::server_config::server::SERVER_ALLOWED_ENV_PREFIXES, ); assert!( @@ -483,12 +483,13 @@ mod tests { } /// The server reads these variables outside its config, so the boot check - /// must accept them. Without the `SERVER_PROCESS_ENV_VARS` chain in + /// must accept them. Without the `server_process_env_vars()` chain in /// `ServerConfig::config_provider`, a debug build refuses to boot. #[test] fn given_the_server_process_variables_when_checking_then_the_server_should_boot() { - let unknown = - server_unknown_env_names(crate::server_config::server::SERVER_PROCESS_ENV_VARS); + let process_env_vars = + crate::server_config::server::server_process_env_vars().collect::>(); + let unknown = server_unknown_env_names(&process_env_vars); assert!( unknown.is_empty(), diff --git a/core/configs/src/configs_impl/mod.rs b/core/configs/src/configs_impl/mod.rs index b942b34c95..a9bf2bf91c 100644 --- a/core/configs/src/configs_impl/mod.rs +++ b/core/configs/src/configs_impl/mod.rs @@ -34,7 +34,8 @@ mod typed_env_provider; pub use env_listing::{ CONNECTORS_CONFIG_PATH_ENV, CONNECTORS_ENV_PATH_ENV, CONNECTORS_RUNTIME_ENV_VARS, - MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV, MCP_RUNTIME_ENV_VARS, print_env_var_names, + MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV, MCP_RUNTIME_ENV_VARS, PLUGIN_CONFIG_ENV_SEGMENT, + print_env_var_names, }; pub use env_mapping::{ConfigEnvMappings, EnvVarMapping, EnvVarTemplate, expand_env_templates}; pub use error::ConfigurationError; diff --git a/core/configs/src/configs_impl/typed_env_provider.rs b/core/configs/src/configs_impl/typed_env_provider.rs index 55fd7bf1db..74dd06324b 100644 --- a/core/configs/src/configs_impl/typed_env_provider.rs +++ b/core/configs/src/configs_impl/typed_env_provider.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use super::PLUGIN_CONFIG_ENV_SEGMENT; use super::env_mapping::ConfigEnvMappings; use super::error::ConfigurationError; use super::parsing::parse_env_value; @@ -292,7 +293,7 @@ impl TypedEnvProvider { .any(|p| key.starts_with(p)) } WarningContext::ConnectorConfig(prefix) => { - let plugin_config_prefix = format!("{}PLUGIN_CONFIG_", prefix); + let plugin_config_prefix = format!("{}{PLUGIN_CONFIG_ENV_SEGMENT}", prefix); key.starts_with(&plugin_config_prefix) } }; @@ -620,13 +621,21 @@ mod tests { #[test] #[serial_test::serial] fn ignored_env_vars_are_skipped_by_the_unknown_variable_scan() { - let runtime_env_vars = super::super::CONNECTORS_RUNTIME_ENV_VARS - .iter() - .chain(super::super::MCP_RUNTIME_ENV_VARS) - .copied() - .filter(|name| name.ends_with("_PATH")); + let runtime_env_vars = [ + "IGGY_CONNECTORS_CONFIG_PATH", + "IGGY_CONNECTORS_ENV_PATH", + "IGGY_MCP_CONFIG_PATH", + "IGGY_MCP_ENV_PATH", + ]; + + for name in runtime_env_vars { + assert!( + is_runtime_env_var(name), + "{name} is read before config loading and must be skipped by the scan" + ); + } - for name in runtime_env_vars.clone() { + for name in runtime_env_vars { // SAFETY: the race is process-wide, not per key: `set_var` is unsound // against any concurrent environment access. `serial_test::serial` on // this test is what prevents that. diff --git a/core/configs/src/lib.rs b/core/configs/src/lib.rs index 07b5463bfa..8e266121bb 100644 --- a/core/configs/src/lib.rs +++ b/core/configs/src/lib.rs @@ -26,8 +26,8 @@ pub use configs_impl::{ CONNECTORS_CONFIG_PATH_ENV, CONNECTORS_ENV_PATH_ENV, CONNECTORS_RUNTIME_ENV_VARS, ConfigEnvMappings, ConfigProvider, ConfigurationError, ConfigurationType, EnvVarMapping, EnvVarTemplate, FileConfigProvider, MCP_CONFIG_PATH_ENV, MCP_ENV_PATH_ENV, - MCP_RUNTIME_ENV_VARS, RelocatedKey, RelocatedTarget, TypedEnvProvider, expand_env_templates, - parse_env_value_to_json, print_env_var_names, + MCP_RUNTIME_ENV_VARS, PLUGIN_CONFIG_ENV_SEGMENT, RelocatedKey, RelocatedTarget, + TypedEnvProvider, expand_env_templates, parse_env_value_to_json, print_env_var_names, }; pub use server_config::{ cluster, message_bus, metadata, partition, quic, server, sharding, tcp, websocket, diff --git a/core/configs/src/server_config/cluster.rs b/core/configs/src/server_config/cluster.rs index eb1e603a46..140847951a 100644 --- a/core/configs/src/server_config/cluster.rs +++ b/core/configs/src/server_config/cluster.rs @@ -1578,6 +1578,15 @@ mod tests { .expect("advertised address template"); assert_eq!(template.max_elements, &[256, 16]); + + let mapping = ::env_mappings() + .iter() + .find(|mapping| mapping.env_name == "NODES_3_ADVERTISED_ADDRESSES_7_CLIENT_CIDR") + .expect("expanded advertised address mapping"); + assert_eq!( + mapping.config_path, + "nodes.3.advertised_addresses.7.client_cidr" + ); } } diff --git a/core/configs/src/server_config/server.rs b/core/configs/src/server_config/server.rs index ab04c74401..5963203aa0 100644 --- a/core/configs/src/server_config/server.rs +++ b/core/configs/src/server_config/server.rs @@ -47,11 +47,9 @@ pub use crate::common::server::{ TelemetryConfig, TelemetryLogsConfig, TelemetryTracesConfig, TelemetryTransport, }; -/// Vars used by sibling binaries (iggy CLI) or test/CI-only: never advertised -/// via `--list-config-env-vars`, but still non-config. Included in -/// `SERVER_PROCESS_ENV_VARS` to suppress "unknown env var" warnings during -/// config scanning. -pub const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ +/// Vars used by sibling binaries (iggy CLI) or test/CI-only. These suppress +/// unknown-name warnings but are not advertised by the server. +const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ "IGGY_TEST_VERBOSE", "IGGY_TEST_CLUSTER_NODES", "IGGY_TEST_CLEANUP_DISABLED", @@ -61,33 +59,28 @@ pub const SERVER_SCAN_ONLY_ENV_VARS: &[&str] = &[ "IGGY_PASSWORD", ]; -/// All non-config env vars the server process scans for. -pub const SERVER_PROCESS_ENV_VARS: &[&str] = &[ +/// Non-config env vars supported by the server and advertised to operators. +const SERVER_RUNTIME_ENV_VARS: &[&str] = &[ "IGGY_CONFIG_PATH", "IGGY_ENV_PATH", "IGGY_DISPLAY_CONFIG", "IGGY_ROOT_USERNAME", "IGGY_ROOT_PASSWORD", - "IGGY_TEST_VERBOSE", - "IGGY_TEST_CLUSTER_NODES", - "IGGY_TEST_CLEANUP_DISABLED", "IGGY_SHARD_RUNTIME_CAPACITY", "IGGY_SHARD_EVENT_INTERVAL", - "IGGY_CI_BUILD", - "IGGY_HOME", - "IGGY_USERNAME", - "IGGY_PASSWORD", ]; -/// Vars safe to advertise to end users via `--list-config-env-vars` — every -/// entry in `SERVER_PROCESS_ENV_VARS` except the sibling-binary/test-only -/// ones in `SERVER_SCAN_ONLY_ENV_VARS`. Computed rather than retyped, so the -/// two views cannot drift apart. +/// Non-config vars advertised by `iggy-server --list-config-env-vars`. pub fn server_runtime_env_vars() -> impl Iterator { - SERVER_PROCESS_ENV_VARS + SERVER_RUNTIME_ENV_VARS.iter().copied() +} + +/// All non-config vars accepted by the server's unknown-name scan. +pub fn server_process_env_vars() -> impl Iterator { + SERVER_RUNTIME_ENV_VARS .iter() + .chain(SERVER_SCAN_ONLY_ENV_VARS) .copied() - .filter(|name| !SERVER_SCAN_ONLY_ENV_VARS.contains(name)) } pub(crate) const SERVER_ALLOWED_ENV_PREFIXES: &[&str] = @@ -288,7 +281,7 @@ impl ServerConfig { .with_known_env_names( Self::all_env_var_names() .into_iter() - .chain(SERVER_PROCESS_ENV_VARS.iter().copied()) + .chain(server_process_env_vars()) .collect(), ) .with_allowed_env_prefixes(SERVER_ALLOWED_ENV_PREFIXES) @@ -494,7 +487,7 @@ mod tests { #[test] #[serial_test::serial] fn env_provider_accepts_server_process_env_vars() { - for name in SERVER_PROCESS_ENV_VARS { + for name in server_process_env_vars() { // SAFETY: the race is process-wide, not per key: `set_var` is unsound // against any concurrent environment access. `serial_test::serial` on // this test is what prevents that. @@ -503,7 +496,7 @@ mod tests { let data = ServerConfigEnvProvider::default().data(); - for name in SERVER_PROCESS_ENV_VARS { + for name in server_process_env_vars() { // SAFETY: paired with the set above. unsafe { env::remove_var(name) }; } @@ -518,4 +511,29 @@ mod tests { "none of these variables is a config value, so none of them may reach the map: {profile:?}" ); } + + #[test] + fn server_process_env_vars_include_every_scan_only_name() { + let process_names = server_process_env_vars().collect::>(); + for name in [ + "IGGY_TEST_VERBOSE", + "IGGY_TEST_CLUSTER_NODES", + "IGGY_TEST_CLEANUP_DISABLED", + "IGGY_CI_BUILD", + "IGGY_HOME", + "IGGY_USERNAME", + "IGGY_PASSWORD", + ] { + assert!( + process_names.contains(name), + "missing scan-only name {name}" + ); + } + assert!( + SERVER_RUNTIME_ENV_VARS + .iter() + .all(|name| !SERVER_SCAN_ONLY_ENV_VARS.contains(name)), + "advertised and scan-only env vars must stay disjoint" + ); + } } diff --git a/core/configs_derive/src/config_env.rs b/core/configs_derive/src/config_env.rs index 97d801265f..176efe0f02 100644 --- a/core/configs_derive/src/config_env.rs +++ b/core/configs_derive/src/config_env.rs @@ -243,8 +243,6 @@ fn generate_struct_impl( fields: Vec, ) -> TokenStream2 { let prefix_str = prefix.as_deref().unwrap_or(""); - let has_prefix = !prefix_str.is_empty(); - // Metadata name: use provided or derive from type name (e.g., "ServerConfig" -> "server-config") let metadata_name = name.unwrap_or_else(|| { let type_name = struct_name.to_string(); @@ -265,12 +263,12 @@ fn generate_struct_impl( let builder_name = format_ident!("{}EnvBuilder", struct_name); let own_template_entries = mappings.iter().map(|mapping| { - let env_suffix = &mapping.env_suffix; + let env_name = format!("{}{}", prefix_str, mapping.env_suffix); let config_path = &mapping.config_path; let is_secret = mapping.is_secret; quote! { configs::EnvVarTemplate { - env_name: #env_suffix, + env_name: #env_name, config_path: #config_path, is_secret: #is_secret, max_elements: &[], @@ -279,50 +277,41 @@ fn generate_struct_impl( }); let nested_template_extends = nested_fields.iter().map(|info| { let ty = &info.element_type; - let segment = &info.field_env_segment; - let field_name = &info.field_name; - if info.is_vec { + let env_segment = if info.is_vec { + format!("{}{}_", prefix_str, info.field_env_segment) + } else { + format!("{}{}", prefix_str, info.field_env_segment) + }; + let config_segment = if info.is_vec { + format!("{}.", info.field_name) + } else { + info.field_name.clone() + }; + let limits = if info.is_vec { let max_elements = info.max_elements; quote! { - for template in <#ty as configs::ConfigEnvMappings>::env_templates() { - let env_name = Box::leak(format!("{}__{}", #segment, template.env_name).into_boxed_str()); - let config_path = Box::leak(format!("{}..{}", #field_name, template.config_path).into_boxed_str()); - let limits = Box::leak(std::iter::once(#max_elements) - .chain(template.max_elements.iter().copied()) - .collect::>() - .into_boxed_slice()); - all_templates.push(configs::EnvVarTemplate { - env_name, - config_path, - is_secret: template.is_secret, - max_elements: limits, - }); - } + Box::leak(std::iter::once(#max_elements) + .chain(template.max_elements.iter().copied()) + .collect::>() + .into_boxed_slice()) } } else { - quote! { - for template in <#ty as configs::ConfigEnvMappings>::env_templates() { - let env_name = Box::leak(format!("{}_{}", #segment, template.env_name).into_boxed_str()); - let config_path = Box::leak(format!("{}.{}", #field_name, template.config_path).into_boxed_str()); - all_templates.push(configs::EnvVarTemplate { - env_name, - config_path, - is_secret: template.is_secret, - max_elements: template.max_elements, - }); - } - } - } - }); - let template_prefix_application = if has_prefix { + quote! { template.max_elements } + }; quote! { - for template in &mut all_templates { - template.env_name = Box::leak(format!("{}{}", #prefix_str, template.env_name).into_boxed_str()); + for template in <#ty as configs::ConfigEnvMappings>::env_templates() { + let env_name = Box::leak(format!("{}_{}", #env_segment, template.env_name).into_boxed_str()); + let config_path = Box::leak(format!("{}.{}", #config_segment, template.config_path).into_boxed_str()); + let max_elements = #limits; + all_templates.push(configs::EnvVarTemplate { + env_name, + config_path, + is_secret: template.is_secret, + max_elements, + }); } } - } else { - quote! {} - }; + }); quote! { impl #impl_generics #struct_name #ty_generics #where_clause { @@ -346,7 +335,6 @@ fn generate_struct_impl( TEMPLATES.get_or_init(|| { let mut all_templates = vec![#(#own_template_entries),*]; #(#nested_template_extends)* - #template_prefix_application all_templates }) } diff --git a/core/connectors/runtime/README.md b/core/connectors/runtime/README.md index 580b7793eb..bf4f920e07 100644 --- a/core/connectors/runtime/README.md +++ b/core/connectors/runtime/README.md @@ -47,6 +47,10 @@ IGGY_CONNECTORS_CONFIG_PATH=connectors.toml cargo run --bin iggy-connectors Supported scalar fields and indexed list entries use environment variables with nested keys joined by underscores, for example `IGGY_CONNECTORS_IGGY_USERNAME`. Header and URL-template maps are configured in TOML. The runtime loads the first `.env` file found in the working directory or its parents, or the file specified by `IGGY_CONNECTORS_ENV_PATH`. +Run `iggy-connectors --list-config-env-vars` to print the supported names and templates without loading configuration or plugins. `` is a stream-vector index from 0 through 255. `` is the uppercased connector key and applies only to the local provider. + +`` sets one lowercased top-level `plugin_config` key with underscores preserved (`A_B` becomes `a_b`, not nested `a.b`); nested plugin configuration keys cannot be set through environment variables. `FORMAT` is reserved and listed separately as `..._PLUGIN_CONFIG_FORMAT`. + Source destination topics must persist every acknowledged batch before the runtime checkpoints the source or invokes its Ack hook. Missing topics are therefore created with `durability = "persisted"` and `messages_required_to_save = 1`. An existing topic must use `durability = "persisted"`; its save threshold may differ because persisted acknowledgments already wait for durable storage. ## State storage diff --git a/core/connectors/runtime/src/configs/connectors/local_provider.rs b/core/connectors/runtime/src/configs/connectors/local_provider.rs index bb8b8aa668..e3c59791c6 100644 --- a/core/connectors/runtime/src/configs/connectors/local_provider.rs +++ b/core/connectors/runtime/src/configs/connectors/local_provider.rs @@ -21,7 +21,7 @@ use crate::configs::connectors::{ SourceConfig, }; use crate::error::RuntimeError; -use ::configs::{ConfigProvider, FileConfigProvider, TypedEnvProvider}; +use ::configs::{ConfigProvider, FileConfigProvider, PLUGIN_CONFIG_ENV_SEGMENT, TypedEnvProvider}; use async_trait::async_trait; use dashmap::DashMap; use figment::value::Dict; @@ -349,7 +349,7 @@ impl LocalConnectorsConfigProvider { let connector_type = base_config.connector_type().to_uppercase(); let key = base_config.key().to_uppercase(); let prefix = format!( - "{}PLUGIN_CONFIG_", + "{}{PLUGIN_CONFIG_ENV_SEGMENT}", connector_env_prefix(&connector_type, &key) ); @@ -404,7 +404,7 @@ impl BaseConnectorConfig { /// do. Shared by the runtime's actual override lookup here and by /// `--list-config-env-vars`'s listing in `main.rs` (passing the literal /// `` as the key), so the two can't drift apart on the prefix the -/// server actually reads. +/// connectors runtime actually reads. pub(crate) fn connector_env_prefix(connector_type: &str, key: &str) -> String { format!( "{}{connector_type}_{key}_", diff --git a/core/connectors/runtime/src/error.rs b/core/connectors/runtime/src/error.rs index 156427b131..9d415a7e6f 100644 --- a/core/connectors/runtime/src/error.rs +++ b/core/connectors/runtime/src/error.rs @@ -69,6 +69,10 @@ pub enum RuntimeError { CannotConvertConfiguration, #[error("IO operation failed with error: {0:?}")] IoError(#[from] std::io::Error), + #[error("Failed to list configuration environment variables")] + ListConfigEnvVars(#[source] std::io::Error), + #[error("Failed to create Tokio runtime")] + RuntimeCreation(#[source] std::io::Error), #[error("HTTP request failed: {0}")] HttpRequestFailed(String), #[error("Token file not found: {0}")] diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs index af14a411fc..ce0adf89c4 100644 --- a/core/connectors/runtime/src/main.rs +++ b/core/connectors/runtime/src/main.rs @@ -22,7 +22,7 @@ use crate::configs::connectors::{ use crate::metrics::ConnectorType; use ::configs::{ CONNECTORS_CONFIG_PATH_ENV, CONNECTORS_ENV_PATH_ENV, CONNECTORS_RUNTIME_ENV_VARS, - ConfigEnvMappings, ConfigProvider, print_env_var_names, + ConfigEnvMappings, ConfigProvider, PLUGIN_CONFIG_ENV_SEGMENT, print_env_var_names, }; use clap::Parser; use configs::connectors::ConfigFormat; @@ -84,7 +84,9 @@ sorted and deduplicated. Template syntax: - represents connector keys (uppercased from config). Overrides via require the local connectors provider. - represents plugin configuration field names, excluding - FORMAT (handled separately as a strongly-typed field) + FORMAT (handled separately as a strongly-typed field). It sets one + lowercased top-level plugin_config key: A_B becomes a_b, not nested a.b. + Nested plugin configuration keys cannot be set via environment variables. Exits immediately before any startup."# )] @@ -142,13 +144,14 @@ fn print_ascii_art(text: &str) { fn main() -> Result<(), RuntimeError> { let args = Args::parse(); if args.list_config_env_vars { - print_config_env_vars()?; + print_config_env_vars().map_err(RuntimeError::ListConfigEnvVars)?; return Ok(()); } capture_allowed_cpus(); Builder::new_multi_thread() .enable_all() - .build()? + .build() + .map_err(RuntimeError::RuntimeCreation)? .block_on(run()) } @@ -164,7 +167,7 @@ fn print_config_env_vars() -> std::io::Result<()> { // drift from the names the runtime actually reads. let prefix = crate::configs::connectors::local_provider::connector_env_prefix(kind, ""); - let plugin_config = format!("{prefix}PLUGIN_CONFIG_"); + let plugin_config = format!("{prefix}{PLUGIN_CONFIG_ENV_SEGMENT}"); templates .iter() .map(move |template| format!("{prefix}{}", template.env_name)) diff --git a/core/integration/src/harness/config/resolve.rs b/core/integration/src/harness/config/resolve.rs index 10994ce372..1a50b83747 100644 --- a/core/integration/src/harness/config/resolve.rs +++ b/core/integration/src/harness/config/resolve.rs @@ -25,7 +25,9 @@ use std::collections::HashMap; /// `ServerConfig::all_env_var_names` cannot know them. `IGGY_CONFIG_PATH` /// selects the config file itself and the root credentials are consumed by /// `args.rs` before the config loads; `IGGY_TEST_VERBOSE` is harness-only. -pub const NON_CONFIG_ENV_VARS: &[&str] = configs::server::SERVER_PROCESS_ENV_VARS; +fn non_config_env_vars() -> impl Iterator { + configs::server::server_process_env_vars() +} /// Resolve config paths to environment variable names. /// @@ -111,8 +113,8 @@ fn find_mapping(path: &str) -> Option<&'static EnvVarMapping> { /// /// Names outside the `IGGY_` prefix are left alone: those address the process /// environment (`RUST_LOG`, test scaffolding), not the config schema. -/// `NON_CONFIG_ENV_VARS` carries the `IGGY_`-prefixed names the server reads -/// outside the config struct. +/// `server_process_env_vars()` supplies the `IGGY_`-prefixed names the server +/// reads outside the config struct. /// /// # Errors /// @@ -125,7 +127,7 @@ pub fn validate_env_var_names(envs: &HashMap) -> Result<(), Stri .filter(|name| { name.starts_with("IGGY_") && !known.contains(&name.as_str()) - && !NON_CONFIG_ENV_VARS.contains(&name.as_str()) + && !non_config_env_vars().any(|known| known == name.as_str()) }) .collect(); if unknown.is_empty() { @@ -283,9 +285,8 @@ mod tests { #[test] fn validate_env_var_names_accepts_the_non_config_variables() { - let envs: HashMap = NON_CONFIG_ENV_VARS - .iter() - .map(|name| ((*name).to_string(), "value".to_string())) + let envs: HashMap = non_config_env_vars() + .map(|name| (name.to_string(), "value".to_string())) .collect(); assert!( validate_env_var_names(&envs).is_ok(), diff --git a/core/integration/tests/config_env_listing/mod.rs b/core/integration/tests/config_env_listing/mod.rs index 7c61e09b18..205b295e93 100644 --- a/core/integration/tests/config_env_listing/mod.rs +++ b/core/integration/tests/config_env_listing/mod.rs @@ -21,15 +21,12 @@ use std::time::Duration; const LIST_ENV_VARS_TIMEOUT: Duration = Duration::from_secs(5); -/// Runs `binary --list-config-env-vars` (plus `extra_args`) with `env` applied, -/// isolated to `directory`. Shared by every test below so each only states -/// what differs: the binary, the env it sets, and any extra flags. -fn run_list_config_env_vars_in( +fn list_config_env_vars_in( binary: &str, directory: &std::path::Path, env: I, extra_args: &[&str], -) -> std::process::Output +) -> Vec where I: IntoIterator, K: AsRef, @@ -38,39 +35,32 @@ where let mut cmd = Command::cargo_bin(binary).expect("binary should be built"); cmd.current_dir(directory) .arg("--list-config-env-vars") + .envs(env) + .args(extra_args) .timeout(LIST_ENV_VARS_TIMEOUT); - for (key, value) in env { - cmd.env(key, value); - } - - for arg in extra_args { - cmd.arg(arg); - } - - cmd.output().expect("listing command should run") + let output = cmd.output().expect("listing command should run"); + assert!( + output.status.success(), + "{binary} failed: {:?}\nstderr: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ); + assert!(output.stderr.is_empty(), "{binary} wrote to stderr"); + let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names = stdout.lines().map(str::to_owned).collect::>(); + assert!(!names.is_empty(), "{binary} returned no variables"); + names } -/// Convenience wrapper over [`run_list_config_env_vars_in`] for tests that -/// don't need to inspect the working directory afterward. -fn run_list_config_env_vars( - binary: &str, - env: I, - extra_args: &[&str], -) -> std::process::Output -where - I: IntoIterator, - K: AsRef, - V: AsRef, -{ +fn list_config_env_vars(binary: &str) -> Vec { let directory = tempfile::tempdir().expect("temporary directory"); - run_list_config_env_vars_in(binary, directory.path(), env, extra_args) -} - -/// Convenience wrapper over [`run_list_config_env_vars`] for tests that don't -/// need to set any environment variables. -fn run_list_config_env_vars_plain(binary: &str, extra_args: &[&str]) -> std::process::Output { - run_list_config_env_vars(binary, std::iter::empty::<(&str, &str)>(), extra_args) + list_config_env_vars_in( + binary, + directory.path(), + std::iter::empty::<(&str, &str)>(), + &[], + ) } #[test] @@ -105,7 +95,7 @@ fn config_env_listing_exits_before_startup_for_each_binary() { .expect("read temporary directory") .map(|entry| entry.expect("directory entry").file_name()) .collect(); - let output = run_list_config_env_vars_in( + let names = list_config_env_vars_in( binary, directory.path(), [ @@ -116,17 +106,9 @@ fn config_env_listing_exits_before_startup_for_each_binary() { &[], ); - assert!( - output.status.success(), - "{binary} failed: {:?}\nstderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - assert!(output.stderr.is_empty(), "{binary} wrote to stderr"); - - let baseline = run_list_config_env_vars_plain(binary, &[]); + let baseline = list_config_env_vars(binary); assert_eq!( - output.stdout, baseline.stdout, + names, baseline, "{binary} output changed with invalid config, dotenv or env values" ); let entries_after: HashSet<_> = std::fs::read_dir(directory.path()) @@ -138,9 +120,6 @@ fn config_env_listing_exits_before_startup_for_each_binary() { "{binary} created startup side effects" ); - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); - let names: Vec<_> = stdout.lines().collect(); - assert!(!names.is_empty(), "{binary} returned no variables"); assert!( names.windows(2).all(|pair| pair[0] < pair[1]), "{binary} output must be sorted and deduplicated" @@ -154,11 +133,8 @@ fn config_env_listing_exits_before_startup_for_each_binary() { #[test] fn config_env_listing_includes_vector_index_templates() { - // Verify that vector fields are correctly represented with placeholder - let output = run_list_config_env_vars_plain("iggy-server", &[]); - - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); - let names: HashSet<_> = stdout.lines().collect(); + let names = list_config_env_vars("iggy-server"); + let names: HashSet<_> = names.iter().map(String::as_str).collect(); // Cluster nodes should have indexed templates assert!( @@ -179,37 +155,15 @@ fn config_env_listing_includes_vector_index_templates() { #[test] fn config_env_listing_includes_connector_templates() { - // Verify that connector SINK/SOURCE templates are correctly formatted - let output = run_list_config_env_vars_plain("iggy-connectors", &[]); - - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); - let names: HashSet<_> = stdout.lines().collect(); - - // Both SINK and SOURCE templates should be present - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SINK__")), - "connectors should list SINK templates with placeholder" - ); - assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__")), - "connectors should list SOURCE templates with placeholder" - ); + let names = list_config_env_vars("iggy-connectors"); + let names: HashSet<_> = names.iter().map(String::as_str).collect(); - // Plugin config templates should be present assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_")), + names.contains("IGGY_CONNECTORS_SINK__PLUGIN_CONFIG_"), "connectors should list SINK plugin config templates" ); assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__PLUGIN_CONFIG_")), + names.contains("IGGY_CONNECTORS_SOURCE__PLUGIN_CONFIG_"), "connectors should list SOURCE plugin config templates" ); assert!( @@ -221,24 +175,19 @@ fn config_env_listing_includes_connector_templates() { "connectors should list the typed SOURCE plugin config format" ); - // Verify SINK and SOURCE ENABLED fields are present assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SINK__ENABLED")), + names.contains("IGGY_CONNECTORS_SINK__ENABLED"), "connectors should list SINK__ENABLED" ); assert!( - names - .iter() - .any(|name| name.contains("IGGY_CONNECTORS_SOURCE__ENABLED")), + names.contains("IGGY_CONNECTORS_SOURCE__ENABLED"), "connectors should list SOURCE__ENABLED" ); assert_eq!( - stdout - .lines() - .filter(|name| *name == "IGGY_CONNECTORS_CONNECTORS_CONFIG_TYPE") + names + .iter() + .filter(|name| **name == "IGGY_CONNECTORS_CONNECTORS_CONFIG_TYPE") .count(), 1, "enum variants should produce one deduplicated tag name" @@ -247,9 +196,8 @@ fn config_env_listing_includes_connector_templates() { #[test] fn config_env_listing_excludes_other_processes_variables() { - let server = run_list_config_env_vars_plain("iggy-server", &[]); - let server = String::from_utf8(server.stdout).expect("UTF-8 output"); - let server_names: HashSet<_> = server.lines().collect(); + let server = list_config_env_vars("iggy-server"); + let server_names: HashSet<_> = server.iter().map(String::as_str).collect(); for excluded in [ "IGGY_CI_BUILD", "IGGY_HOME", @@ -274,11 +222,10 @@ fn config_env_listing_excludes_other_processes_variables() { ("iggy-connectors", "IGGY_CONNECTORS_"), ("iggy-mcp", "IGGY_MCP_"), ] { - let output = run_list_config_env_vars_plain(binary, &[]); - let stdout = String::from_utf8(output.stdout).expect("UTF-8 output"); + let names = list_config_env_vars(binary); assert!( - stdout - .lines() + names + .iter() .all(|name| name == "IGGY_DISPLAY_CONFIG" || name.starts_with(prefix)), "{binary} listed a variable outside {prefix}" ); @@ -286,21 +233,29 @@ fn config_env_listing_excludes_other_processes_variables() { } #[test] -fn config_env_listing_exits_before_runtime_with_invalid_env_values() { - // Verify that invalid environment values don't cause failures - let output = run_list_config_env_vars( - "iggy-server", - [("IGGY_HTTP_ADDRESS", "invalid::address")], // Invalid address - &[], - ); - - assert!( - output.status.success(), - "listing should exit before validating environment values: {:?}\nstderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - assert!(output.stderr.is_empty(), "should not log validation errors"); +fn config_env_listing_includes_each_process_runtime_variables() { + for (binary, expected) in [ + ( + "iggy-server", + &["IGGY_ROOT_PASSWORD", "IGGY_SHARD_RUNTIME_CAPACITY"][..], + ), + ( + "iggy-connectors", + &["IGGY_CONNECTORS_CONFIG_PATH", "IGGY_CONNECTORS_ENV_PATH"][..], + ), + ( + "iggy-mcp", + &["IGGY_MCP_CONFIG_PATH", "IGGY_MCP_ENV_PATH"][..], + ), + ] { + let names = list_config_env_vars(binary); + for expected_name in expected { + assert!( + names.iter().any(|name| name == expected_name), + "{binary} did not list {expected_name}" + ); + } + } } #[test] @@ -314,19 +269,13 @@ fn config_env_listing_with_fresh_does_not_wipe_data_dir() { std::fs::write(&sentinel, "sentinel content").expect("write sentinel file"); // Run with both --fresh and --list-config-env-vars - let output = run_list_config_env_vars_in( + let output_with_fresh = list_config_env_vars_in( "iggy-server", directory.path(), [("IGGY_PATH", data_dir.to_string_lossy().into_owned())], &["--fresh"], ); - assert!( - output.status.success(), - "command failed: {:?}\nstderr: {}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); assert!( sentinel.exists(), "sentinel file should not be deleted by early exit" @@ -337,13 +286,7 @@ fn config_env_listing_with_fresh_does_not_wipe_data_dir() { "sentinel file should not be modified" ); - // Verify the output is the same as without --fresh - let output_with_fresh = String::from_utf8(output.stdout).expect("UTF-8 output"); - - let output_without_fresh = run_list_config_env_vars_plain("iggy-server", &[]); - - let output_without_fresh = - String::from_utf8(output_without_fresh.stdout).expect("UTF-8 output"); + let output_without_fresh = list_config_env_vars("iggy-server"); assert_eq!( output_with_fresh, output_without_fresh, "output should be identical with and without --fresh"