From 4c54433f915ae09e6664d087384911c480c1c6d3 Mon Sep 17 00:00:00 2001 From: Coding Agent Date: Tue, 11 Aug 2026 19:40:06 +0000 Subject: [PATCH 1/2] fix: filter target input modalities Signed-off-by: Coding Agent --- crates/libsy-llm-client/src/client.rs | 365 ++++++++++++++++-- crates/libsy/src/algorithms/fall_through.rs | 1 + crates/libsy/src/algorithms/llm_class.rs | 8 + crates/libsy/src/algorithms/passthrough.rs | 1 + crates/libsy/src/algorithms/rand.rs | 1 + crates/libsy/src/algorithms/stage.rs | 3 + .../src/algorithms/subagent_affinity_tests.rs | 1 + crates/libsy/src/algorithms/util/llm_judge.rs | 1 + crates/libsy/src/core/algorithm.rs | 9 +- crates/protocol/src/llm.rs | 16 + crates/switchyard-py/src/libsy_bindings.rs | 41 +- crates/switchyard-server/src/config.rs | 29 +- crates/switchyard-server/tests/server.rs | 1 + switchyard_rust/libsy.py | 12 +- 14 files changed, 454 insertions(+), 35 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index d6ca1191d..a61093d27 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -4,7 +4,7 @@ //! [`TranslatingLlmClient`] — the crate's single public entry point: encode a neutral //! request, call the configured backend over HTTP, decode the neutral response. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::time::{Duration, SystemTime}; use async_trait::async_trait; @@ -13,7 +13,8 @@ use reqwest::RequestBuilder; use reqwest::header::{HeaderMap, RETRY_AFTER}; use serde_json::{Map, Value}; use switchyard_protocol::{ - Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient, + ContentBlock, Decision, InputModality, LlmRequest, LlmResponse, Metadata, Request, Response, + RoutedLlmClient, }; use switchyard_translation::{ WireFormat, decode_aggregated_response, decode_request, decode_stream, @@ -59,6 +60,7 @@ pub struct ModelConfig { model_name: String, default_backend: Backend, other_backends: Option>, + input_modalities: Option>, } impl ModelConfig { @@ -73,6 +75,22 @@ impl ModelConfig { model_name: model_name.into(), default_backend, other_backends, + input_modalities: None, + } + } + + /// A model config with an explicit target-local input modality allowlist. + pub fn with_input_modalities( + model_name: impl Into, + default_backend: Backend, + other_backends: Option>, + input_modalities: Option>, + ) -> Self { + Self { + model_name: model_name.into(), + default_backend, + other_backends, + input_modalities: input_modalities.map(|modalities| modalities.into_iter().collect()), } } } @@ -115,16 +133,9 @@ impl TranslatingLlmClient { /// format matches, otherwise a matching entry in `other_backends`; `None` when /// the model is unknown or has no backend for `format`. pub fn backend_for(&self, model: &str, format: WireFormat) -> Option<&Backend> { - self.model_to_config.get(model).and_then(|config| { - if config.default_backend.wire_format() == format { - Some(&config.default_backend) - } else { - config - .other_backends - .as_ref() - .and_then(|backends| backends.iter().find(|b| b.wire_format() == format)) - } - }) + self.model_to_config + .get(model) + .and_then(|config| backend_for_config(config, format)) } /// Whether `model` has an Anthropic backend that supports token counting. @@ -138,10 +149,17 @@ impl TranslatingLlmClient { /// Returns an error when the model has no Anthropic backend or the upstream /// request fails or returns invalid JSON. pub async fn count_tokens(&self, model: &str, request: Request) -> Result { - let backend = self - .backend_for(model, WireFormat::AnthropicMessages) - .ok_or_else(|| LlmClientError::Configuration { - message: format!("model {model} has no Anthropic backend for count_tokens"), + let config = + self.model_to_config + .get(model) + .ok_or_else(|| LlmClientError::Configuration { + message: format!("model {model} has no Anthropic backend for count_tokens"), + })?; + let backend = + backend_for_config(config, WireFormat::AnthropicMessages).ok_or_else(|| { + LlmClientError::Configuration { + message: format!("model {model} has no Anthropic backend for count_tokens"), + } })?; let Request { llm_request, @@ -155,6 +173,7 @@ impl TranslatingLlmClient { llm_request, metadata.as_ref(), model, + config.input_modalities.as_ref(), UpstreamEndpoint::CountTokens, ) .await?; @@ -189,12 +208,15 @@ impl TranslatingLlmClient { mut llm_request: LlmRequest, metadata: Option<&Metadata>, model: &str, + input_modalities: Option<&BTreeSet>, endpoint: UpstreamEndpoint, ) -> Result { // The resolved name is the upstream model id (per the crate contract). llm_request.model = Some(model.to_string()); + filter_request_input_modalities(&mut llm_request, input_modalities); let mut body = encode_request(&llm_request, wire_format) .map_err(|error| LlmClientError::RequestEncoding(error.to_string()))?; + filter_encoded_input_modalities(&mut body, wire_format, input_modalities); // `encode_request` round-trips a preserved same-format body verbatim, // which keeps the caller's original `model`; force the resolved model so // the upstream always sees the target id. @@ -380,20 +402,19 @@ impl TranslatingLlmClient { message: "no model given".to_string(), })?; - let orig_format = metadata.as_ref().and_then(|m| m.wire_format); - let wire_format = orig_format.unwrap_or( + let config = self.model_to_config .get(&model) - .map(|config| config.default_backend.wire_format()) .ok_or_else(|| LlmClientError::Configuration { message: format!("no backend configured for model {model:?}"), - })?, - ); - let backend = - self.backend_for(&model, wire_format) - .ok_or_else(|| LlmClientError::Configuration { - message: format!("model {model:?} has no backend for format {wire_format}"), })?; + let orig_format = metadata.as_ref().and_then(|m| m.wire_format); + let wire_format = orig_format.unwrap_or(config.default_backend.wire_format()); + let backend = backend_for_config(config, wire_format).ok_or_else(|| { + LlmClientError::Configuration { + message: format!("model {model:?} has no backend for format {wire_format}"), + } + })?; let http_response = self .send_encoded( @@ -402,6 +423,7 @@ impl TranslatingLlmClient { llm_request, metadata.as_ref(), &model, + config.input_modalities.as_ref(), UpstreamEndpoint::Completion, ) .await?; @@ -651,6 +673,180 @@ fn apply_extra_headers(mut builder: RequestBuilder, backend: &Backend) -> Reques builder } +fn backend_for_config(config: &ModelConfig, format: WireFormat) -> Option<&Backend> { + if config.default_backend.wire_format() == format { + Some(&config.default_backend) + } else { + config + .other_backends + .as_ref() + .and_then(|backends| backends.iter().find(|b| b.wire_format() == format)) + } +} + +fn filter_request_input_modalities( + request: &mut LlmRequest, + input_modalities: Option<&BTreeSet>, +) { + let Some(allowed) = input_modalities else { + return; + }; + for instruction in &mut request.instructions { + filter_content_blocks(&mut instruction.content, allowed); + } + for message in &mut request.messages { + filter_content_blocks(&mut message.content, allowed); + } +} + +fn filter_content_blocks(content: &mut Vec, allowed: &BTreeSet) { + for block in content.iter_mut() { + if let ContentBlock::ToolResult(result) = block { + filter_content_blocks(&mut result.content, allowed); + } + } + content.retain(|block| match block_input_modality(block) { + Some(modality) => allowed.contains(&modality), + None => true, + }); +} + +fn block_input_modality(block: &ContentBlock) -> Option { + match block { + ContentBlock::Text { .. } + | ContentBlock::Reasoning { .. } + | ContentBlock::Refusal { .. } => Some(InputModality::Text), + ContentBlock::Image { .. } => Some(InputModality::Image), + ContentBlock::Audio { .. } => Some(InputModality::Audio), + ContentBlock::Video { .. } => Some(InputModality::Video), + ContentBlock::File { .. } => Some(InputModality::File), + ContentBlock::ToolCall(_) | ContentBlock::ToolResult(_) | ContentBlock::Unknown { .. } => { + None + } + } +} + +fn filter_encoded_input_modalities( + body: &mut Value, + wire_format: WireFormat, + input_modalities: Option<&BTreeSet>, +) { + let Some(allowed) = input_modalities else { + return; + }; + match wire_format { + WireFormat::OpenAiChat => filter_openai_chat_body(body, allowed), + WireFormat::OpenAiResponses => filter_openai_responses_body(body, allowed), + WireFormat::AnthropicMessages => filter_anthropic_body(body, allowed), + } +} + +fn filter_openai_chat_body(body: &mut Value, allowed: &BTreeSet) { + let Value::Object(object) = body else { + return; + }; + if let Some(Value::Array(messages)) = object.get_mut("messages") { + for message in messages { + filter_json_content_field(message, allowed, openai_chat_json_modality); + } + } +} + +fn filter_openai_responses_body(body: &mut Value, allowed: &BTreeSet) { + let Value::Object(object) = body else { + return; + }; + if let Some(input) = object.get_mut("input") { + match input { + Value::Array(items) => { + for item in items { + filter_json_content_field(item, allowed, openai_responses_json_modality); + } + } + Value::Object(_) => { + filter_json_content_field(input, allowed, openai_responses_json_modality) + } + _ => {} + } + } +} + +fn filter_anthropic_body(body: &mut Value, allowed: &BTreeSet) { + let Value::Object(object) = body else { + return; + }; + if let Some(Value::Array(system)) = object.get_mut("system") { + filter_json_content_array(system, allowed, anthropic_json_modality); + } + if let Some(Value::Array(messages)) = object.get_mut("messages") { + for message in messages { + filter_json_content_field(message, allowed, anthropic_json_modality); + } + } +} + +fn filter_json_content_field( + value: &mut Value, + allowed: &BTreeSet, + modality: fn(&Value) -> Option, +) { + let Value::Object(object) = value else { + return; + }; + if let Some(Value::Array(content)) = object.get_mut("content") { + filter_json_content_array(content, allowed, modality); + } +} + +fn filter_json_content_array( + content: &mut Vec, + allowed: &BTreeSet, + modality: fn(&Value) -> Option, +) { + for block in content.iter_mut() { + if block.get("type").and_then(Value::as_str) == Some("tool_result") { + filter_json_content_field(block, allowed, modality); + } + } + content.retain(|block| match modality(block) { + Some(modality) => allowed.contains(&modality), + None => true, + }); +} + +fn openai_chat_json_modality(block: &Value) -> Option { + match block.get("type").and_then(Value::as_str) { + Some("text") => Some(InputModality::Text), + Some("image_url") => Some(InputModality::Image), + Some("input_audio") => Some(InputModality::Audio), + Some("input_video") => Some(InputModality::Video), + Some("file") => Some(InputModality::File), + _ => None, + } +} + +fn openai_responses_json_modality(block: &Value) -> Option { + match block.get("type").and_then(Value::as_str) { + Some("input_text" | "output_text" | "refusal" | "reasoning") => Some(InputModality::Text), + Some("input_image") => Some(InputModality::Image), + Some("input_audio") => Some(InputModality::Audio), + Some("input_video") => Some(InputModality::Video), + Some("input_file") => Some(InputModality::File), + _ => None, + } +} + +fn anthropic_json_modality(block: &Value) -> Option { + match block.get("type").and_then(Value::as_str) { + Some("text" | "thinking" | "redacted_thinking") => Some(InputModality::Text), + Some("image") => Some(InputModality::Image), + Some("audio") => Some(InputModality::Audio), + Some("video") => Some(InputModality::Video), + Some("document") => Some(InputModality::File), + _ => None, + } +} + // Overwrites the outbound body's `model` field with the resolved model id. fn set_json_model(body: &mut Value, model: &str) { if let Value::Object(object) = body { @@ -802,7 +998,10 @@ mod tests { use std::thread::JoinHandle; use serde_json::json; - use switchyard_protocol::{LlmRequest, completion_text, text_request}; + use switchyard_protocol::{ + ContentBlock, FormatId, ImageSource, LlmRequest, Message, Role, completion_text, + text_request, + }; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, ResponseTemplate}; @@ -852,6 +1051,18 @@ mod tests { )] } + fn responses_map_with_input_modalities( + base_url: &str, + input_modalities: Vec, + ) -> Vec { + vec![ModelConfig::with_input_modalities( + "gpt", + Backend::OpenAiResponses(config(base_url)), + None, + Some(input_modalities), + )] + } + fn chat_map_with_retries(base_url: &str, max_retries: u32) -> Vec { vec![ModelConfig::new( "gpt", @@ -1082,6 +1293,108 @@ mod tests { Ok(()) } + #[tokio::test] + async fn input_modalities_filter_preserved_openai_responses_body_without_mutating_request() + -> std::result::Result<(), Box> { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/responses")) + .and(|request: &wiremock::Request| { + let body: Value = serde_json::from_slice(&request.body).unwrap_or(Value::Null); + let content = &body["input"][0]["content"]; + body["model"] == json!("gpt") + && content + == &json!([ + {"type": "input_text", "text": "describe this"}, + {"type": "vendor_extension", "payload": {"kind": "kept"}} + ]) + && body["tools"][0]["parameters"]["properties"]["content"]["type"] + == json!("string") + }) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "resp_1", + "model": "gpt", + "output": [{ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok"}] + }], + "usage": {} + }))) + .expect(1) + .mount(&server) + .await; + + let preserved_body = json!({ + "model": "switchyard", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "describe this"}, + {"type": "input_image", "image_url": "data:image/png;base64,abc"}, + {"type": "input_file", "file": {"file_id": "file-1"}}, + {"type": "vendor_extension", "payload": {"kind": "kept"}} + ] + }], + "tools": [{ + "type": "function", + "name": "keep_schema", + "parameters": {"type": "object", "properties": {"content": {"type": "string"}}} + }] + }); + let mut llm_request = LlmRequest { + model: Some("gpt".to_string()), + messages: vec![Message { + role: Role::User, + content: vec![ + ContentBlock::Text { + text: "describe this".to_string(), + }, + ContentBlock::Image { + source: ImageSource::Url { + url: "data:image/png;base64,abc".to_string(), + detail: None, + }, + }, + ContentBlock::Unknown { + provider: FormatId::from(WireFormat::OpenAiResponses), + raw: json!({"type": "vendor_extension", "payload": {"kind": "kept"}}), + }, + ], + }], + ..LlmRequest::default() + }; + llm_request + .preservation + .requests + .insert(WireFormat::OpenAiResponses.into(), preserved_body); + let original_request = llm_request.clone(); + let client = TranslatingLlmClient::new(&responses_map_with_input_modalities( + &format!("{}/v1", server.uri()), + vec![InputModality::Text], + ))?; + + let response = client + .call_rewrite_model( + Request { + llm_request: llm_request.clone(), + raw_request: None, + metadata: Some(Metadata { + wire_format: Some(WireFormat::OpenAiResponses), + ..Metadata::default() + }), + }, + None, + ) + .await?; + + assert_eq!(llm_request, original_request); + let agg = response.llm_response.into_agg().await?; + assert_eq!(completion_text(&agg), "ok"); + Ok(()) + } + #[tokio::test] async fn invalid_json_is_a_response_translation_error() -> std::result::Result<(), Box> { diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index f1012fb04..906e2df95 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -518,6 +518,7 @@ mod tests { .iter() .map(|name| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }) .collect(), ) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 80c173a87..04b0a9518 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -1081,6 +1081,7 @@ mod tests { fn router() -> Result> { let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; Ok(Arc::new(LlmTaskClassifier::new( LlmClassifierConfig::Capability { @@ -1167,6 +1168,7 @@ mod tests { let recorder = Arc::new(Recorder::default()); let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { judge_target: target("judge"), @@ -1189,6 +1191,7 @@ mod tests { let recorder = Arc::new(Recorder::default()); let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { judge_target: target("judge"), @@ -1214,6 +1217,7 @@ mod tests { let recorder = Arc::new(Recorder::default()); let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { judge_target: target("judge"), @@ -1238,6 +1242,7 @@ mod tests { let recorder = Arc::new(Recorder::default()); let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { judge_target: target("judge"), @@ -1303,6 +1308,7 @@ mod tests { fn invalid_classifier_config_is_rejected() -> Result<()> { let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] { assert!( @@ -1796,6 +1802,7 @@ mod tests { fn escalation_router() -> Result> { let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; Ok(Arc::new(LlmTaskClassifier::new( LlmClassifierConfig::Escalation { @@ -1845,6 +1852,7 @@ mod tests { let recorder = Arc::new(Recorder::default()); let target = |name: &str| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { judge_target: target("judge"), diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs index 1c7e322b1..ab3392344 100644 --- a/crates/libsy/src/algorithms/passthrough.rs +++ b/crates/libsy/src/algorithms/passthrough.rs @@ -62,6 +62,7 @@ mod tests { }; let algorithm: Arc = Arc::new(Passthrough::new(LlmTarget { semantic_name: MODEL_ID.to_string(), + input_modalities: None, })); let (trace, response) = test_drive(algorithm, request, echo()).await?; diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs index 2a7b61c84..bbcf70b0e 100644 --- a/crates/libsy/src/algorithms/rand.rs +++ b/crates/libsy/src/algorithms/rand.rs @@ -199,6 +199,7 @@ mod tests { .iter() .map(|name| LlmTarget { semantic_name: (*name).to_string(), + input_modalities: None, }) .collect(); LlmTargetSet::new(targets) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index c2ee63c39..40bf6327c 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -232,6 +232,7 @@ mod tests { fn tier_target(name: &str) -> LlmTarget { LlmTarget { semantic_name: name.to_string(), + input_modalities: None, } } @@ -322,6 +323,7 @@ mod tests { config.llm_fallback = Some(LlmFallback { judge_target: LlmTarget { semantic_name: "judge".to_string(), + input_modalities: None, }, config: TaskClassifierConfig { base_threshold: -0.1, @@ -405,6 +407,7 @@ mod tests { fn recording_target(name: &str) -> LlmTarget { LlmTarget { semantic_name: name.to_string(), + input_modalities: None, } } diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs index 02b808934..e56479f13 100644 --- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs +++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs @@ -49,6 +49,7 @@ fn targets() -> LlmTargetSet { .iter() .map(|name| LlmTarget { semantic_name: (*name).to_string(), + input_modalities: None, }) .collect(), ) diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index 464544e24..0137f3b72 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -386,6 +386,7 @@ mod tests { TestJudge, LlmTarget { semantic_name: "judge".to_string(), + input_modalities: None, }, TestPolicy, ) diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index 519b1e4bb..ada4748f9 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -27,7 +27,9 @@ use tracing::Instrument; /// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and /// [`switchyard_protocol::LlmResponse`] carries either a live /// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate. -use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutingFallbackReason}; +use switchyard_protocol::{ + Decision, InputModality, LlmClientError, Request, Response, RoutingFallbackReason, +}; use crate::{DriverError, LibsyError, Result, observability}; @@ -289,6 +291,8 @@ pub struct LlmTarget { /// `"strong"`, or the model id when they coincide. Mapping it to a provider model /// id is the consumer's concern, never the algorithm's. pub semantic_name: String, + /// Optional authoritative allowlist of input modalities accepted by this target. + pub input_modalities: Option>, } /// The set of targets an algorithm may route among. An algorithm is constructed @@ -717,6 +721,7 @@ mod tests { .iter() .map(|name| LlmTarget { semantic_name: name.to_string(), + input_modalities: None, }) .collect(); LlmTargetSet::new(targets) @@ -1321,9 +1326,11 @@ mod tests { let algo = Arc::new(Hedge { winner: LlmTarget { semantic_name: "winner".to_string(), + input_modalities: None, }, loser: LlmTarget { semantic_name: "loser".to_string(), + input_modalities: None, }, }); let serve = move |decision: Decision, _request: Request| { diff --git a/crates/protocol/src/llm.rs b/crates/protocol/src/llm.rs index d8def175a..43faace49 100644 --- a/crates/protocol/src/llm.rs +++ b/crates/protocol/src/llm.rs @@ -10,6 +10,22 @@ use serde_json::{Map, Value}; use crate::format::FormatId; +/// Input content modalities a target can explicitly accept. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InputModality { + /// Plain text, refusal text, and reasoning text content. + Text, + /// Image content. + Image, + /// Audio content. + Audio, + /// Video content. + Video, + /// File or document content. + File, +} + /// Actor role normalized across provider APIs. #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 8be1bede0..020175328 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -18,8 +18,8 @@ use switchyard_libsy::{ }; use switchyard_llm_client::ClientRouter; use switchyard_protocol::{ - AggLlmResponse, Decision, LlmClientError, LlmResponse, Metadata, Request, Response, - RoutedLlmClient, + AggLlmResponse, Decision, InputModality, LlmClientError, LlmResponse, Metadata, Request, + Response, RoutedLlmClient, }; use crate::errors::py_libsy_error; @@ -75,12 +75,14 @@ impl RoutedLlmClient for PythonLlmClient { struct PyLlmTarget { name: String, client: Py, + input_modalities: Option>, } impl PyLlmTarget { fn clone_core(&self, _py: Python<'_>) -> LlmTarget { LlmTarget { semantic_name: self.name.clone(), + input_modalities: self.input_modalities.clone(), } } @@ -100,7 +102,13 @@ impl PyLlmTarget { #[pymethods] impl PyLlmTarget { #[new] - fn new(py: Python<'_>, name: String, client: Py) -> PyResult { + #[pyo3(signature = (name, client, *, input_modalities=None))] + fn new( + py: Python<'_>, + name: String, + client: Py, + input_modalities: Option>, + ) -> PyResult { let call = client .bind(py) .getattr("call") @@ -110,7 +118,19 @@ impl PyLlmTarget { "client.call must be callable as async call(request)", )); } - Ok(Self { name, client }) + let input_modalities = input_modalities + .map(|modalities| { + modalities + .into_iter() + .map(|modality| parse_input_modality(&modality)) + .collect::>>() + }) + .transpose()?; + Ok(Self { + name, + client, + input_modalities, + }) } #[getter] @@ -123,6 +143,19 @@ impl PyLlmTarget { } } +fn parse_input_modality(value: &str) -> PyResult { + match value { + "text" => Ok(InputModality::Text), + "image" => Ok(InputModality::Image), + "audio" => Ok(InputModality::Audio), + "video" => Ok(InputModality::Video), + "file" => Ok(InputModality::File), + _ => Err(PyValueError::new_err(format!( + "unknown input modality {value:?}; expected one of text, image, audio, video, file" + ))), + } +} + /// Classifier settings shared by standalone and stage-router classifiers. #[pyclass( name = "TaskClassifierConfig", diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index f7be8c646..fb708bce8 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -20,7 +20,7 @@ use switchyard_llm_client::{ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; -use switchyard_protocol::RoutedLlmClient; +use switchyard_protocol::{InputModality, RoutedLlmClient}; use crate::{CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState}; @@ -132,10 +132,11 @@ impl ServerConfig { let model_configs = models_by_client .get_mut(&target.llm_client) .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?; - model_configs.push(ModelConfig::new( + model_configs.push(ModelConfig::with_input_modalities( &target.id, build_backend(&target.llm_client, client_config, &target.extra_body)?, None, + target.input_modalities.clone(), )); } @@ -159,6 +160,7 @@ impl ServerConfig { name.clone(), LlmTarget { semantic_name: config.id.clone(), + input_modalities: config.input_modalities.clone(), }, ) }) @@ -251,6 +253,7 @@ struct LlmClientConfig { struct TargetConfig { id: String, llm_client: String, + input_modalities: Option>, #[serde(default)] extra_body: BTreeMap, } @@ -1459,6 +1462,28 @@ target = "azure" Ok(()) } + #[test] + fn target_input_modalities_are_parsed_and_validated() -> ServerResult<()> { + let configured = VALID_CONFIG.replacen( + "llm_client = \"responses\"", + "llm_client = \"responses\"\ninput_modalities = [\"text\", \"image\"]", + 1, + ); + let config: ServerConfig = toml::from_str(&configured) + .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?; + assert_eq!( + config + .targets + .get("strong") + .and_then(|target| target.input_modalities.as_ref()), + Some(&vec![InputModality::Text, InputModality::Image]) + ); + + let invalid = configured.replace("\"image\"", "\"pdf\""); + assert!(error_message(&invalid).contains("unknown variant")); + Ok(()) + } + #[test] fn retry_budget_defaults_and_accepts_an_override() -> ServerResult<()> { let default: ServerConfig = toml::from_str(VALID_CONFIG).map_err(|error| { diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 65bd5ab79..c0934d0c0 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -232,6 +232,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult None: ... + def __init__( + self, + name: str, + client: LlmClient, + *, + input_modalities: Sequence[InputModality] | None = None, + ) -> None: ... @property def name(self) -> str: ... From ca8144539a98c302ced3ba9f4d7cc538df41696b Mon Sep 17 00:00:00 2001 From: Coding Agent Date: Tue, 11 Aug 2026 19:56:50 +0000 Subject: [PATCH 2/2] Refactor JSON modality parsing Signed-off-by: Coding Agent --- crates/libsy-llm-client/src/client.rs | 76 +++++++++++++-------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index a61093d27..3da3d6547 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -747,7 +747,7 @@ fn filter_openai_chat_body(body: &mut Value, allowed: &BTreeSet) }; if let Some(Value::Array(messages)) = object.get_mut("messages") { for message in messages { - filter_json_content_field(message, allowed, openai_chat_json_modality); + filter_json_content_field(message, allowed, JsonModalityFormat::OpenAiChat); } } } @@ -760,11 +760,11 @@ fn filter_openai_responses_body(body: &mut Value, allowed: &BTreeSet { for item in items { - filter_json_content_field(item, allowed, openai_responses_json_modality); + filter_json_content_field(item, allowed, JsonModalityFormat::OpenAiResponses); } } Value::Object(_) => { - filter_json_content_field(input, allowed, openai_responses_json_modality) + filter_json_content_field(input, allowed, JsonModalityFormat::OpenAiResponses) } _ => {} } @@ -776,73 +776,71 @@ fn filter_anthropic_body(body: &mut Value, allowed: &BTreeSet) { return; }; if let Some(Value::Array(system)) = object.get_mut("system") { - filter_json_content_array(system, allowed, anthropic_json_modality); + filter_json_content_array(system, allowed, JsonModalityFormat::Anthropic); } if let Some(Value::Array(messages)) = object.get_mut("messages") { for message in messages { - filter_json_content_field(message, allowed, anthropic_json_modality); + filter_json_content_field(message, allowed, JsonModalityFormat::Anthropic); } } } +#[derive(Clone, Copy)] +enum JsonModalityFormat { + OpenAiChat, + OpenAiResponses, + Anthropic, +} + fn filter_json_content_field( value: &mut Value, allowed: &BTreeSet, - modality: fn(&Value) -> Option, + format: JsonModalityFormat, ) { let Value::Object(object) = value else { return; }; if let Some(Value::Array(content)) = object.get_mut("content") { - filter_json_content_array(content, allowed, modality); + filter_json_content_array(content, allowed, format); } } fn filter_json_content_array( content: &mut Vec, allowed: &BTreeSet, - modality: fn(&Value) -> Option, + format: JsonModalityFormat, ) { for block in content.iter_mut() { if block.get("type").and_then(Value::as_str) == Some("tool_result") { - filter_json_content_field(block, allowed, modality); + filter_json_content_field(block, allowed, format); } } - content.retain(|block| match modality(block) { + content.retain(|block| match json_input_modality(format, block) { Some(modality) => allowed.contains(&modality), None => true, }); } -fn openai_chat_json_modality(block: &Value) -> Option { - match block.get("type").and_then(Value::as_str) { - Some("text") => Some(InputModality::Text), - Some("image_url") => Some(InputModality::Image), - Some("input_audio") => Some(InputModality::Audio), - Some("input_video") => Some(InputModality::Video), - Some("file") => Some(InputModality::File), - _ => None, - } -} - -fn openai_responses_json_modality(block: &Value) -> Option { - match block.get("type").and_then(Value::as_str) { - Some("input_text" | "output_text" | "refusal" | "reasoning") => Some(InputModality::Text), - Some("input_image") => Some(InputModality::Image), - Some("input_audio") => Some(InputModality::Audio), - Some("input_video") => Some(InputModality::Video), - Some("input_file") => Some(InputModality::File), - _ => None, - } -} - -fn anthropic_json_modality(block: &Value) -> Option { - match block.get("type").and_then(Value::as_str) { - Some("text" | "thinking" | "redacted_thinking") => Some(InputModality::Text), - Some("image") => Some(InputModality::Image), - Some("audio") => Some(InputModality::Audio), - Some("video") => Some(InputModality::Video), - Some("document") => Some(InputModality::File), +fn json_input_modality(format: JsonModalityFormat, block: &Value) -> Option { + match (format, block.get("type").and_then(Value::as_str)?) { + (JsonModalityFormat::OpenAiChat | JsonModalityFormat::Anthropic, "text") + | ( + JsonModalityFormat::OpenAiResponses, + "input_text" | "output_text" | "refusal" | "reasoning", + ) + | (JsonModalityFormat::Anthropic, "thinking" | "redacted_thinking") => { + Some(InputModality::Text) + } + (JsonModalityFormat::OpenAiChat, "image_url") + | (JsonModalityFormat::OpenAiResponses, "input_image") + | (JsonModalityFormat::Anthropic, "image") => Some(InputModality::Image), + (JsonModalityFormat::OpenAiChat | JsonModalityFormat::OpenAiResponses, "input_audio") + | (JsonModalityFormat::Anthropic, "audio") => Some(InputModality::Audio), + (JsonModalityFormat::OpenAiChat | JsonModalityFormat::OpenAiResponses, "input_video") + | (JsonModalityFormat::Anthropic, "video") => Some(InputModality::Video), + (JsonModalityFormat::OpenAiChat, "file") + | (JsonModalityFormat::OpenAiResponses, "input_file") + | (JsonModalityFormat::Anthropic, "document") => Some(InputModality::File), _ => None, } }