fix: filter target input modalities

crates/libsy/src/algorithms/fall_through.rs

@@ -518,6 +518,7 @@ mod tests {
                 .iter()
                 .map(|name| LlmTarget {
                     semantic_name: name.to_string(),
+                    input_modalities: None,
                 })
                 .collect(),
         )

crates/libsy/src/algorithms/llm_class.rs

@@ -1081,6 +1081,7 @@ mod tests {
     fn router() -> Result<Arc<LlmTaskClassifier>> {
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         Ok(Arc::new(LlmTaskClassifier::new(
             LlmClassifierConfig::Capability {
@@ -1167,6 +1168,7 @@ mod tests {
         let recorder = Arc::new(Recorder::default());
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
             judge_target: target("judge"),
@@ -1189,6 +1191,7 @@ mod tests {
         let recorder = Arc::new(Recorder::default());
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
             judge_target: target("judge"),
@@ -1214,6 +1217,7 @@ mod tests {
         let recorder = Arc::new(Recorder::default());
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
             judge_target: target("judge"),
@@ -1238,6 +1242,7 @@ mod tests {
         let recorder = Arc::new(Recorder::default());
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability {
             judge_target: target("judge"),
@@ -1303,6 +1308,7 @@ mod tests {
     fn invalid_classifier_config_is_rejected() -> Result<()> {
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] {
             assert!(
@@ -1796,6 +1802,7 @@ mod tests {
     fn escalation_router() -> Result<Arc<LlmTaskClassifier>> {
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         Ok(Arc::new(LlmTaskClassifier::new(
             LlmClassifierConfig::Escalation {
@@ -1845,6 +1852,7 @@ mod tests {
         let recorder = Arc::new(Recorder::default());
         let target = |name: &str| LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         };
         let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation {
             judge_target: target("judge"),

crates/libsy/src/algorithms/passthrough.rs

@@ -62,6 +62,7 @@ mod tests {
         };
         let algorithm: Arc<dyn Algorithm> = Arc::new(Passthrough::new(LlmTarget {
             semantic_name: MODEL_ID.to_string(),
+            input_modalities: None,
         }));
         let (trace, response) = test_drive(algorithm, request, echo()).await?;
 

crates/libsy/src/algorithms/rand.rs

@@ -199,6 +199,7 @@ mod tests {
             .iter()
             .map(|name| LlmTarget {
                 semantic_name: (*name).to_string(),
+                input_modalities: None,
             })
             .collect();
         LlmTargetSet::new(targets)

crates/libsy/src/algorithms/stage.rs

@@ -232,6 +232,7 @@ mod tests {
     fn tier_target(name: &str) -> LlmTarget {
         LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         }
     }
 
@@ -322,6 +323,7 @@ mod tests {
         config.llm_fallback = Some(LlmFallback {
             judge_target: LlmTarget {
                 semantic_name: "judge".to_string(),
+                input_modalities: None,
             },
             config: TaskClassifierConfig {
                 base_threshold: -0.1,
@@ -405,6 +407,7 @@ mod tests {
     fn recording_target(name: &str) -> LlmTarget {
         LlmTarget {
             semantic_name: name.to_string(),
+            input_modalities: None,
         }
     }
 

crates/libsy/src/algorithms/subagent_affinity_tests.rs

@@ -49,6 +49,7 @@ fn targets() -> LlmTargetSet {
             .iter()
             .map(|name| LlmTarget {
                 semantic_name: (*name).to_string(),
+                input_modalities: None,
             })
             .collect(),
     )

crates/libsy/src/algorithms/util/llm_judge.rs

@@ -386,6 +386,7 @@ mod tests {
             TestJudge,
             LlmTarget {
                 semantic_name: "judge".to_string(),
+                input_modalities: None,
             },
             TestPolicy,
         )

crates/libsy/src/core/algorithm.rs

@@ -27,7 +27,9 @@ use tracing::Instrument;
 /// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and
 /// [`switchyard_protocol::LlmResponse`] carries either a live
 /// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate.
-use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutingFallbackReason};
+use switchyard_protocol::{
+    Decision, InputModality, LlmClientError, Request, Response, RoutingFallbackReason,
+};
 
 use crate::{DriverError, LibsyError, Result, observability};
 
@@ -289,6 +291,8 @@ pub struct LlmTarget {
     /// `"strong"`, or the model id when they coincide. Mapping it to a provider model
     /// id is the consumer's concern, never the algorithm's.
     pub semantic_name: String,
+    /// Optional authoritative allowlist of input modalities accepted by this target.
+    pub input_modalities: Option<Vec<InputModality>>,
 }
 
 /// The set of targets an algorithm may route among. An algorithm is constructed
@@ -717,6 +721,7 @@ mod tests {
             .iter()
             .map(|name| LlmTarget {
                 semantic_name: name.to_string(),
+                input_modalities: None,
             })
             .collect();
         LlmTargetSet::new(targets)
@@ -1321,9 +1326,11 @@ mod tests {
         let algo = Arc::new(Hedge {
             winner: LlmTarget {
                 semantic_name: "winner".to_string(),
+                input_modalities: None,
             },
             loser: LlmTarget {
                 semantic_name: "loser".to_string(),
+                input_modalities: None,
             },
         });
         let serve = move |decision: Decision, _request: Request| {

crates/protocol/src/llm.rs

@@ -10,6 +10,22 @@ use serde_json::{Map, Value};
 
 use crate::format::FormatId;
 
+/// Input content modalities a target can explicitly accept.
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum InputModality {
+    /// Plain text, refusal text, and reasoning text content.
+    Text,
+    /// Image content.
+    Image,
+    /// Audio content.
+    Audio,
+    /// Video content.
+    Video,
+    /// File or document content.
+    File,
+}
+
 /// Actor role normalized across provider APIs.
 #[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]

crates/switchyard-py/src/libsy_bindings.rs

@@ -18,8 +18,8 @@ use switchyard_libsy::{
 };
 use switchyard_llm_client::ClientRouter;
 use switchyard_protocol::{
-    AggLlmResponse, Decision, LlmClientError, LlmResponse, Metadata, Request, Response,
-    RoutedLlmClient,
+    AggLlmResponse, Decision, InputModality, LlmClientError, LlmResponse, Metadata, Request,
+    Response, RoutedLlmClient,
 };
 
 use crate::errors::py_libsy_error;
@@ -75,12 +75,14 @@ impl RoutedLlmClient for PythonLlmClient {
 struct PyLlmTarget {
     name: String,
     client: Py<PyAny>,
+    input_modalities: Option<Vec<InputModality>>,
 }
 
 impl PyLlmTarget {
     fn clone_core(&self, _py: Python<'_>) -> LlmTarget {
         LlmTarget {
             semantic_name: self.name.clone(),
+            input_modalities: self.input_modalities.clone(),
         }
     }
 
@@ -100,7 +102,13 @@ impl PyLlmTarget {
 #[pymethods]
 impl PyLlmTarget {
     #[new]
-    fn new(py: Python<'_>, name: String, client: Py<PyAny>) -> PyResult<Self> {
+    #[pyo3(signature = (name, client, *, input_modalities=None))]
+    fn new(
+        py: Python<'_>,
+        name: String,
+        client: Py<PyAny>,
+        input_modalities: Option<Vec<String>>,
+    ) -> PyResult<Self> {
         let call = client
             .bind(py)
             .getattr("call")
@@ -110,7 +118,19 @@ impl PyLlmTarget {
                 "client.call must be callable as async call(request)",
             ));
         }
-        Ok(Self { name, client })
+        let input_modalities = input_modalities
+            .map(|modalities| {
+                modalities
+                    .into_iter()
+                    .map(|modality| parse_input_modality(&modality))
+                    .collect::<PyResult<Vec<_>>>()
+            })
+            .transpose()?;
+        Ok(Self {
+            name,
+            client,
+            input_modalities,
+        })
     }
 
     #[getter]
@@ -123,6 +143,19 @@ impl PyLlmTarget {
     }
 }
 
+fn parse_input_modality(value: &str) -> PyResult<InputModality> {
+    match value {
+        "text" => Ok(InputModality::Text),
+        "image" => Ok(InputModality::Image),
+        "audio" => Ok(InputModality::Audio),
+        "video" => Ok(InputModality::Video),
+        "file" => Ok(InputModality::File),
+        _ => Err(PyValueError::new_err(format!(
+            "unknown input modality {value:?}; expected one of text, image, audio, video, file"
+        ))),
+    }
+}
+
 /// Classifier settings shared by standalone and stage-router classifiers.
 #[pyclass(
     name = "TaskClassifierConfig",

crates/switchyard-server/src/config.rs

@@ -20,7 +20,7 @@ use switchyard_llm_client::{
     Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig,
     TranslatingLlmClient,
 };
-use switchyard_protocol::RoutedLlmClient;
+use switchyard_protocol::{InputModality, RoutedLlmClient};
 
 use crate::{CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState};
 
@@ -132,10 +132,11 @@ impl ServerConfig {
             let model_configs = models_by_client
                 .get_mut(&target.llm_client)
                 .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?;
-            model_configs.push(ModelConfig::new(
+            model_configs.push(ModelConfig::with_input_modalities(
                 &target.id,
                 build_backend(&target.llm_client, client_config, &target.extra_body)?,
                 None,
+                target.input_modalities.clone(),
             ));
         }
 
@@ -159,6 +160,7 @@ impl ServerConfig {
                     name.clone(),
                     LlmTarget {
                         semantic_name: config.id.clone(),
+                        input_modalities: config.input_modalities.clone(),
                     },
                 )
             })
@@ -251,6 +253,7 @@ struct LlmClientConfig {
 struct TargetConfig {
     id: String,
     llm_client: String,
+    input_modalities: Option<Vec<InputModality>>,
     #[serde(default)]
     extra_body: BTreeMap<String, Value>,
 }
@@ -1459,6 +1462,28 @@ target = "azure"
         Ok(())
     }
 
+    #[test]
+    fn target_input_modalities_are_parsed_and_validated() -> ServerResult<()> {
+        let configured = VALID_CONFIG.replacen(
+            "llm_client = \"responses\"",
+            "llm_client = \"responses\"\ninput_modalities = [\"text\", \"image\"]",
+            1,
+        );
+        let config: ServerConfig = toml::from_str(&configured)
+            .map_err(|error| ServerError::new(format!("failed to parse config: {error}")))?;
+        assert_eq!(
+            config
+                .targets
+                .get("strong")
+                .and_then(|target| target.input_modalities.as_ref()),
+            Some(&vec![InputModality::Text, InputModality::Image])
+        );
+
+        let invalid = configured.replace("\"image\"", "\"pdf\"");
+        assert!(error_message(&invalid).contains("unknown variant"));
+        Ok(())
+    }
+
     #[test]
     fn retry_budget_defaults_and_accepts_an_override() -> ServerResult<()> {
         let default: ServerConfig = toml::from_str(VALID_CONFIG).map_err(|error| {

crates/switchyard-server/tests/server.rs

@@ -232,6 +232,7 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult<Server
                     .iter()
                     .map(|model| LlmTarget {
                         semantic_name: (*model).to_string(),
+                        input_modalities: None,
                     })
                     .collect(),
             );

switchyard_rust/libsy.py

@@ -38,13 +38,21 @@ async def call(
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
-    from typing import final
+    from typing import Literal, final
+
+    InputModality = Literal["text", "image", "audio", "video", "file"]
 
     class LibsyError(RuntimeError): ...
 
     @final
     class LlmTarget:
-        def __init__(self, name: str, client: LlmClient) -> None: ...
+        def __init__(
+            self,
+            name: str,
+            client: LlmClient,
+            *,
+            input_modalities: Sequence[InputModality] | None = None,
+        ) -> None: ...
 
         @property
         def name(self) -> str: ...