diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 467aa1b7470..5022a37ef69 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -44,8 +44,8 @@ jobs: with: python-version: ${{ matrix.python-version }} os: ${{ runner.os }} - # RedisVL still declares Python <3.15. - exclude-packages: ${{ matrix.python-version == '3.10' && 'agent-framework-github-copilot agent-framework-azure-cosmos-memory' || matrix.python-version == '3.15' && 'agent-framework-redis' || '' }} + # Foundry Local requires Python 3.11-3.14; RedisVL still declares Python <3.15. + exclude-packages: ${{ matrix.python-version == '3.10' && 'agent-framework-github-copilot agent-framework-azure-cosmos-memory agent-framework-foundry-local' || matrix.python-version == '3.15' && 'agent-framework-redis agent-framework-foundry-local' || '' }} env: # Configure a constant location for the uv cache UV_CACHE_DIR: /tmp/.uv-cache diff --git a/python/packages/core/pyproject.toml b/python/packages/core/pyproject.toml index bbcdc3c84a8..416bfc1553b 100644 --- a/python/packages/core/pyproject.toml +++ b/python/packages/core/pyproject.toml @@ -53,7 +53,7 @@ all = [ "agent-framework-durabletask>=1.0.0b260922,<2", "agent-framework-foundry>=1.15.0,<2", "agent-framework-foundry-hosting>=1.0.0b261022,<2", - "agent-framework-foundry-local>=1.0.0b261015,<2", + "agent-framework-foundry-local>=1.0.0b261015,<2; python_version >= '3.11' and python_version < '3.15'", "agent-framework-gemini>=1.0.0b261022,<2", "agent-framework-github-copilot>=2.1.0,<3; python_version >= '3.11'", "agent-framework-hyperlight>=1.0.0b261022,<2; (sys_platform == 'linux' and platform_machine == 'x86_64') or (sys_platform == 'win32' and platform_machine == 'AMD64')", diff --git a/python/packages/foundry_local/AGENTS.md b/python/packages/foundry_local/AGENTS.md index 9f208b8e40e..449d76a295d 100644 --- a/python/packages/foundry_local/AGENTS.md +++ b/python/packages/foundry_local/AGENTS.md @@ -2,19 +2,45 @@ Integration with Microsoft Foundry Local for local model inference. +Uses `foundry_local_sdk` (SDK 2.2.1+, below 3.0) on Python 3.11-3.14. Clients share +`FoundryLocalManager.instance` and use its built-in web service with the existing +OpenAI transport. Model lifecycle operations live on catalog model objects. +The adapter defaults to the public catalog and `/v1`. Use `catalog_type="local"` +or `foundry_local_sdk.catalog.CatalogType.LOCAL` for SDK-registered BYOM models: +both lookup and HTTP inference use the local catalog and `/catalogs/local/v1`. +Catalogs never fall back to each other; per-request model overrides remain in the +selected catalog. The application owns local-model registration and assets. + +`FoundryLocalClient` exposes the SDK `Configuration` for initial manager setup, +download progress/cancellation controls, and exact device/variant selection. +`prepare_model=False` defers download/load of the configured variant until inference, +on a worker thread under the shared setup lock. Already loaded variants are reused. +Per-request model overrides remain caller-prepared; they do not prepare the configured model. +Clients own their HTTP connections, not the shared native manager. +Failed setup stops a web service it started on an existing manager without closing +that manager or stopping a previously running service. + +`FoundryLocalChatOptions` adds `top_k`, `do_sample`, and `early_stopping`. +Request preparation maps these to string-valued native decoder metadata and +unwraps OpenAI's JSON-schema envelope for the HTTP service. Explicit decoder +controls override conflicting metadata without mutating caller options. +Required tool calls use tool grammar before structured-output grammar; the standard +function loop applies the structured format to the final reply. + ## Main Classes - **`FoundryLocalClient`** - Chat client for Foundry Local models - **`FoundryLocalChatOptions`** - Options TypedDict for Foundry Local parameters -- **`FoundryLocalSettings`** - Pydantic settings for configuration +- **`FoundryLocalSettings`** - Settings TypedDict for model configuration ## Usage ```python +from agent_framework import Message from agent_framework.foundry import FoundryLocalClient client = FoundryLocalClient(model="your-local-model") -response = await client.get_response("Hello") +response = await client.get_response([Message(role="user", contents=["Hello"])]) ``` ## Import Path diff --git a/python/packages/foundry_local/README.md b/python/packages/foundry_local/README.md index 269219fac2d..44a0d83033f 100644 --- a/python/packages/foundry_local/README.md +++ b/python/packages/foundry_local/README.md @@ -1,6 +1,7 @@ # Get Started with Microsoft Agent Framework Foundry Local -Please install this package as the extra for `agent-framework`: +Requires Python 3.11-3.14 and Foundry Local SDK 2.2.1 or newer, below 3.0. +Install the integration: ```bash pip install agent-framework-foundry-local --pre @@ -8,6 +9,172 @@ pip install agent-framework-foundry-local --pre and see the [README](https://github.com/microsoft/agent-framework/tree/main/python/README.md) for more information. +## Usage + +```python +from agent_framework import Agent +from agent_framework.foundry import FoundryLocalClient + +client = FoundryLocalClient(model="phi-4-mini") +agent = Agent(client=client, instructions="You are a helpful assistant.") +response = await agent.run("Why is the sky blue?") +print(response.text) +``` + +The client initializes the Foundry Local SDK singleton, prepares the selected model, +and starts its built-in OpenAI-compatible HTTP service. Streaming, middleware, and +function invocation continue to use Agent Framework's OpenAI transport. + +All clients in a process share the SDK manager and web service. Concurrent client +construction is serialized through model preparation and failed-setup cleanup. +Deferred preparation is serialized with setup too; HTTP inference does not hold that lock. +To customize cache directories or other SDK settings, pass +`configuration=Configuration(app_name="my-app", ...)` +when creating the first client, or initialize `FoundryLocalManager` yourself before +creating clients. Register accelerated execution providers through the SDK manager +before selecting GPU/NPU variants. A `device` value of `"CPU"`, `"GPU"`, or `"NPU"` +selects a matching variant; an exact model ID is never replaced by a sibling variant. + +`bootstrap=False` requires an already initialized SDK manager with a running web +service in the same process. `prepare_model=False` defers native download/load until +the first inference request for the configured model. The adapter prepares the exact +selected variant on a worker thread before sending HTTP inference; an already loaded +variant is reused. This preserves lazy first-call loading even though the HTTP service +does not load models automatically. Applications can still download/load the variant +manually beforehand to avoid first-request preparation latency. +`timeout` applies to HTTP inference requests, not native preparation. + +`download_progress_callback` receives download percentages from 0 to 100, and +`download_cancel_event` accepts a `threading.Event` to cancel the download. +Both apply to eager and deferred preparation. Deferred progress callbacks run on a +worker thread. These controls do not cancel model loading or inference; asynchronous +request cancellation uses the normal Agent Framework API and does not interrupt +native preparation already running. + +Clients do not close or unload the shared manager. After all clients finish, close +their HTTP clients and call `client.manager.close()` once at application shutdown. +If construction fails after creating a new manager, that manager is closed; an +existing application's manager is not closed. If failed construction started +its web service, that service is stopped; a previously running service is left running. + +## Bring Your Own Model + +Use `catalog_type="local"` (or `CatalogType.LOCAL`) for models registered through +the SDK's local catalog. Model lookup and HTTP inference both use that catalog: +local clients target `/catalogs/local/v1`, while the default public clients use +`/v1`. There is no fallback between catalogs. + +Register compatible ONNX Runtime GenAI model assets through the SDK before +constructing the client. The adapter does not import or convert model files. +Registration does not transfer ownership of the files. For example: + +```python +from agent_framework import Agent +from agent_framework.foundry import FoundryLocalClient +from foundry_local_sdk import Configuration, FoundryLocalManager, ModelInfoBuilder +from foundry_local_sdk.catalog import CatalogType + +FoundryLocalManager.initialize(Configuration(app_name="my-local-agent")) +manager = FoundryLocalManager.instance +if manager is None: + raise RuntimeError("Foundry Local manager was not initialized.") +local_catalog = manager.get_catalog(CatalogType.LOCAL) +with ModelInfoBuilder() as metadata: + metadata.set_string_property("task", "chat-completion") + registered_model = local_catalog.register_model( + "your-model-directory", "my-local-model:1", metadata + ) + +client = None +try: + client = FoundryLocalClient(model=registered_model.id, catalog_type="local") + agent = Agent(client=client, instructions="You are a helpful assistant.") + response = await agent.run("What is 2 + 2?") + print(response.text) +finally: + if client is not None: + await client.client.close() + registered_model.unload() + local_catalog.unregister_model(registered_model.id) + manager.close() +``` + +The normal preparation, exact variant selection, streaming, middleware, tools, +and structured-output APIs also apply to local models, subject to model/runtime +capabilities. `prepare_model=False` also supports lazy preparation of registered local models. +Clients do not unregister local models or delete their assets; the application +owns registration and manager shutdown. + +## Chat Options + +`FoundryLocalChatOptions` includes the standard generation settings and native +`top_k`, `do_sample`, and `early_stopping` controls. The client encodes these three +controls as the HTTP service's string-valued decoder metadata, preserving zero +and `False` values. Explicit controls take precedence over conflicting values in +`metadata` or `extra_body["metadata"]`. Other native settings can be supplied +through these dictionaries. + +Pydantic response models and JSON schemas use the native HTTP service's bare-schema +format automatically. JSON-object mode constrains output to an object; it does +not guarantee particular fields or values. Supported settings and structured +output capabilities depend on the model and runtime. Backend errors remain +visible as `ChatClientException`. + +```python +from agent_framework import Message +from agent_framework.foundry import FoundryLocalChatOptions, FoundryLocalClient +from pydantic import BaseModel + + +class Answer(BaseModel): + answer: int + + +client = FoundryLocalClient(model="phi-4-mini") +options: FoundryLocalChatOptions[Answer] = { + "response_format": Answer, + "max_tokens": 128, + "temperature": 0.0, + "top_k": 1, + "do_sample": False, +} +response = await client.get_response( + [Message(role="user", contents=["What is 2 + 2?"])], + options=options, +) +assert isinstance(response.value, Answer) +print(response.value.answer) +``` + +Agent sessions, middleware, streaming, and automatic tool execution retain their +standard Agent Framework APIs. A per-request `model` option overrides inference +targeting only: that variant must also be loaded in the client's selected catalog. +An override does not switch between the public and local catalogs. +When a tool is required (including a named tool), the request uses the native tool +grammar first, then applies the requested structured format to the final reply +after automatic tool execution. With automatic execution disabled, the caller +must handle the returned function call before requesting a structured final reply. + +## Scope + +This adapter targets the SDK's public and local model catalogs and HTTP chat service. +Local models must be registered through the SDK before use. Native-session-only features, +audio transcription, and embeddings are outside this chat integration. + +Image content follows the OpenAI transport's serialization, but a successful HTTP +request or the presence of vision assets does not establish that a particular +model/runtime consumes those images. Verify multimodal behavior for the selected +configuration before relying on it. + +## Migration from SDK 0.5.1 + +This is a breaking SDK migration: Python 3.10 and SDK 0.5.1/1.x are no longer supported. +Update SDK imports from `foundry_local` to `foundry_local_sdk`. Use +`client.manager.catalog.list_models()` instead of `list_catalog_models()`, inspect +metadata through `model.info`, and call `download()`/`load()` on catalog model objects +instead of `download_model()`/`load_model()` on the manager. The SDK now runs in-process; +a separately started CLI service does not satisfy `bootstrap=False`. + ## Foundry Local Sample See the [Foundry Local provider sample](../../samples/02-agents/providers/foundry/foundry_local_agent.py) for a runnable example. diff --git a/python/packages/foundry_local/agent_framework_foundry_local/_foundry_local_client.py b/python/packages/foundry_local/agent_framework_foundry_local/_foundry_local_client.py index a36647b8e35..5012c2f5265 100644 --- a/python/packages/foundry_local/agent_framework_foundry_local/_foundry_local_client.py +++ b/python/packages/foundry_local/agent_framework_foundry_local/_foundry_local_client.py @@ -2,9 +2,11 @@ from __future__ import annotations +import asyncio import sys from collections.abc import Awaitable, Callable, Mapping, Sequence -from typing import Any, ClassVar, Generic, Literal, cast, overload +from threading import Event, Lock +from typing import Any, ClassVar, Generic, Literal, overload from agent_framework import ( ChatAndFunctionMiddlewareTypes, @@ -20,11 +22,12 @@ TokenizerProtocol, ) from agent_framework._settings import load_settings +from agent_framework.exceptions import SettingNotFoundError from agent_framework.observability import ChatTelemetryLayer from agent_framework_openai._chat_completion_client import RawOpenAIChatCompletionClient -from foundry_local import FoundryLocalManager -from foundry_local.models import DeviceType, FoundryModelInfo -from openai import AsyncOpenAI +from foundry_local_sdk import Configuration, DeviceType, FoundryLocalManager, IModel +from foundry_local_sdk.catalog import CatalogType +from openai import DEFAULT_TIMEOUT, AsyncOpenAI from pydantic import BaseModel from ._feature_usage import FeatureIndex @@ -46,6 +49,7 @@ ] ResponseModelT = TypeVar("ResponseModelT", bound=BaseModel | None, default=None) +_MANAGER_SETUP_LOCK = Lock() # region Foundry Local Chat Options TypedDict @@ -82,22 +86,33 @@ class FoundryLocalChatOptions(ChatOptions[ResponseModelT], Generic[ResponseModel # Options not supported in Foundry Local: user: Not used locally. store: Not applicable for local inference. - metadata: Not applicable for local inference. - # Foundry Local-specific options: + top_k: Number of highest-probability tokens to consider; 0 disables the limit. + do_sample: Whether to sample instead of using greedy decoding. + early_stopping: Whether to stop beam search early. + metadata: Native decoder settings encoded as string-valued metadata. extra_body: Additional request body parameters to pass to the model. Can be used for model-specific options not covered by standard API. Note: The actual options supported depend on the specific model being used. Some models (like Phi-4) may not support all OpenAI API features. - Options not supported by the model will typically be ignored. + Unsupported options may fail the request rather than being ignored. """ # Foundry Local-specific options extra_body: dict[str, Any] """Additional request body parameters for model-specific options.""" + top_k: int + """Number of highest-probability tokens to consider; 0 disables the limit.""" + + do_sample: bool + """Whether to sample instead of using greedy decoding.""" + + early_stopping: bool + """Whether to stop beam search early.""" + # ChatOptions fields not applicable for local inference user: None # type: ignore[misc] """Not used for local model inference.""" @@ -133,7 +148,7 @@ class FoundryLocalSettings(TypedDict, total=False): class FoundryLocalClient( - FunctionInvocationLayer[FoundryLocalChatOptionsT], + FunctionInvocationLayer[FoundryLocalChatOptionsT | FoundryLocalChatOptions[Any]], ChatMiddlewareLayer[FoundryLocalChatOptionsT], ChatTelemetryLayer[FoundryLocalChatOptionsT], RawOpenAIChatCompletionClient[FoundryLocalChatOptionsT], @@ -149,7 +164,7 @@ def get_response( messages: Sequence[Message], *, stream: Literal[False] = ..., - options: ChatOptions[ResponseModelT], + options: ChatOptions[ResponseModelT] | FoundryLocalChatOptions[ResponseModelT], compaction_strategy: CompactionStrategy | None = None, tokenizer: TokenizerProtocol | None = None, function_invocation_kwargs: Mapping[str, Any] | None = None, @@ -163,7 +178,7 @@ def get_response( messages: Sequence[Message], *, stream: Literal[False] = ..., - options: FoundryLocalChatOptionsT | ChatOptions[None] | None = None, + options: FoundryLocalChatOptionsT | FoundryLocalChatOptions[Any] | ChatOptions[None] | None = None, compaction_strategy: CompactionStrategy | None = None, tokenizer: TokenizerProtocol | None = None, function_invocation_kwargs: Mapping[str, Any] | None = None, @@ -177,7 +192,7 @@ def get_response( messages: Sequence[Message], *, stream: Literal[True], - options: FoundryLocalChatOptionsT | ChatOptions[Any] | None = None, + options: FoundryLocalChatOptionsT | FoundryLocalChatOptions[Any] | ChatOptions[Any] | None = None, compaction_strategy: CompactionStrategy | None = None, tokenizer: TokenizerProtocol | None = None, function_invocation_kwargs: Mapping[str, Any] | None = None, @@ -190,7 +205,7 @@ def get_response( messages: Sequence[Message], *, stream: bool = False, - options: FoundryLocalChatOptionsT | ChatOptions[Any] | None = None, + options: FoundryLocalChatOptionsT | FoundryLocalChatOptions[Any] | ChatOptions[Any] | None = None, compaction_strategy: CompactionStrategy | None = None, tokenizer: TokenizerProtocol | None = None, function_invocation_kwargs: Mapping[str, Any] | None = None, @@ -198,16 +213,22 @@ def get_response( middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, ) -> Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]: """Get a response from the Foundry Local chat client with all standard layers enabled.""" - super_get_response = cast( - "Callable[..., Awaitable[ChatResponse[Any]] | ResponseStream[ChatResponseUpdate, ChatResponse[Any]]]", - super().get_response, - ) effective_client_kwargs = dict(client_kwargs) if client_kwargs is not None else {} if middleware is not None: effective_client_kwargs["middleware"] = middleware - return super_get_response( + if stream: + return super().get_response( + messages=messages, + stream=True, + options=options, + compaction_strategy=compaction_strategy, + tokenizer=tokenizer, + function_invocation_kwargs=function_invocation_kwargs, + client_kwargs=effective_client_kwargs, + ) + return super().get_response( messages=messages, - stream=stream, + stream=False, options=options, compaction_strategy=compaction_strategy, tokenizer=tokenizer, @@ -215,6 +236,94 @@ def get_response( client_kwargs=effective_client_kwargs, ) + def _inner_get_response( + self, + *, + messages: Sequence[Message], + options: Mapping[str, Any], + stream: bool = False, + **kwargs: Any, + ) -> Awaitable[ChatResponse] | ResponseStream[ChatResponseUpdate, ChatResponse]: + inner_get_response = super()._inner_get_response + if not self._prepare_model_on_request or (options.get("model") or self.model) != self._model_handle.id: + return inner_get_response(messages=messages, options=options, stream=stream, **kwargs) + + if stream: + + async def _get_stream() -> ResponseStream[ChatResponseUpdate, ChatResponse]: + await asyncio.to_thread(self._ensure_model_loaded) + response = inner_get_response(messages=messages, options=options, stream=True, **kwargs) + if not isinstance(response, ResponseStream): + raise TypeError("Streaming responses must return a ResponseStream.") + return response # type: ignore[reportUnknownVariableType] + + return ResponseStream[ChatResponseUpdate, ChatResponse].from_awaitable(_get_stream()) + + async def _get_response() -> ChatResponse: + await asyncio.to_thread(self._ensure_model_loaded) + return await inner_get_response(messages=messages, options=options, stream=False, **kwargs) + + return _get_response() + + def _ensure_model_loaded(self) -> None: + with _MANAGER_SETUP_LOCK: + if not self._model_handle.is_loaded: + self._prepare_model() + + def _prepare_model(self) -> None: + if self._download_progress_callback is None and self._download_cancel_event is None: + self._model_handle.download() + else: + self._model_handle.download( + progress_callback=self._download_progress_callback, cancel_event=self._download_cancel_event + ) + self._model_handle.load() + + def _prepare_options(self, messages: Sequence[Message], options: Mapping[str, Any]) -> dict[str, Any]: + prepared = super()._prepare_options(messages, options) + native_metadata: dict[str, str] = {} + for key in ("top_k", "do_sample", "early_stopping"): + value = prepared.pop(key, None) + if value is not None: + native_metadata[key] = str(value).lower() if isinstance(value, bool) else str(value) + extra_body: Mapping[str, Any] | None = prepared.get("extra_body") + if extra_body is not None and not isinstance(extra_body, Mapping): + raise ValueError("extra_body must be a mapping.") + metadata: Mapping[str, Any] | None = prepared.get("metadata") + extra_metadata: Mapping[str, Any] | None = extra_body.get("metadata") if extra_body is not None else None + for name, value in (("metadata", metadata), ("extra_body metadata", extra_metadata)): + if value is not None and not isinstance(value, Mapping): + raise ValueError(f"{name} must be a mapping.") + if native_metadata or (metadata is not None and extra_metadata is not None): + merged_metadata: dict[str, Any] = { + **(metadata or {}), + **(extra_metadata or {}), + **native_metadata, + } + prepared["metadata"] = merged_metadata + if extra_body is not None and "metadata" in extra_body: + prepared["extra_body"] = {**extra_body, "metadata": merged_metadata} + # The HTTP service uses max_tokens and a bare schema, not OpenAI's newer token key or named schema envelope. + if "max_completion_tokens" in prepared: + prepared["max_tokens"] = prepared.pop("max_completion_tokens") + response_format: Mapping[str, Any] | None = prepared.get("response_format") + if isinstance(response_format, Mapping): + if response_format.get("type") == "json_schema": + schema: Mapping[str, Any] | None = response_format.get("json_schema") + if isinstance(schema, Mapping): + prepared["response_format"] = { + **response_format, + "json_schema": schema.get("schema", schema), + } + elif response_format.get("type") not in ("json_object", "text"): + # Older OpenAI adapters pass raw JSON schemas through unchanged. + prepared["response_format"] = {"type": "json_schema", "json_schema": response_format} + tool_choice: str | Mapping[str, Any] | None = prepared.get("tool_choice") + if tool_choice == "required" or (isinstance(tool_choice, Mapping) and tool_choice.get("type") == "function"): + # Native schema grammar overrides forced tools; retain it only for the final reply. + prepared.pop("response_format", None) + return prepared + def __init__( self, model: str | None = None, @@ -222,7 +331,11 @@ def __init__( bootstrap: bool = True, timeout: float | None = None, prepare_model: bool = True, - device: DeviceType | None = None, + catalog_type: CatalogType | Literal["public", "local"] = CatalogType.PUBLIC, + download_progress_callback: Callable[[float], None] | None = None, + download_cancel_event: Event | None = None, + device: DeviceType | Literal["CPU", "GPU", "NPU"] | None = None, + configuration: Configuration | None = None, additional_properties: dict[str, Any] | None = None, middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, function_invocation_configuration: FunctionInvocationConfiguration | None = None, @@ -234,18 +347,33 @@ def __init__( Keyword Args: model: The Foundry Local model ID or alias to use. If not provided, it will be loaded from the FoundryLocalSettings. - bootstrap: Whether to start the Foundry Local service if not already running. - Default is True. - timeout: Optional timeout for requests to Foundry Local. - This timeout is applied to any call to the Foundry Local service. + bootstrap: Whether to initialize the SDK singleton and start its built-in web + service if needed. Default is True. When False, the manager and web + service must already be running in this process. + timeout: Optional timeout for HTTP inference requests. Uses the OpenAI + client's default when omitted; does not apply to native model preparation. prepare_model: Whether to download the model into the cache, and load the model into the inferencing service upon initialization. Default is True. - If false, the first call to generate a completion will load the model, - and might take a long time. + If False, preparation is deferred until the first inference request for the + configured model. An already loaded variant is reused. The first request + may take longer while downloading and loading the model. + catalog_type: The catalog used for model lookup and HTTP inference. + Defaults to "public". Use "local" for models already registered through + the SDK's local catalog. Also accepts foundry_local_sdk.catalog.CatalogType. + Per-request model overrides remain in the selected catalog. + download_progress_callback: Optional callback receiving native download progress + as a percentage from 0 to 100, including deferred preparation. + Deferred callbacks run on a worker thread. + download_cancel_event: Optional event that cancels native model download when set. + Also applies to deferred preparation; does not cancel model loading or inference. + Cancelling an awaiting request does not interrupt native preparation already in progress. device: The device type to use for model inference. The device is used to select the appropriate model variant. If not provided, the default device for your system will be used. - The values are in the foundry_local.models.DeviceType enum. + Accepts "CPU", "GPU", "NPU", or foundry_local_sdk.DeviceType. + configuration: SDK configuration used when initializing a new manager. + Defaults to Configuration(app_name="agent-framework"). To share a manager, + initialize it before creating clients and omit this parameter. additional_properties: Additional properties stored on the client instance. middleware: Optional sequence of ChatAndFunctionMiddlewareTypes to apply to requests. function_invocation_configuration: Optional configuration for function invocation support. @@ -261,7 +389,10 @@ def __init__( client = FoundryLocalClient(model="phi-4-mini") - agent = client.as_agent( + from agent_framework import Agent + + agent = Agent( + client=client, name="LocalAgent", instructions="You are a helpful agent.", tools=get_weather, @@ -272,14 +403,13 @@ def __init__( os.environ["FOUNDRY_LOCAL_MODEL"] = "phi-4-mini" client = FoundryLocalClient() - # A FoundryLocalManager is created and if set, the service is started. - # The FoundryLocalManager is available via the `manager` property. + # The SDK singleton is shared by all clients in this process. # For instance to find out which models are available: - for model in client.manager.list_catalog_models(): - print(f"- {model.alias} for {model.task} - id={model.id}") + for model in client.manager.catalog.list_models(): + print(f"- {model.alias} for {model.info.task} - id={model.id}") # Other options include specifying the device type: - from foundry_local.models import DeviceType + from foundry_local_sdk import DeviceType client = FoundryLocalClient( model="phi-4-mini", @@ -290,30 +420,27 @@ def __init__( model="phi-4-mini", prepare_model=False, ) - # Beware, in this case the first request to generate a completion - # will take a long time as the model is loaded then. - # Alternatively, you could call the `download_model` and `load_model` methods - # on the `manager` property manually. - client.manager.download_model("phi-4-mini", device=DeviceType.CPU) - client.manager.load_model("phi-4-mini", device=DeviceType.CPU) - - # You can also use the CLI: - `foundry model load phi-4-mini --device Auto` - - # Using custom ChatOptions with type safety: - from typing import TypedDict + # The first inference request prepares the exact resolved variant. + # Alternatively, prepare it manually before inference: + model = client.manager.catalog.get_model_variant(client.model) + if model is not None: + model.download() + model.load() + + # Using native ChatOptions with type safety: + from agent_framework import Message from agent_framework.foundry import FoundryLocalChatOptions - class MyOptions(FoundryLocalChatOptions, total=False): - my_custom_option: str - - client: FoundryLocalClient[MyOptions] = FoundryLocalClient(model="phi-4-mini") - response = await client.get_response("Hello", options={"my_custom_option": "value"}) + options: FoundryLocalChatOptions[None] = {"top_k": 40, "max_tokens": 100} + response = await client.get_response( + [Message(role="user", contents=["Hello"])], + options=options, + ) Raises: - ValueError: If the specified model ID or alias is not found. - Sometimes a model might be available but if you have specified a device - type that is not supported by the model, it will not be found. + ValueError: If the model or requested device is unavailable, bootstrap is + disabled without a running manager/service, or configuration is supplied + for an already initialized manager. """ settings = load_settings( @@ -324,29 +451,91 @@ class MyOptions(FoundryLocalChatOptions, total=False): env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) - model_setting: str = settings["model"] # type: ignore[assignment] - - manager = FoundryLocalManager(bootstrap=bootstrap, timeout=timeout) - model_info: FoundryModelInfo | None = manager.get_model_info( - model_setting, - device=device, + model_setting = settings.get("model") + if model_setting is None: + raise SettingNotFoundError("Required setting 'model' was not provided.") + device_type = DeviceType(device) if device is not None else None + catalog_kind = ( + {"public": CatalogType.PUBLIC, "local": CatalogType.LOCAL}.get(catalog_type) + if isinstance(catalog_type, str) + else catalog_type ) - if model_info is None: - message = ( - f"Model with ID or alias '{model_setting}:{device.value}' not found in Foundry Local." - if device - else (f"Model with ID or alias '{model_setting}' for your current device not found in Foundry Local.") - ) - raise ValueError(message) - if prepare_model: - manager.download_model(model_info.id, device=device) - manager.load_model(model_info.id, device=device) - - super().__init__( - model=model_info.id, - async_client=AsyncOpenAI(base_url=manager.endpoint, api_key=manager.api_key), - additional_properties=additional_properties, - middleware=middleware, - function_invocation_configuration=function_invocation_configuration, - ) - self.manager = manager + if not isinstance(catalog_kind, CatalogType): + raise ValueError("catalog_type must be 'public', 'local', or a CatalogType.") + + with _MANAGER_SETUP_LOCK: + manager = FoundryLocalManager.instance + owns_manager = manager is None + if manager is None: + if not bootstrap: + raise ValueError( + "bootstrap=False requires an initialized Foundry Local manager and running web service." + ) + FoundryLocalManager.initialize(configuration or Configuration(app_name="agent-framework")) + manager = FoundryLocalManager.instance + if manager is None: + raise RuntimeError("Foundry Local manager initialization did not create a singleton.") + elif configuration is not None: + raise ValueError( + "Foundry Local is already initialized; configure the shared manager before creating clients." + ) + + initialized = False + started_web_service = False + try: + if not manager.urls: + if not bootstrap: + raise ValueError("bootstrap=False requires a running Foundry Local web service.") + manager.start_web_service() + started_web_service = bool(manager.urls) + if not manager.urls: + raise ValueError("Foundry Local web service did not provide a bound URL.") + + catalog = manager.catalog if catalog_kind == CatalogType.PUBLIC else manager.get_catalog(catalog_kind) + is_variant_id = ":" in model_setting + model_handle = ( + catalog.get_model_variant(model_setting) if is_variant_id else catalog.get_model(model_setting) + ) + if model_handle is not None and device_type is not None: + model_handle = next( + ( + variant + for variant in ([model_handle] if is_variant_id else model_handle.variants) + if (runtime := variant.info.runtime) is not None and runtime.device_type == device_type + ), + None, + ) + elif model_handle is not None and not is_variant_id: + # Pin the alias's chosen variant before download changes the cached preference. + model_handle = catalog.get_model_variant(model_handle.id) + if model_handle is None: + target = f"{model_setting}:{device_type.value}" if device_type is not None else model_setting + raise ValueError(f"Model with ID or alias '{target}' not found in Foundry Local.") + + model_id = model_handle.id + self._model_handle: IModel = model_handle + self._prepare_model_on_request = not prepare_model + self._download_progress_callback = download_progress_callback + self._download_cancel_event = download_cancel_event + if prepare_model: + self._prepare_model() + + catalog_path = "/catalogs/local" if catalog_kind == CatalogType.LOCAL else "" + super().__init__( + model=model_id, + async_client=AsyncOpenAI( + base_url=f"{manager.urls[0].rstrip('/')}{catalog_path}/v1", + api_key="foundry-local", + timeout=timeout if timeout is not None else DEFAULT_TIMEOUT, + ), + additional_properties=additional_properties, + middleware=middleware, + function_invocation_configuration=function_invocation_configuration, + ) + self.manager = manager + initialized = True + finally: + if owns_manager and not initialized: + manager.close() + elif started_web_service and not initialized: + manager.stop_web_service() diff --git a/python/packages/foundry_local/pyproject.toml b/python/packages/foundry_local/pyproject.toml index 43b56795cfd..55905709146 100644 --- a/python/packages/foundry_local/pyproject.toml +++ b/python/packages/foundry_local/pyproject.toml @@ -3,7 +3,7 @@ name = "agent-framework-foundry-local" description = "Foundry Local integration for Microsoft Agent Framework." authors = [{ name = "Microsoft", email = "af-support@microsoft.com"}] readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11,<3.15" version = "1.0.0b261015" license-files = ["LICENSE"] urls.homepage = "https://aka.ms/agent-framework" @@ -15,18 +15,16 @@ classifiers = [ "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", - "Programming Language :: Python :: 3.15", "Typing :: Typed", ] dependencies = [ "agent-framework-core>=1.13.0,<2", "agent-framework-openai>=1.6.0,<2", - "foundry-local-sdk>=0.5.1,<0.5.2", + "foundry-local-sdk>=2.2.1,<3", ] [tool.uv] @@ -67,7 +65,7 @@ exclude = ['tests'] [tool.mypy] plugins = ['pydantic.mypy'] strict = true -python_version = "3.10" +python_version = "3.11" ignore_missing_imports = true disallow_untyped_defs = true no_implicit_optional = true diff --git a/python/packages/foundry_local/tests/conftest.py b/python/packages/foundry_local/tests/conftest.py index cd4356d0b56..99ec805b9ea 100644 --- a/python/packages/foundry_local/tests/conftest.py +++ b/python/packages/foundry_local/tests/conftest.py @@ -1,7 +1,10 @@ # Copyright (c) Microsoft. All rights reserved. +from collections.abc import Callable, Iterator +from types import SimpleNamespace from typing import Any -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch +from foundry_local_sdk import Catalog, DeviceType, FoundryLocalManager, IModel from pytest import fixture @@ -42,14 +45,43 @@ def foundry_local_unit_test_env(monkeypatch: Any, exclude_list: list[str], overr @fixture -def mock_foundry_local_manager() -> MagicMock: - """Fixture that provides a mock FoundryLocalManager.""" - mock_manager = MagicMock() - mock_manager.endpoint = "http://localhost:5272/v1" - mock_manager.api_key = "test-api-key" +def mock_foundry_local_manager_factory() -> Callable[[], MagicMock]: + """Create independent mock managers with their own catalog models.""" + + def create_manager() -> MagicMock: + mock_manager = MagicMock(spec=FoundryLocalManager) + mock_manager.urls = ["http://localhost:5272"] + mock_manager.catalog = MagicMock(spec=Catalog) + + mock_model = MagicMock(spec=IModel) + mock_model.id = "test-model-id" + mock_model.info = SimpleNamespace(runtime=SimpleNamespace(device_type=DeviceType.CPU)) + mock_model.variants = [mock_model] + mock_model.is_loaded = False + mock_model.load.side_effect = lambda: setattr(mock_model, "is_loaded", True) + mock_manager.catalog.get_model.return_value = mock_model + mock_manager.catalog.get_model_variant.return_value = mock_model + + local_catalog = MagicMock(spec=Catalog) + local_model = MagicMock(spec=IModel) + local_model.id = "local-model:1" + local_model.info = SimpleNamespace(runtime=SimpleNamespace(device_type=DeviceType.CPU)) + local_model.variants = [local_model] + local_model.is_loaded = False + local_model.load.side_effect = lambda: setattr(local_model, "is_loaded", True) + local_catalog.get_model.return_value = local_model + local_catalog.get_model_variant.return_value = local_model + mock_manager.get_catalog.return_value = local_catalog + return mock_manager + + return create_manager + - mock_model_info = MagicMock() - mock_model_info.id = "test-model-id" - mock_manager.get_model_info.return_value = mock_model_info +@fixture +def mock_foundry_local_manager(mock_foundry_local_manager_factory: Callable[[], MagicMock]) -> Iterator[MagicMock]: + """Fixture that provides a mock FoundryLocalManager.""" + mock_manager = mock_foundry_local_manager_factory() - return mock_manager + with patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class: + manager_class.instance = mock_manager + yield mock_manager diff --git a/python/packages/foundry_local/tests/test_foundry_local_client.py b/python/packages/foundry_local/tests/test_foundry_local_client.py index 2ba5e9d89c0..8ce116f4128 100644 --- a/python/packages/foundry_local/tests/test_foundry_local_client.py +++ b/python/packages/foundry_local/tests/test_foundry_local_client.py @@ -1,18 +1,45 @@ # Copyright (c) Microsoft. All rights reserved. +import asyncio import inspect +import json +from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import asynccontextmanager +from copy import deepcopy +from importlib import import_module +from queue import Queue +from threading import Event, Lock, get_ident +from types import SimpleNamespace +from typing import Any, Literal, cast from unittest.mock import MagicMock, patch import pytest -from agent_framework import Agent, SupportsChatGetResponse +from agent_framework import ( + Agent, + ChatAndFunctionMiddlewareTypes, + ChatContext, + ChatOptions, + Content, + FunctionInvocationConfiguration, + Message, + SupportsChatGetResponse, + chat_middleware, + tool, +) from agent_framework._settings import load_settings -from agent_framework.exceptions import SettingNotFoundError -from agent_framework.foundry import FoundryLocalClient +from agent_framework.exceptions import ChatClientException, SettingNotFoundError +from agent_framework.foundry import FoundryLocalChatOptions, FoundryLocalClient +from agent_framework_openai import RawOpenAIChatCompletionClient +from foundry_local_sdk import Configuration, DeviceType, IModel +from foundry_local_sdk.catalog import CatalogType +from openai import AsyncOpenAI, DefaultAsyncHttpxClient +from pydantic import BaseModel from agent_framework_foundry_local._feature_usage import FeatureIndex from agent_framework_foundry_local._foundry_local_client import FoundryLocalSettings -# Settings Tests +_OPENAI_HTTPX = cast(Any, import_module(DefaultAsyncHttpxClient.__mro__[1].__module__.partition(".")[0])) def test_foundry_local_owns_foundry_local_feature() -> None: @@ -20,213 +47,1416 @@ def test_foundry_local_owns_foundry_local_feature() -> None: def test_foundry_local_settings_init_from_env(foundry_local_unit_test_env: dict[str, str]) -> None: - """Test FoundryLocalSettings initialization from environment variables.""" settings = load_settings(FoundryLocalSettings, env_prefix="FOUNDRY_LOCAL_") - assert settings["model"] == foundry_local_unit_test_env["FOUNDRY_LOCAL_MODEL"] def test_foundry_local_settings_init_with_explicit_values() -> None: - """Test FoundryLocalSettings initialization with explicit values.""" - settings = load_settings( - FoundryLocalSettings, - env_prefix="FOUNDRY_LOCAL_", - model="custom-model-id", - ) - + settings = load_settings(FoundryLocalSettings, env_prefix="FOUNDRY_LOCAL_", model="custom-model-id") assert settings["model"] == "custom-model-id" @pytest.mark.parametrize("exclude_list", [["FOUNDRY_LOCAL_MODEL"]], indirect=True) def test_foundry_local_settings_missing_model(foundry_local_unit_test_env: dict[str, str]) -> None: - """Test FoundryLocalSettings when model is missing raises error.""" with pytest.raises(SettingNotFoundError, match="Required setting 'model'"): - load_settings( - FoundryLocalSettings, - env_prefix="FOUNDRY_LOCAL_", - required_fields=["model"], - ) + load_settings(FoundryLocalSettings, env_prefix="FOUNDRY_LOCAL_", required_fields=["model"]) def test_foundry_local_settings_explicit_overrides_env(foundry_local_unit_test_env: dict[str, str]) -> None: - """Test that explicit values override environment variables.""" settings = load_settings(FoundryLocalSettings, env_prefix="FOUNDRY_LOCAL_", model="override-model-id") - assert settings["model"] == "override-model-id" assert settings["model"] != foundry_local_unit_test_env["FOUNDRY_LOCAL_MODEL"] -# Client Initialization Tests +def test_foundry_local_client_init(mock_foundry_local_manager: MagicMock) -> None: + client = FoundryLocalClient(model="test-model-id") + assert client.model == "test-model-id" + assert client.manager is mock_foundry_local_manager + assert isinstance(client, SupportsChatGetResponse) + assert str(client.client.base_url) == "http://localhost:5272/v1/" + assert client.client.api_key == "foundry-local" -def test_foundry_local_client_init(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient initialization with mocked manager.""" - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, - ): - client = FoundryLocalClient(model="test-model-id") - assert client.model == "test-model-id" - assert client.manager is mock_foundry_local_manager - assert isinstance(client, SupportsChatGetResponse) +@pytest.mark.parametrize("catalog_type", ["local", CatalogType.LOCAL]) +@pytest.mark.parametrize("model", ["local-model", "local-model:1"]) +def test_foundry_local_client_selects_local_catalog( + mock_foundry_local_manager: MagicMock, catalog_type: Literal["local"] | CatalogType, model: str +) -> None: + client = FoundryLocalClient(model=model, catalog_type=catalog_type) + local_catalog = mock_foundry_local_manager.get_catalog.return_value + mock_foundry_local_manager.get_catalog.assert_called_once_with(CatalogType.LOCAL) + mock_foundry_local_manager.catalog.get_model.assert_not_called() + mock_foundry_local_manager.catalog.get_model_variant.assert_not_called() + local_catalog.get_model_variant.assert_called_with("local-model:1") + local_catalog.get_model_variant.return_value.download.assert_called_once_with() + local_catalog.get_model_variant.return_value.load.assert_called_once_with() + assert client.model == "local-model:1" + assert str(client.client.base_url) == "http://localhost:5272/catalogs/local/v1/" -def test_agent_accepts_foundry_local_client(mock_foundry_local_manager: MagicMock) -> None: - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, + +@pytest.mark.parametrize("catalog_type", ["public", CatalogType.PUBLIC]) +def test_foundry_local_client_selects_public_catalog( + mock_foundry_local_manager: MagicMock, catalog_type: Literal["public"] | CatalogType +) -> None: + client = FoundryLocalClient(model="test-model-id", catalog_type=catalog_type) + mock_foundry_local_manager.get_catalog.assert_not_called() + assert client.model == "test-model-id" + assert str(client.client.base_url) == "http://localhost:5272/v1/" + + +@pytest.mark.parametrize("catalog_type", ["missing", None, 2]) +def test_foundry_local_client_rejects_invalid_catalog_before_setup( + mock_foundry_local_manager: MagicMock, catalog_type: Any +) -> None: + with ( + patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class, + pytest.raises(ValueError, match="catalog_type"), ): - client = FoundryLocalClient(model="test-model-id") + manager_class.instance = None + FoundryLocalClient(model="test-model-id", catalog_type=catalog_type) + manager_class.initialize.assert_not_called() + mock_foundry_local_manager.start_web_service.assert_not_called() + + +def test_foundry_local_client_local_catalog_does_not_fall_back_to_public( + mock_foundry_local_manager: MagicMock, +) -> None: + mock_foundry_local_manager.get_catalog.return_value.get_model_variant.return_value = None + with pytest.raises(ValueError, match="not found"): + FoundryLocalClient(model="test-model-id:1", catalog_type="local") + mock_foundry_local_manager.catalog.get_model_variant.assert_not_called() + mock_foundry_local_manager.close.assert_not_called() + mock_foundry_local_manager.stop_web_service.assert_not_called() + + +def test_foundry_local_client_local_catalog_respects_manual_preparation(mock_foundry_local_manager: MagicMock) -> None: + client = FoundryLocalClient(model="local-model:1", catalog_type="local", prepare_model=False, bootstrap=False) + local_model = mock_foundry_local_manager.get_catalog.return_value.get_model_variant.return_value + local_model.download.assert_not_called() + local_model.load.assert_not_called() + mock_foundry_local_manager.start_web_service.assert_not_called() + assert str(client.client.base_url) == "http://localhost:5272/catalogs/local/v1/" + + +def test_foundry_local_client_local_catalog_selects_exact_device_variant(mock_foundry_local_manager: MagicMock) -> None: + with pytest.raises(ValueError, match="not found"): + FoundryLocalClient(model="local-model:1", catalog_type="local", device=DeviceType("GPU")) + local_catalog = mock_foundry_local_manager.get_catalog.return_value + local_catalog.get_model.assert_not_called() + local_catalog.get_model_variant.return_value.load.assert_not_called() + + +def test_foundry_local_client_local_catalog_preserves_download_controls(mock_foundry_local_manager: MagicMock) -> None: + progress: list[float] = [] + cancel_event = Event() + FoundryLocalClient( + model="local-model:1", + catalog_type="local", + download_progress_callback=progress.append, + download_cancel_event=cancel_event, + ) + local_model = mock_foundry_local_manager.get_catalog.return_value.get_model_variant.return_value + local_model.download.assert_called_once_with(progress_callback=progress.append, cancel_event=cancel_event) + local_model.load.assert_called_once_with() + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("override_model", [False, True]) +async def test_foundry_local_client_local_catalog_http_route( + mock_foundry_local_manager: MagicMock, stream: bool, override_model: bool +) -> None: + requests: list[Any] = [] + + def respond(request: Any) -> Any: + requests.append(request) + return _completion_response({"content": "Hello"}, stream=stream) + async with _client_with_transport(respond, catalog_type="local", model="local-model:1") as client: + messages = [Message(role="user", contents=["Hi"])] + options: FoundryLocalChatOptions[None] = {"model": "other-local:2"} if override_model else {} + if stream: + response = await client.get_response(messages, stream=True, options=options).get_final_response() + else: + response = await client.get_response(messages, options=options) + assert response.text == "Hello" + assert requests[0].url.path == "/catalogs/local/v1/chat/completions" + assert json.loads(requests[0].content)["model"] == ("other-local:2" if override_model else "local-model:1") + + +def test_agent_accepts_foundry_local_client(mock_foundry_local_manager: MagicMock) -> None: + client = FoundryLocalClient(model="test-model-id") agent = Agent(client=client, instructions="test agent") assert agent.client is client def test_foundry_local_client_get_response_uses_explicit_runtime_buckets() -> None: - """Foundry Local should expose explicit runtime buckets instead of raw kwargs.""" signature = inspect.signature(FoundryLocalClient.get_response) - assert "client_kwargs" in signature.parameters assert "function_invocation_kwargs" in signature.parameters assert all(parameter.kind != inspect.Parameter.VAR_KEYWORD for parameter in signature.parameters.values()) -def test_foundry_local_client_init_with_bootstrap_false(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient initialization with bootstrap=False.""" - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, - ) as mock_manager_class: - FoundryLocalClient(model="test-model-id", bootstrap=False) +def test_foundry_local_client_initializes_singleton(mock_foundry_local_manager: MagicMock) -> None: + with patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class: + manager_class.instance = None + manager_class.initialize.side_effect = lambda _: setattr(manager_class, "instance", mock_foundry_local_manager) - mock_manager_class.assert_called_once_with( - bootstrap=False, - timeout=None, - ) + client = FoundryLocalClient(model="test-model-id") + manager_class.initialize.assert_called_once() + assert manager_class.initialize.call_args.args[0].app_name == "agent-framework" + assert client.manager is mock_foundry_local_manager -def test_foundry_local_client_init_with_timeout(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient initialization with custom timeout.""" - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, - ) as mock_manager_class: - FoundryLocalClient(model="test-model-id", timeout=60.0) - - mock_manager_class.assert_called_once_with( - bootstrap=True, - timeout=60.0, - ) +def test_foundry_local_client_accepts_configuration(mock_foundry_local_manager: MagicMock) -> None: + configuration = Configuration(app_name="my-app", model_cache_dir="my-models") + with patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class: + manager_class.instance = None + manager_class.initialize.side_effect = lambda _: setattr(manager_class, "instance", mock_foundry_local_manager) + + FoundryLocalClient(model="test-model-id", configuration=configuration) + + manager_class.initialize.assert_called_once_with(configuration) + + +def test_foundry_local_client_reuses_singleton(mock_foundry_local_manager: MagicMock) -> None: + with patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class: + manager_class.instance = mock_foundry_local_manager + FoundryLocalClient(model="test-model-id") + FoundryLocalClient(model="test-model-id") + manager_class.initialize.assert_not_called() + mock_foundry_local_manager.start_web_service.assert_not_called() -def test_foundry_local_client_init_model_not_found(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient initialization when model is not found.""" - mock_foundry_local_manager.get_model_info.return_value = None + +@pytest.mark.parametrize("first_fails", [False, True]) +def test_foundry_local_client_serializes_concurrent_setup( + mock_foundry_local_manager_factory: Callable[[], MagicMock], first_fails: bool +) -> None: + first_manager = mock_foundry_local_manager_factory() + first_manager.urls = None + replacement_manager = mock_foundry_local_manager_factory() + preparing = Event() + release_preparation = Event() + lock_attempts: Queue[None] = Queue() + setup_lock = Lock() + observed_lock = MagicMock() + + def enter_lock() -> None: + lock_attempts.put(None) + setup_lock.acquire() + + observed_lock.__enter__.side_effect = enter_lock + observed_lock.__exit__.side_effect = lambda *_: setup_lock.release() + + def prepare_first_model() -> None: + assert setup_lock.locked(), "Model preparation must remain inside the shared setup lock." + preparing.set() + assert release_preparation.wait(5), "Timed out waiting to release model preparation." + if first_fails: + raise RuntimeError("model preparation failed") + + first_manager.catalog.get_model_variant.return_value.download.side_effect = prepare_first_model + first_manager.start_web_service.side_effect = lambda: setattr(first_manager, "urls", ["http://localhost:5272"]) + + def create_client(prepare_model: bool = True) -> FoundryLocalClient[FoundryLocalChatOptions[None]]: + return FoundryLocalClient(model="test-model-id", prepare_model=prepare_model) with ( + patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class, + patch("agent_framework_foundry_local._foundry_local_client._MANAGER_SETUP_LOCK", observed_lock, create=True), patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, + "agent_framework_foundry_local._foundry_local_client.AsyncOpenAI", return_value=MagicMock(spec=AsyncOpenAI) ), - pytest.raises(ValueError, match="not found in Foundry Local"), + ThreadPoolExecutor(max_workers=2) as executor, ): - FoundryLocalClient(model="unknown-model") + manager_class.instance = None + managers = iter([first_manager, replacement_manager]) + manager_class.initialize.side_effect = lambda _: setattr(manager_class, "instance", next(managers)) + + def close_first_manager() -> None: + assert setup_lock.locked(), "Failed setup must finish cleanup before another client can use the manager." + manager_class.instance = None + first_manager.close.side_effect = close_first_manager + first = executor.submit(create_client) + try: + assert preparing.wait(5), "First client did not reach model preparation." + lock_attempts.get(timeout=5) + second = executor.submit(create_client, prepare_model=False) + lock_attempts.get(timeout=5) + assert not second.done() + manager_class.initialize.assert_called_once() + first_manager.start_web_service.assert_called_once_with() + finally: + release_preparation.set() -def test_foundry_local_client_uses_model_info_id(mock_foundry_local_manager: MagicMock) -> None: - """Test that client uses the model ID from model_info, not the alias.""" - mock_model_info = MagicMock() - mock_model_info.id = "resolved-model-id" - mock_foundry_local_manager.get_model_info.return_value = mock_model_info + if first_fails: + with pytest.raises(RuntimeError, match="model preparation failed"): + first.result(timeout=5) + assert second.result(timeout=5).manager is replacement_manager + assert manager_class.initialize.call_count == 2 + first_manager.close.assert_called_once_with() + replacement_manager.close.assert_not_called() + else: + assert first.result(timeout=5).manager is first_manager + assert second.result(timeout=5).manager is first_manager + manager_class.initialize.assert_called_once() + first_manager.close.assert_not_called() + first_manager.start_web_service.assert_called_once_with() - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, + +def test_foundry_local_client_rejects_configuration_for_existing_manager(mock_foundry_local_manager: MagicMock) -> None: + with pytest.raises(ValueError, match="already initialized"): + FoundryLocalClient(model="test-model-id", configuration=Configuration(app_name="another-app")) + + +def test_foundry_local_client_bootstrap_false_requires_manager(mock_foundry_local_manager: MagicMock) -> None: + with ( + patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class, + pytest.raises(ValueError, match="bootstrap=False"), ): - client = FoundryLocalClient(model="model-alias") + manager_class.instance = None + FoundryLocalClient(model="test-model-id", bootstrap=False) + manager_class.initialize.assert_not_called() + + +def test_foundry_local_client_bootstrap_false_requires_service(mock_foundry_local_manager: MagicMock) -> None: + mock_foundry_local_manager.urls = None + with pytest.raises(ValueError, match="bootstrap=False"): + FoundryLocalClient(model="test-model-id", bootstrap=False) + mock_foundry_local_manager.start_web_service.assert_not_called() - assert client.model == "resolved-model-id" + +def test_foundry_local_client_bootstrap_false_reuses_service(mock_foundry_local_manager: MagicMock) -> None: + client = FoundryLocalClient(model="test-model-id", bootstrap=False) + assert client.manager is mock_foundry_local_manager + mock_foundry_local_manager.start_web_service.assert_not_called() + + +def test_foundry_local_client_starts_web_service(mock_foundry_local_manager: MagicMock) -> None: + mock_foundry_local_manager.urls = None + mock_foundry_local_manager.start_web_service.side_effect = lambda: setattr( + mock_foundry_local_manager, "urls", ["http://127.0.0.1:4321/"] + ) + client = FoundryLocalClient(model="test-model-id") + mock_foundry_local_manager.start_web_service.assert_called_once_with() + assert str(client.client.base_url) == "http://127.0.0.1:4321/v1/" + + +def test_foundry_local_client_requires_bound_url(mock_foundry_local_manager: MagicMock) -> None: + mock_foundry_local_manager.urls = [] + with pytest.raises(ValueError, match="bound URL"): + FoundryLocalClient(model="test-model-id") + + +@pytest.mark.parametrize("service_running", [False, True]) +@pytest.mark.parametrize("failure_stage", ["lookup", "download", "load", "http"]) +@pytest.mark.parametrize("catalog_type", ["public", "local"]) +def test_foundry_local_client_rolls_back_service_started_on_existing_manager( + mock_foundry_local_manager: MagicMock, + service_running: bool, + failure_stage: str, + catalog_type: Literal["public", "local"], +) -> None: + if not service_running: + mock_foundry_local_manager.urls = None + mock_foundry_local_manager.start_web_service.side_effect = lambda: setattr( + mock_foundry_local_manager, "urls", ["http://localhost:5272"] + ) + catalog = ( + mock_foundry_local_manager.catalog + if catalog_type == "public" + else mock_foundry_local_manager.get_catalog.return_value + ) + model = catalog.get_model_variant.return_value + if failure_stage == "lookup": + catalog.get_model.return_value = None + elif failure_stage in ("download", "load"): + getattr(model, failure_stage).side_effect = RuntimeError("setup failed") + with ( + patch("agent_framework_foundry_local._foundry_local_client.AsyncOpenAI") as http_client, + pytest.raises(ValueError if failure_stage == "lookup" else RuntimeError), + ): + if failure_stage == "http": + http_client.side_effect = RuntimeError("setup failed") + FoundryLocalClient(model="test-model-id", catalog_type=catalog_type) + mock_foundry_local_manager.close.assert_not_called() + if service_running: + mock_foundry_local_manager.stop_web_service.assert_not_called() + else: + mock_foundry_local_manager.start_web_service.assert_called_once_with() + mock_foundry_local_manager.stop_web_service.assert_called_once_with() + + +def test_foundry_local_client_init_with_timeout(mock_foundry_local_manager: MagicMock) -> None: + client = FoundryLocalClient(model="test-model-id", timeout=60.0) + assert client.client.timeout == 60.0 + + +def test_foundry_local_client_default_timeout(mock_foundry_local_manager: MagicMock) -> None: + client = FoundryLocalClient(model="test-model-id") + assert client.client.timeout is not None + + +def test_foundry_local_client_init_model_not_found(mock_foundry_local_manager: MagicMock) -> None: + mock_foundry_local_manager.catalog.get_model.return_value = None + with pytest.raises(ValueError, match="not found in Foundry Local"): + FoundryLocalClient(model="unknown-model") + + +def test_foundry_local_client_uses_resolved_variant_id(mock_foundry_local_manager: MagicMock) -> None: + mock_model = mock_foundry_local_manager.catalog.get_model_variant.return_value + mock_model.id = "resolved-model-id:1" + + client = FoundryLocalClient(model="model-alias") + + assert client.model == "resolved-model-id:1" + mock_foundry_local_manager.catalog.get_model_variant.assert_called_once_with("resolved-model-id:1") + mock_model.download.assert_called_once_with() + mock_model.load.assert_called_once_with() + + +def test_foundry_local_client_exact_id_does_not_resolve_alias(mock_foundry_local_manager: MagicMock) -> None: + mock_model = mock_foundry_local_manager.catalog.get_model_variant.return_value + mock_model.id = "exact-variant:1" + + client = FoundryLocalClient(model="exact-variant:1") + + assert client.model == "exact-variant:1" + mock_foundry_local_manager.catalog.get_model_variant.assert_called_once_with("exact-variant:1") + mock_foundry_local_manager.catalog.get_model.assert_not_called() + + +def test_foundry_local_client_passes_download_controls(mock_foundry_local_manager: MagicMock) -> None: + progress = MagicMock() + cancel_event = Event() + FoundryLocalClient(model="test-model-id", download_progress_callback=progress, download_cancel_event=cancel_event) + mock_model = mock_foundry_local_manager.catalog.get_model_variant.return_value + mock_model.download.assert_called_once_with(progress_callback=progress, cancel_event=cancel_event) + mock_model.load.assert_called_once_with() + + +def test_foundry_local_client_cancelled_download_does_not_load(mock_foundry_local_manager: MagicMock) -> None: + cancel_event = Event() + cancel_event.set() + mock_model = mock_foundry_local_manager.catalog.get_model_variant.return_value + mock_model.download.side_effect = RuntimeError("download cancelled") + + with pytest.raises(RuntimeError, match="download cancelled"): + FoundryLocalClient(model="test-model-id", download_cancel_event=cancel_event) + + mock_model.download.assert_called_once_with(progress_callback=None, cancel_event=cancel_event) + mock_model.load.assert_not_called() + mock_foundry_local_manager.close.assert_not_called() + + +def test_foundry_local_client_skips_download_controls_without_preparation( + mock_foundry_local_manager: MagicMock, +) -> None: + FoundryLocalClient( + model="test-model-id", + prepare_model=False, + download_progress_callback=MagicMock(), + download_cancel_event=Event(), + ) + mock_model = mock_foundry_local_manager.catalog.get_model_variant.return_value + mock_model.download.assert_not_called() + mock_model.load.assert_not_called() def test_foundry_local_client_init_from_env( foundry_local_unit_test_env: dict[str, str], mock_foundry_local_manager: MagicMock ) -> None: - """Test FoundryLocalClient initialization using environment variables.""" - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, - ): - client = FoundryLocalClient() + client = FoundryLocalClient() + assert client.model == foundry_local_unit_test_env["FOUNDRY_LOCAL_MODEL"] - assert client.model == foundry_local_unit_test_env["FOUNDRY_LOCAL_MODEL"] +@pytest.mark.parametrize("device", [DeviceType.GPU, "GPU"]) +def test_foundry_local_client_selects_device_variant( + mock_foundry_local_manager: MagicMock, device: DeviceType | Literal["GPU"] +) -> None: + mock_alias = mock_foundry_local_manager.catalog.get_model.return_value + mock_gpu = MagicMock(spec=IModel) + mock_gpu.id = "gpu-variant:1" + mock_gpu.info = SimpleNamespace(runtime=SimpleNamespace(device_type=DeviceType.GPU)) + mock_alias.variants = [mock_foundry_local_manager.catalog.get_model_variant.return_value, mock_gpu] -def test_foundry_local_client_init_with_device(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient initialization with device parameter.""" - from foundry_local.models import DeviceType + client = FoundryLocalClient(model="model-alias", device=device) + + assert client.model == "gpu-variant:1" + mock_gpu.download.assert_called_once_with() + mock_gpu.load.assert_called_once_with() + mock_alias.select_variant.assert_not_called() + + +def test_foundry_local_client_exact_id_rejects_wrong_device(mock_foundry_local_manager: MagicMock) -> None: + with pytest.raises(ValueError, match="exact-variant:1:GPU.*not found"): + FoundryLocalClient(model="exact-variant:1", device=DeviceType("GPU")) + mock_foundry_local_manager.catalog.get_model.return_value.download.assert_not_called() + + +def test_foundry_local_client_missing_device_variant(mock_foundry_local_manager: MagicMock) -> None: + with pytest.raises(ValueError, match="model-alias:GPU.*not found"): + FoundryLocalClient(model="model-alias", device=DeviceType("GPU")) + + +def test_foundry_local_client_unknown_runtime_does_not_match_device(mock_foundry_local_manager: MagicMock) -> None: + mock_model = mock_foundry_local_manager.catalog.get_model_variant.return_value + mock_model.info.runtime = None + with pytest.raises(ValueError, match="model-alias:CPU.*not found"): + FoundryLocalClient(model="model-alias", device=DeviceType("CPU")) + + +def test_foundry_local_client_invalid_device(mock_foundry_local_manager: MagicMock) -> None: + with pytest.raises(ValueError, match="DeviceType"): + FoundryLocalClient(model="model-alias", device=cast(DeviceType, "invalid")) + mock_foundry_local_manager.catalog.get_model.assert_not_called() + + +def test_foundry_local_client_init_with_prepare_model_false(mock_foundry_local_manager: MagicMock) -> None: + FoundryLocalClient(model="test-model-id", prepare_model=False) + mock_model = mock_foundry_local_manager.catalog.get_model_variant.return_value + mock_model.download.assert_not_called() + mock_model.load.assert_not_called() + + +def test_foundry_local_client_does_not_hide_sdk_failure(mock_foundry_local_manager: MagicMock) -> None: + mock_foundry_local_manager.catalog.get_model.side_effect = RuntimeError("catalog unavailable") + with pytest.raises(RuntimeError, match="catalog unavailable"): + FoundryLocalClient(model="model-alias") + mock_foundry_local_manager.close.assert_not_called() + + +def test_foundry_local_client_closes_new_manager_on_failure(mock_foundry_local_manager: MagicMock) -> None: + mock_foundry_local_manager.catalog.get_model.return_value = None + with patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class: + manager_class.instance = None + manager_class.initialize.side_effect = lambda _: setattr(manager_class, "instance", mock_foundry_local_manager) + + with pytest.raises(ValueError, match="not found in Foundry Local"): + FoundryLocalClient(model="unknown-model") + + mock_foundry_local_manager.close.assert_called_once_with() + + +def test_foundry_local_client_keeps_new_manager_on_success(mock_foundry_local_manager: MagicMock) -> None: + with patch("agent_framework_foundry_local._foundry_local_client.FoundryLocalManager") as manager_class: + manager_class.instance = None + manager_class.initialize.side_effect = lambda _: setattr(manager_class, "instance", mock_foundry_local_manager) + + FoundryLocalClient(model="test-model-id") + + mock_foundry_local_manager.close.assert_not_called() - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, - ): - FoundryLocalClient(model="test-model-id", device=DeviceType.CPU) - mock_foundry_local_manager.get_model_info.assert_called_once_with( - "test-model-id", - device=DeviceType.CPU, +def test_foundry_local_client_does_not_hide_web_service_failure(mock_foundry_local_manager: MagicMock) -> None: + mock_foundry_local_manager.urls = None + mock_foundry_local_manager.start_web_service.side_effect = RuntimeError("service unavailable") + with pytest.raises(RuntimeError, match="service unavailable"): + FoundryLocalClient(model="test-model-id") + mock_foundry_local_manager.close.assert_not_called() + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_client_uses_openai_transport(mock_foundry_local_manager: MagicMock, stream: bool) -> None: + requests: list[Any] = [] + + def respond(request: Any) -> Any: + requests.append(request) + completion = { + "id": "completion-1", + "created": 0, + "model": "test-model-id", + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}], + } + if not stream: + return _OPENAI_HTTPX.Response(200, json=completion) + completion["object"] = "chat.completion.chunk" + completion["choices"] = [ + {"index": 0, "delta": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"} + ] + return _OPENAI_HTTPX.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(completion)}\n\ndata: [DONE]\n\n", ) - mock_foundry_local_manager.download_model.assert_called_once_with( - "test-model-id", - device=DeviceType.CPU, + + async with DefaultAsyncHttpxClient(transport=_OPENAI_HTTPX.MockTransport(respond)) as http_client: + async_client = AsyncOpenAI( + base_url="http://localhost:5272/v1", api_key="foundry-local", http_client=http_client ) - mock_foundry_local_manager.load_model.assert_called_once_with( - "test-model-id", - device=DeviceType.CPU, + with patch("agent_framework_foundry_local._foundry_local_client.AsyncOpenAI", return_value=async_client): + client = FoundryLocalClient(model="test-model-id", prepare_model=False) + messages = [Message(role="user", contents=["Hi"])] + if stream: + result = "".join([update.text async for update in client.get_response(messages, stream=True)]) + else: + result = (await client.get_response(messages)).text + + assert result == "Hello" + assert len(requests) == 1 + assert requests[0].url.path == "/v1/chat/completions" + payload = json.loads(requests[0].content) + assert payload["model"] == "test-model-id" + assert payload["stream"] is stream + + +@asynccontextmanager +async def _client_with_transport( + respond: Callable[[Any], Any], + *, + middleware: Sequence[ChatAndFunctionMiddlewareTypes] | None = None, + function_invocation_configuration: FunctionInvocationConfiguration | None = None, + catalog_type: Literal["public", "local"] = "public", + model: str = "test-model-id", + prepare_model: bool = False, + device: DeviceType | Literal["CPU", "GPU", "NPU"] | None = None, + download_progress_callback: Callable[[float], None] | None = None, + download_cancel_event: Event | None = None, +) -> AsyncGenerator[FoundryLocalClient[FoundryLocalChatOptions[None]]]: + async with ( + DefaultAsyncHttpxClient(transport=_OPENAI_HTTPX.MockTransport(respond), trust_env=False) as http_client, + AsyncOpenAI( + base_url=f"http://localhost:5272{'/catalogs/local' if catalog_type == 'local' else ''}/v1", + api_key="foundry-local", + http_client=http_client, + max_retries=0, + ) as async_client, + ): + with patch("agent_framework_foundry_local._foundry_local_client.AsyncOpenAI", return_value=async_client): + yield FoundryLocalClient( + model=model, + catalog_type=catalog_type, + prepare_model=prepare_model, + device=device, + download_progress_callback=download_progress_callback, + download_cancel_event=download_cancel_event, + middleware=middleware, + function_invocation_configuration=function_invocation_configuration, + ) + + +def _completion_response(message: dict[str, Any], *, stream: bool, finish_reason: str = "stop") -> Any: + usage = {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8} + message = {"role": "assistant", **message} + if not stream: + return _OPENAI_HTTPX.Response( + 200, + json={ + "id": "completion-1", + "created": 0, + "model": "test-model-id", + "object": "chat.completion", + "choices": [{"index": 0, "message": message, "finish_reason": finish_reason}], + "usage": usage, + }, ) + if "tool_calls" in message: + message["tool_calls"] = [{"index": index, **call} for index, call in enumerate(message["tool_calls"])] + chunk = { + "id": "completion-1", + "created": 0, + "model": "test-model-id", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": message, "finish_reason": finish_reason}], + "usage": usage, + } + return _OPENAI_HTTPX.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(chunk)}\n\ndata: [DONE]\n\n", + ) -def test_foundry_local_client_init_model_not_found_with_device(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient error message includes device when model not found with device specified.""" - from foundry_local.models import DeviceType +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_eager_preparation_still_prepares_during_construction( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + model = mock_foundry_local_manager.catalog.get_model_variant.return_value - mock_foundry_local_manager.get_model_info.return_value = None + def respond(request: Any) -> Any: + model.download.assert_called_once_with() + model.load.assert_called_once_with() + return _completion_response({"content": "Hello"}, stream=stream) - with ( - patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, - ), - pytest.raises(ValueError, match="unknown-model:GPU.*not found"), - ): - FoundryLocalClient(model="unknown-model", device=DeviceType.GPU) + async with _client_with_transport(respond, prepare_model=True) as client: + model.download.assert_called_once_with() + model.load.assert_called_once_with() + messages = [Message(role="user", contents=["Hi"])] + if stream: + response = await client.get_response(messages, stream=True).get_final_response() + else: + response = await client.get_response(messages) + assert response.text == "Hello" -def test_foundry_local_client_init_with_prepare_model_false(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient initialization with prepare_model=False skips download and load.""" - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, - ): - FoundryLocalClient(model="test-model-id", prepare_model=False) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("catalog_type", ["public", "local"]) +async def test_foundry_local_lazy_preparation_pins_variant_and_preserves_download_controls( + mock_foundry_local_manager: MagicMock, stream: bool, catalog_type: Literal["public", "local"] +) -> None: + catalog = ( + mock_foundry_local_manager.catalog + if catalog_type == "public" + else mock_foundry_local_manager.get_catalog.return_value + ) + variant = MagicMock(spec=IModel) + variant.id = "selected-gpu:2" + variant.info = SimpleNamespace(runtime=SimpleNamespace(device_type=DeviceType.GPU)) + variant.is_loaded = False + catalog.get_model.return_value.variants.append(variant) + event_loop_thread = get_ident() + progress_threads: list[int] = [] + progress: list[float] = [] + cancel_event = Event() + payloads: list[dict[str, Any]] = [] + + def report_progress(value: float) -> None: + progress_threads.append(get_ident()) + progress.append(value) + + def download(*, progress_callback: Callable[[float], None], cancel_event: Event) -> None: + assert not cancel_event.is_set() + assert get_ident() != event_loop_thread + progress_callback(100.0) + + def load() -> None: + assert get_ident() != event_loop_thread + variant.is_loaded = True + + variant.download.side_effect = download + variant.load.side_effect = load + + def respond(request: Any) -> Any: + assert variant.is_loaded + payloads.append(json.loads(request.content)) + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport( + respond, + model="model-alias", + catalog_type=catalog_type, + device="GPU", + download_progress_callback=report_progress, + download_cancel_event=cancel_event, + ) as client: + variant.download.assert_not_called() + variant.load.assert_not_called() + for _ in range(2): + messages = [Message(role="user", contents=["Hi"])] + if stream: + response_stream = client.get_response(messages, stream=True) + if not payloads: + variant.load.assert_not_called() + response = await response_stream.get_final_response() + else: + response = await client.get_response(messages) + assert response.text == "Hello" + assert [payload["model"] for payload in payloads] == ["selected-gpu:2", "selected-gpu:2"] + variant.download.assert_called_once_with(progress_callback=report_progress, cancel_event=cancel_event) + variant.load.assert_called_once_with() + catalog.get_model.return_value.select_variant.assert_not_called() + assert progress == [100.0] + assert progress_threads[0] != event_loop_thread + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_lazy_preparation_reuses_loaded_model( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + model = mock_foundry_local_manager.catalog.get_model_variant.return_value + + def respond(request: Any) -> Any: + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond) as client: + # Manual preparation after client construction must also be respected. + model.is_loaded = True + messages = [Message(role="user", contents=["Hi"])] + if stream: + response = await client.get_response(messages, stream=True).get_final_response() + else: + response = await client.get_response(messages) + assert response.text == "Hello" + model.download.assert_not_called() + model.load.assert_not_called() + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("stage", ["download", "load"]) +async def test_foundry_local_lazy_preparation_failure_does_not_send_inference_and_can_retry( + mock_foundry_local_manager: MagicMock, stream: bool, stage: str +) -> None: + model = mock_foundry_local_manager.catalog.get_model_variant.return_value + failed_operation = model.download if stage == "download" else model.load + failed_operation.side_effect = RuntimeError(f"{stage} failed") + requests: list[Any] = [] + + def respond(request: Any) -> Any: + requests.append(request) + assert model.is_loaded + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond) as client: + messages = [Message(role="user", contents=["Hi"])] + with pytest.raises(RuntimeError, match=f"{stage} failed"): + if stream: + await client.get_response(messages, stream=True).get_final_response() + else: + await client.get_response(messages) + assert not requests + if stage == "download": + model.load.assert_not_called() + failed_operation.side_effect = None if stage == "download" else lambda: setattr(model, "is_loaded", True) + if stream: + response = await client.get_response(messages, stream=True).get_final_response() + else: + response = await client.get_response(messages) + assert response.text == "Hello" + assert len(requests) == 1 + mock_foundry_local_manager.close.assert_not_called() + mock_foundry_local_manager.stop_web_service.assert_not_called() + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_lazy_preparation_serializes_clients_without_blocking_event_loop( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + model = mock_foundry_local_manager.catalog.get_model_variant.return_value + started = Event() + release = Event() + requests: list[Any] = [] + + def load() -> None: + started.set() + if not release.wait(timeout=5): + raise TimeoutError("Test did not release native loading.") + model.is_loaded = True + + model.load.side_effect = load + + def respond(request: Any) -> Any: + assert model.is_loaded + requests.append(request) + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond) as first, _client_with_transport(respond) as second: + + async def request_response(client: FoundryLocalClient[FoundryLocalChatOptions[None]]) -> str: + messages = [Message(role="user", contents=["Hi"])] + if stream: + return (await client.get_response(messages, stream=True).get_final_response()).text + return (await client.get_response(messages)).text + + first_request = asyncio.create_task(request_response(first)) + second_request = asyncio.create_task(request_response(second)) + try: + assert await asyncio.to_thread(started.wait, 5) + await asyncio.sleep(0) + assert not first_request.done() + assert not second_request.done() + assert not requests + finally: + release.set() + assert await asyncio.gather(first_request, second_request) == ["Hello", "Hello"] + model.download.assert_called_once_with() + model.load.assert_called_once_with() + assert len(requests) == 2 + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_cancelling_lazy_request_preserves_native_preparation( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + model = mock_foundry_local_manager.catalog.get_model_variant.return_value + started = Event() + release = Event() + requests: list[Any] = [] + + def load() -> None: + started.set() + if not release.wait(timeout=5): + raise TimeoutError("Test did not release native loading.") + model.is_loaded = True + + model.load.side_effect = load + + def respond(request: Any) -> Any: + assert model.is_loaded + requests.append(request) + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond) as client: + + async def request_response() -> str: + messages = [Message(role="user", contents=["Hi"])] + if stream: + return (await client.get_response(messages, stream=True).get_final_response()).text + return (await client.get_response(messages)).text + + first_request = asyncio.create_task(request_response()) + try: + assert await asyncio.to_thread(started.wait, 5) + first_request.cancel() + with pytest.raises(asyncio.CancelledError): + await first_request + assert not requests + assert not model.is_loaded + finally: + release.set() + assert await request_response() == "Hello" + assert len(requests) == 1 + model.download.assert_called_once_with() + model.load.assert_called_once_with() + mock_foundry_local_manager.close.assert_not_called() + mock_foundry_local_manager.stop_web_service.assert_not_called() + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_deferred_download_honors_cancellation_event( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + model = mock_foundry_local_manager.catalog.get_model_variant.return_value + cancel_event = Event() + cancel_event.set() + requests: list[Any] = [] + + def download(*, progress_callback: Callable[[float], None] | None, cancel_event: Event) -> None: + assert cancel_event.is_set() + raise RuntimeError("download cancelled") + + model.download.side_effect = download + + def respond(request: Any) -> Any: + requests.append(request) + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond, download_cancel_event=cancel_event) as client: + model.download.assert_not_called() + with pytest.raises(RuntimeError, match="download cancelled"): + messages = [Message(role="user", contents=["Hi"])] + if stream: + await client.get_response(messages, stream=True).get_final_response() + else: + await client.get_response(messages) + model.download.assert_called_once_with(progress_callback=None, cancel_event=cancel_event) + model.load.assert_not_called() + assert not requests + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_lazy_preparation_does_not_provision_request_overrides( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + model = mock_foundry_local_manager.catalog.get_model_variant.return_value + payloads: list[dict[str, Any]] = [] + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond) as client: + messages = [Message(role="user", contents=["Hi"])] + if stream: + response = await client.get_response( + messages, stream=True, options={"model": "caller-prepared:1"} + ).get_final_response() + else: + response = await client.get_response(messages, options={"model": "caller-prepared:1"}) + assert response.text == "Hello" + assert payloads[0]["model"] == "caller-prepared:1" + model.download.assert_not_called() + model.load.assert_not_called() - mock_foundry_local_manager.download_model.assert_not_called() - mock_foundry_local_manager.load_model.assert_not_called() +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_generation_options_and_usage(mock_foundry_local_manager: MagicMock, stream: bool) -> None: + requests: list[Any] = [] -def test_foundry_local_client_init_calls_download_and_load(mock_foundry_local_manager: MagicMock) -> None: - """Test FoundryLocalClient initialization calls download_model and load_model by default.""" - with patch( - "agent_framework_foundry_local._foundry_local_client.FoundryLocalManager", - return_value=mock_foundry_local_manager, + def respond(request: Any) -> Any: + requests.append(request) + return _completion_response({"content": "Hello"}, stream=stream) + + options: FoundryLocalChatOptions[None] = { + "temperature": 0.0, + "top_p": 0.8, + "max_tokens": 32, + "seed": 0, + "top_k": 0, + "do_sample": False, + "early_stopping": False, + "frequency_penalty": 0.0, + "presence_penalty": 0.0, + "stop": ["END"], + "logit_bias": {"42": -1.0}, + "instructions": "Be concise.", + "metadata": {"top_k": "16", "native_custom": "keep"}, + "extra_body": {"metadata": {"top_k": "8"}, "custom_setting": "test"}, + } + original_options = json.loads(json.dumps(options)) + async with _client_with_transport(respond) as client: + messages = [Message(role="user", contents=["Hi"])] + if stream: + response_stream = client.get_response( + messages, stream=True, options=options, client_kwargs={"extra_headers": {"X-Test-Header": "present"}} + ) + response = await response_stream.get_final_response() + else: + response = await client.get_response( + messages, options=options, client_kwargs={"extra_headers": {"X-Test-Header": "present"}} + ) + + assert response.text == "Hello" + assert response.usage_details is not None + assert response.usage_details["input_token_count"] == 5 + assert response.usage_details["output_token_count"] == 3 + assert requests[0].headers["X-Test-Header"] == "present" + payload = json.loads(requests[0].content) + for key in ( + "temperature", + "top_p", + "max_tokens", + "seed", + "frequency_penalty", + "presence_penalty", + "stop", + "logit_bias", ): - FoundryLocalClient(model="test-model-id") + assert payload[key] == original_options[key] + assert payload["messages"][0] == {"role": "system", "content": "Be concise."} + assert payload["metadata"] == { + "top_k": "0", + "do_sample": "false", + "early_stopping": "false", + "native_custom": "keep", + } + assert all(key not in payload for key in ("top_k", "do_sample", "early_stopping", "max_completion_tokens")) + assert payload["custom_setting"] == "test" + assert options == original_options + if stream: + assert payload["stream_options"] == {"include_usage": True} + mock_foundry_local_manager.close.assert_not_called() - mock_foundry_local_manager.download_model.assert_called_once_with( - "test-model-id", - device=None, - ) - mock_foundry_local_manager.load_model.assert_called_once_with( - "test-model-id", - device=None, + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_decoder_metadata_passthrough(mock_foundry_local_manager: MagicMock, stream: bool) -> None: + payloads: list[dict[str, Any]] = [] + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + return _completion_response({"content": "Hello"}, stream=stream) + + options: FoundryLocalChatOptions[None] = { + "metadata": {"do_sample": "false"}, + "extra_body": {"metadata": {"top_k": "8"}}, + } + original_options = deepcopy(options) + async with _client_with_transport(respond) as client: + messages = [Message(role="user", contents=["Hi"])] + if stream: + await client.get_response(messages, stream=True, options=options).get_final_response() + else: + await client.get_response(messages, options=options) + assert payloads[0]["metadata"] == {"top_k": "8", "do_sample": "false"} + assert options == original_options + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_request_model_override(mock_foundry_local_manager: MagicMock, stream: bool) -> None: + payloads: list[dict[str, Any]] = [] + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond) as client: + messages = [Message(role="user", contents=["Hi"])] + if stream: + await client.get_response(messages, stream=True, options={"model": "other-variant:1"}).get_final_response() + else: + await client.get_response(messages, options={"model": "other-variant:1"}) + assert client.model == "test-model-id" + assert payloads[0]["model"] == "other-variant:1" + + +@pytest.mark.parametrize("invalid_metadata", [0, "invalid", ["top_k", "8"]]) +async def test_foundry_local_invalid_decoder_metadata( + mock_foundry_local_manager: MagicMock, invalid_metadata: object +) -> None: + requests: list[Any] = [] + + def respond(request: Any) -> Any: + requests.append(request) + return _completion_response({"content": "Hello"}, stream=False) + + options: FoundryLocalChatOptions[None] = { + "top_k": 1, + "metadata": cast(dict[str, Any], invalid_metadata), + } + async with _client_with_transport(respond) as client: + with pytest.raises(ValueError, match="metadata must be a mapping"): + await client.get_response([Message(role="user", contents=["Hi"])], options=options) + assert not requests + + +@pytest.mark.parametrize( + "schema", + [ + {"type": "object", "properties": {"answer": {"type": "integer"}}}, + {"properties": {"answer": {"type": "integer"}}}, + {"$defs": {"Answer": {"type": "integer"}}, "$ref": "#/$defs/Answer"}, + ], +) +def test_foundry_local_raw_schema_from_older_openai_adapter( + mock_foundry_local_manager: MagicMock, schema: dict[str, Any] +) -> None: + client = FoundryLocalClient(model="test-model-id") + original_schema = deepcopy(schema) + # agent-framework-openai 1.6.0 passes raw schema dictionaries through unchanged. + with patch.object(RawOpenAIChatCompletionClient, "_prepare_options", return_value={"response_format": schema}): + prepared = client._prepare_options([], {"response_format": schema}) + assert prepared["response_format"] == {"type": "json_schema", "json_schema": schema} + assert schema == original_schema + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + "format_kind", + ["pydantic", "schema", "schema_no_type", "schema_ref", "native_schema", "named_schema", "json_object"], +) +async def test_foundry_local_structured_responses( + mock_foundry_local_manager: MagicMock, stream: bool, format_kind: str +) -> None: + class Answer(BaseModel): + answer: int + + payloads: list[dict[str, Any]] = [] + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + return _completion_response({"content": '{"answer": 4}'}, stream=stream) + + schema: dict[str, Any] = { + "type": "object", + "properties": {"answer": {"type": "integer"}}, + "required": ["answer"], + } + if format_kind == "schema_no_type": + schema.pop("type") + elif format_kind == "schema_ref": + schema = {"$defs": {"Answer": schema}, "$ref": "#/$defs/Answer"} + options: FoundryLocalChatOptions[Answer] = { + "response_format": Answer + if format_kind == "pydantic" + else schema + if format_kind in ("schema", "schema_no_type", "schema_ref") + else {"type": "json_schema", "json_schema": schema} + if format_kind == "native_schema" + else {"type": "json_schema", "json_schema": {"name": "Answer", "schema": schema, "strict": True}} + if format_kind == "named_schema" + else {"type": "json_object"} + } + original_options = deepcopy(options) + async with _client_with_transport(respond) as client: + messages = [Message(role="user", contents=["What is 2+2?"])] + if stream: + response = await client.get_response(messages, stream=True, options=options).get_final_response() + else: + response = await client.get_response(messages, options=options) + if format_kind == "pydantic": + assert isinstance(response.value, Answer) + assert response.value.answer == 4 + else: + assert response.value == {"answer": 4} + assert payloads[0]["response_format"]["type"] == ("json_object" if format_kind == "json_object" else "json_schema") + if format_kind != "json_object": + native_schema = payloads[0]["response_format"]["json_schema"] + assert isinstance(native_schema, dict) + if format_kind == "schema_ref": + assert native_schema["$ref"] == "#/$defs/Answer" + native_schema = native_schema["$defs"]["Answer"] + if format_kind != "schema_no_type": + assert native_schema["type"] == "object" + assert native_schema["properties"]["answer"]["type"] == "integer" + assert options == original_options + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("enabled", [False, True]) +async def test_foundry_local_agent_tool_loop( + mock_foundry_local_manager: MagicMock, stream: bool, enabled: bool +) -> None: + payloads: list[dict[str, Any]] = [] + invocations: list[tuple[int, int]] = [] + + @tool + def add(a: int, b: int) -> int: + """Add two integers.""" + invocations.append((a, b)) + return a + b + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + if len(payloads) == 1: + return _completion_response( + { + "content": None, + "tool_calls": [ + { + "id": "call-add", + "type": "function", + "function": {"name": "add", "arguments": '{"a":2,"b":3}'}, + } + ], + }, + stream=stream, + finish_reason="tool_calls", + ) + return _completion_response({"content": "5"}, stream=stream) + + async with _client_with_transport(respond, function_invocation_configuration={"enabled": enabled}) as client: + agent = Agent(client=client, tools=add, instructions="Use the calculator.") + if stream: + response = await agent.run( + "What is 2+3?", stream=True, options={"tool_choice": "required", "allow_multiple_tool_calls": False} + ).get_final_response() + else: + response = await agent.run( + "What is 2+3?", options={"tool_choice": "required", "allow_multiple_tool_calls": False} + ) + + assert payloads[0]["tools"][0]["function"]["name"] == "add" + assert payloads[0]["tool_choice"] == "required" + assert payloads[0]["parallel_tool_calls"] is False + if enabled: + assert invocations == [(2, 3)] + assert len(payloads) == 2 + assert response.text == "5" + tool_messages = [message for message in payloads[1]["messages"] if message["role"] == "tool"] + assert len(tool_messages) == 1 + assert tool_messages[0]["tool_call_id"] == "call-add" + assert tool_messages[0]["content"] == "5" + else: + assert invocations == [] + assert len(payloads) == 1 + assert any(content.type == "function_call" for message in response.messages for content in message.contents) + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("named_tool", [False, True]) +@pytest.mark.parametrize("json_object", [False, True]) +async def test_foundry_local_required_tools_with_structured_output( + mock_foundry_local_manager: MagicMock, stream: bool, named_tool: bool, json_object: bool +) -> None: + class Answer(BaseModel): + answer: int + + payloads: list[dict[str, Any]] = [] + invocations: list[tuple[int, int]] = [] + + @tool + def add(a: int, b: int) -> int: + """Add two integers.""" + invocations.append((a, b)) + return a + b + + def respond(request: Any) -> Any: + payload = json.loads(request.content) + payloads.append(payload) + if len(payloads) == 1: + assert "response_format" not in payload + assert payload["tool_choice"] == ( + {"type": "function", "function": {"name": "add"}} if named_tool else "required" + ) + return _completion_response( + { + "content": None, + "tool_calls": [ + { + "id": "call-add", + "type": "function", + "function": {"name": "add", "arguments": '{"a":2,"b":3}'}, + } + ], + }, + stream=stream, + finish_reason="tool_calls", + ) + assert payload["response_format"]["type"] == ("json_object" if json_object else "json_schema") + if not json_object: + assert payload["response_format"]["json_schema"]["properties"]["answer"]["type"] == "integer" + return _completion_response({"content": '{"answer":5}'}, stream=stream) + + options: ChatOptions[Answer] = { + "tool_choice": {"mode": "required", "required_function_name": "add"} if named_tool else "required", + "response_format": {"type": "json_object"} if json_object else Answer, + } + original_options = deepcopy(options) + async with _client_with_transport(respond) as client: + agent = Agent(client=client, tools=add) + if stream: + response = await agent.run("What is 2+3?", stream=True, options=options).get_final_response() + else: + response = await agent.run("What is 2+3?", options=options) + assert invocations == [(2, 3)] + assert len(payloads) == 2 + if json_object: + assert response.value == {"answer": 5} + else: + assert isinstance(response.value, Answer) + assert response.value.answer == 5 + assert options == original_options + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_constructor_and_per_call_middleware( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + order: list[str] = [] + payloads: list[dict[str, Any]] = [] + + @chat_middleware + async def constructor_middleware(context: ChatContext, call_next: Callable[[], Awaitable[None]]) -> None: + order.append("constructor-before") + context.options = {**(context.options or {}), "temperature": 0.1} + await call_next() + order.append("constructor-after") + + @chat_middleware + async def per_call_middleware(context: ChatContext, call_next: Callable[[], Awaitable[None]]) -> None: + order.append("per-call-before") + context.options = {**(context.options or {}), "top_p": 0.5} + await call_next() + order.append("per-call-after") + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + return _completion_response({"content": "Hello"}, stream=stream) + + async with _client_with_transport(respond, middleware=[constructor_middleware]) as client: + messages = [Message(role="user", contents=["Hi"])] + if stream: + response = await client.get_response( + messages, stream=True, middleware=[per_call_middleware] + ).get_final_response() + else: + response = await client.get_response(messages, middleware=[per_call_middleware]) + assert response.text == "Hello" + assert order == ["constructor-before", "per-call-before", "per-call-after", "constructor-after"] + assert payloads[0]["temperature"] == 0.1 + assert payloads[0]["top_p"] == 0.5 + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_agent_session_preserves_history( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + payloads: list[dict[str, Any]] = [] + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + return _completion_response({"content": "Remembered" if len(payloads) == 1 else "LANTERN"}, stream=stream) + + async with _client_with_transport(respond) as client: + agent = Agent(client=client, instructions="Remember the conversation.") + session = agent.create_session() + if stream: + await agent.run("Remember LANTERN", session=session, stream=True).get_final_response() + response = await agent.run("What did I say?", session=session, stream=True).get_final_response() + else: + await agent.run("Remember LANTERN", session=session) + response = await agent.run("What did I say?", session=session) + assert response.text == "LANTERN" + assert [message["role"] for message in payloads[1]["messages"]] == ["system", "user", "assistant", "user"] + assert payloads[1]["messages"][1]["content"] == "Remember LANTERN" + assert payloads[1]["messages"][2]["content"] == "Remembered" + + +@pytest.mark.parametrize("stream", [False, True]) +async def test_foundry_local_multimodal_request_serialization( + mock_foundry_local_manager: MagicMock, stream: bool +) -> None: + payloads: list[dict[str, Any]] = [] + + def respond(request: Any) -> Any: + payloads.append(json.loads(request.content)) + return _completion_response({"content": "A red square"}, stream=stream) + + async with _client_with_transport(respond) as client: + messages = [ + Message( + role="user", + contents=[ + "Describe this image", + Content.from_uri("https://example.test/image.png", media_type="image/png"), + ], + ) + ] + if stream: + response = await client.get_response(messages, stream=True).get_final_response() + else: + response = await client.get_response(messages) + assert response.text == "A red square" + assert payloads[0]["messages"] == [ + {"role": "user", "content": "Describe this image"}, + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.test/image.png"}}], + }, + ] + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("status", [400, 404, 429, 503]) +async def test_foundry_local_http_errors_remain_visible( + mock_foundry_local_manager: MagicMock, stream: bool, status: int +) -> None: + def respond(request: Any) -> Any: + return _OPENAI_HTTPX.Response( + status, json={"error": {"message": "native inference failed", "type": "server_error"}} ) + + async with _client_with_transport(respond) as client: + with pytest.raises(ChatClientException, match="native inference failed"): + messages = [Message(role="user", contents=["Hi"])] + if stream: + await client.get_response(messages, stream=True).get_final_response() + else: + await client.get_response(messages) + mock_foundry_local_manager.close.assert_not_called() + + +async def test_foundry_local_request_cancellation_preserves_manager(mock_foundry_local_manager: MagicMock) -> None: + started = asyncio.Event() + release = asyncio.Event() + requests = 0 + + async def respond(request: Any) -> Any: + nonlocal requests + requests += 1 + if requests == 1: + started.set() + await release.wait() + return _completion_response({"content": "Hello"}, stream=False) + + async with _client_with_transport(respond) as client: + messages = [Message(role="user", contents=["Hi"])] + + async def request_response() -> None: + await client.get_response(messages) + + task = asyncio.create_task(request_response()) + try: + await asyncio.wait_for(started.wait(), timeout=5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + response = await client.get_response(messages) + assert response.text == "Hello" + finally: + release.set() + task.cancel() + mock_foundry_local_manager.close.assert_not_called() diff --git a/python/samples/02-agents/providers/foundry/foundry_local_agent.py b/python/samples/02-agents/providers/foundry/foundry_local_agent.py index 95d4e06d21e..6583720e723 100644 --- a/python/samples/02-agents/providers/foundry/foundry_local_agent.py +++ b/python/samples/02-agents/providers/foundry/foundry_local_agent.py @@ -58,21 +58,25 @@ async def main() -> None: print("=== Basic Foundry Local Client Agent Example ===") client = FoundryLocalClient(model="phi-4-mini") - print(f"Client Model ID: {client.model}\n") - print("Other available models (tool calling supported only):") - for model in client.manager.list_catalog_models(): - if model.supports_tool_calling: - print( - f"- {model.alias} for {model.task} - id={model.id} - {(model.file_size_mb / 1000):.2f} GB - {model.license}" - ) - agent = Agent( - client=client, - name="LocalAgent", - instructions="You are a helpful agent.", - tools=get_weather, - ) - await non_streaming_example(agent) - await streaming_example(agent) + try: + print(f"Client Model ID: {client.model}\n") + print("Other available models (tool calling supported only):") + for model in client.manager.catalog.list_models(): + if model.supports_tool_calling: + info = model.info + size = f"{info.file_size_mb / 1000:.2f} GB" if info.file_size_mb is not None else "size unknown" + print(f"- {model.alias} for {info.task} - id={model.id} - {size} - {info.license}") + agent = Agent( + client=client, + name="LocalAgent", + instructions="You are a helpful agent.", + tools=get_weather, + ) + await non_streaming_example(agent) + await streaming_example(agent) + finally: + await client.client.close() + client.manager.close() if __name__ == "__main__": diff --git a/python/uv.lock b/python/uv.lock index 84130377c9f..5e79ff6290f 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -515,7 +515,7 @@ requires-dist = [ { name = "agent-framework-durabletask", marker = "extra == 'all'", specifier = ">=1.0.0b260922,<2" }, { name = "agent-framework-foundry", marker = "extra == 'all'", editable = "packages/foundry" }, { name = "agent-framework-foundry-hosting", marker = "extra == 'all'", editable = "packages/foundry_hosting" }, - { name = "agent-framework-foundry-local", marker = "extra == 'all'", editable = "packages/foundry_local" }, + { name = "agent-framework-foundry-local", marker = "python_full_version >= '3.11' and extra == 'all'", editable = "packages/foundry_local" }, { name = "agent-framework-gemini", marker = "extra == 'all'", editable = "packages/gemini" }, { name = "agent-framework-github-copilot", marker = "python_full_version >= '3.11' and extra == 'all'", editable = "packages/github_copilot" }, { name = "agent-framework-hyperlight", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'all') or (platform_machine == 'AMD64' and sys_platform == 'win32' and extra == 'all')", editable = "packages/hyperlight" }, @@ -710,7 +710,7 @@ dependencies = [ requires-dist = [ { name = "agent-framework-core", editable = "packages/core" }, { name = "agent-framework-openai", editable = "packages/openai" }, - { name = "foundry-local-sdk", specifier = ">=0.5.1,<0.5.2" }, + { name = "foundry-local-sdk", specifier = ">=2.2.1,<3" }, ] [[package]] @@ -2197,17 +2197,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/79/1b8fa1bb3568781e84c9200f951c735f3f157429f44be0495da55894d620/filetype-1.2.0-py2.py3-none-any.whl", hash = "sha256:7ce71b6880181241cf7ac8697a2f1eb6a8bd9b429f7ad6d27b8db9ba5f1c2d25", size = 19970, upload-time = "2022-11-02T17:34:01.425Z" }, ] +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e8/2d/d2a548598be01649e2d46231d151a6c56d10b964d94043a335ae56ea2d92/flatbuffers-25.12.19-py2.py3-none-any.whl", hash = "sha256:7634f50c427838bb021c2d66a3d1168e9d199b0607e6329399f04846d42e20b4", size = 26661, upload-time = "2025-12-19T23:16:13.622Z" }, +] + [[package]] name = "foundry-local-sdk" -version = "0.5.1" +version = "2.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx" }, + { name = "cffi" }, + { name = "onnxruntime" }, + { name = "onnxruntime-genai-core" }, + { name = "openai" }, { name = "pydantic" }, - { name = "tqdm" }, + { name = "requests" }, + { name = "typing-extensions" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/ed/6b/76a7fe8f9f4c52cc84eaa1cd1b66acddf993496d55d6ea587bf0d0854d1c/foundry_local_sdk-0.5.1-py3-none-any.whl", hash = "sha256:f3639a3666bc3a94410004a91671338910ac2e1b8094b1587cc4db0f4a7df07e", size = 14003, upload-time = "2025-11-21T05:39:58.099Z" }, + { url = "https://files.pythonhosted.org/packages/17/ee/9718d428de802ee5264e314c2eda1392cb99ccbe9bca7386eec278103b7f/foundry_local_sdk-2.2.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:7180ea612ab5e2af0f6c8475d413ea592b7c067871b19c9e24a44c0a21f1c6e7", size = 6394441 }, + { url = "https://files.pythonhosted.org/packages/d4/d1/e1678487f99cfc52b573feba04c12133df7446a4d128abf1a489c0092f7f/foundry_local_sdk-2.2.1-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b534b9d3700ab90c69986a817ee542da215fab62e0f13e2e27f05c53b3239154", size = 8993548 }, + { url = "https://files.pythonhosted.org/packages/7c/73/670318151e07fdf131df926a270d2443b7255e3b5851e8bdb5dca98f88c9/foundry_local_sdk-2.2.1-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:6ee4bd5407f5134bb243456603f4398a8732d765c7b3cc4b4d6fa2910ad2bda5", size = 8693997 }, + { url = "https://files.pythonhosted.org/packages/86/7a/ce8b0b2f870590e57d841a95adab0e965d6b3643fce5b20ca82222443a9f/foundry_local_sdk-2.2.1-cp311-abi3-win_amd64.whl", hash = "sha256:2a88dd8ba0ca89ba04b2f03a182238e0632d894b313117975ba0f8a4d68767d3", size = 40815873 }, + { url = "https://files.pythonhosted.org/packages/bc/12/a7d2aabf69089b3711b9de18b7d2ff228af26d401dcf3b7af343c03a3aec/foundry_local_sdk-2.2.1-cp311-abi3-win_arm64.whl", hash = "sha256:f28790aaac7abc4d14b9e6cfae422d3ed2679a8c62f2c7084c0e59297b9a6689", size = 40611400 }, ] [[package]] @@ -3643,6 +3659,56 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/f6/2091e50b8b6c3e6901f6eab283d5efd66fb71c86ddb1b4d68766c3eeba0f/ollama-0.5.3-py3-none-any.whl", hash = "sha256:a8303b413d99a9043dbf77ebf11ced672396b59bec27e6d5db67c88f01b279d2", size = 13490, upload-time = "2025-08-07T21:44:09.353Z" }, ] +[[package]] +name = "onnxruntime" +version = "1.31.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flatbuffers" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "protobuf" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/e7/61b2768393646bd12e31eeb71958193f4e02c98c4980cf9289d19bbb4a8f/onnxruntime-1.31.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:cbf1a7f6470ddfe9dbc781966af8ce4a10e1858d75a93f93cc6b9367c9587870", size = 20871717 }, + { url = "https://files.pythonhosted.org/packages/44/86/e57025ab9c1eb83b6e686c92507fa6b7156d9d375e197a6c3a2afc05a1e2/onnxruntime-1.31.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:37c7dfe398550afdf9670a29315dbb88e49d8afc473ffaf1f410376efbb9c80a", size = 21413529 }, + { url = "https://files.pythonhosted.org/packages/a6/72/6c57163b63b5343853d7f0619c4f424a6e53ee762d7263667ff004bfede1/onnxruntime-1.31.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d4092b78fc5bab77ce6522393098cdb2535423045ecdcff15cc0d022162d6b66", size = 23753636 }, + { url = "https://files.pythonhosted.org/packages/37/de/6cab7e39917cc87728d2f00abe97c81fe86b29f9e1f758627864c28f0c21/onnxruntime-1.31.0-cp311-cp311-win_amd64.whl", hash = "sha256:317608967b03807ed4661113b08293fac02a1db6496a6863a07d9f19232936ad", size = 14885750 }, + { url = "https://files.pythonhosted.org/packages/1d/11/f335a124a1aadda99e5a2b618264606504bd9e3763b1b2486e6441cd65e5/onnxruntime-1.31.0-cp311-cp311-win_arm64.whl", hash = "sha256:e85c1632c0a8cf488bd8f1039f5320877b864c8f9ebd4122fb8bb909f83b7096", size = 14735138 }, + { url = "https://files.pythonhosted.org/packages/b3/bd/2ac094311163b803e3626c3937461d6900934bd56cca7601f6150ff860c3/onnxruntime-1.31.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:aaab9b3af536b06ca27ab5e35e3d429c97457ce76cf298af103f687e8b9975c0", size = 20882054 }, + { url = "https://files.pythonhosted.org/packages/53/1a/561b43ca1536d9e81d1785bb8a1a260a9e314ef6d04976ba0411c652bda1/onnxruntime-1.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:35758d7606d578ec5b9d65f6e8a1f488013194c3f6097038a3223cb26d35ef9a", size = 21420804 }, + { url = "https://files.pythonhosted.org/packages/6c/44/1e9e762b95b7da0a8424913a1ed7c38cdaf88624a3c41ddba24ebac88bc9/onnxruntime-1.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5e129d6c56abd53e659cb70f00a108d6824086470ff99c2e47a82e5786563db3", size = 23760984 }, + { url = "https://files.pythonhosted.org/packages/be/ed/b12cea136ccd7b03d924f46b8393faf7ceac21115c0c50e729faa248cf23/onnxruntime-1.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:09d56445c1753e66e0912de69d3f0184016ad9a191dcd6925bf5dd570d2bfbe5", size = 14888841 }, + { url = "https://files.pythonhosted.org/packages/02/ad/37bbc51dcb5cd105c5b2fe98f122b23e90171c2719516964edc65bb1d4cc/onnxruntime-1.31.0-cp312-cp312-win_arm64.whl", hash = "sha256:5c54a0eb7b2b4eef3eb9dcfaf82f5ce880db07288dc309574f6657e9da5cc754", size = 14740604 }, + { url = "https://files.pythonhosted.org/packages/e0/2b/117f94d73a3bac4276c285c47e384e1b3ea67b191aa4c7592df9d3f4a136/onnxruntime-1.31.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:0ba02a44acb6203040354d9a1f160e3f37a43feac7bb05caa3e0ea545efed505", size = 20881803 }, + { url = "https://files.pythonhosted.org/packages/8a/d0/3677fe93ec0fa3c637744aa4c3ae6ef89a93ee229cd3c5157820f267c7bd/onnxruntime-1.31.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:ad663106f6eeff3d454f24a786450459d07f30e74863851104fc1b8b3f368127", size = 21420629 }, + { url = "https://files.pythonhosted.org/packages/0d/ac/67ebbaab4b3083f2a6b27ee6c4aa400c7f8d6c72b5499aac7e4cd6ba74f5/onnxruntime-1.31.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:37fd78cee5160c7a43a1730ccb3682ffd880af9c9e80385d625c0c2f8b125809", size = 23760708 }, + { url = "https://files.pythonhosted.org/packages/c4/86/05ed2056f43b27aaf12ebc592ebd9037a26bed315958cf882f43425fd469/onnxruntime-1.31.0-cp313-cp313-win_amd64.whl", hash = "sha256:73e0165d58ece068c2a8a1c477c90b38e5a8adbbd399fdfdfd4bd79cbc28ff8d", size = 14888306 }, + { url = "https://files.pythonhosted.org/packages/c9/93/d33bae7b1a78780c4946ce03989c59a67d42d7015ad62d2098975fc5a580/onnxruntime-1.31.0-cp313-cp313-win_arm64.whl", hash = "sha256:e51d10d2e2e1e5bbf9b126a0cd9853d3e6c4e21424518dd50160b91471be33dc", size = 14740892 }, + { url = "https://files.pythonhosted.org/packages/12/05/cf44f7642269b285aada4b662c4662b14ac63f6e03e129d939c4a956a0f5/onnxruntime-1.31.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:e0e050bf9ec754950a6ba9830e4032f4004d972c6f38c5642fef26d44d894965", size = 21432644 }, + { url = "https://files.pythonhosted.org/packages/b5/8e/673315b2dd2eb99b2f4774d7a5986fe00d933ebed17ee72c441f579226e6/onnxruntime-1.31.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:e93d7c5fad20afa697ac16f376fd0306ed180f9a376e86106cc0b7d84f53ef87", size = 23773868 }, + { url = "https://files.pythonhosted.org/packages/9d/fb/b4c52e500c6f3d00dfc22fad4d7513524f3ea2100a24a077ee3b0daf552d/onnxruntime-1.31.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:278e0dc922ec69b05a28f59110d5421e2ec8b1d0dd46c6b10c063069a4051e72", size = 20883462 }, + { url = "https://files.pythonhosted.org/packages/37/fb/8be04665b700cb6e874d944e9932bb3c3969d3f53e820f5c42bfd26565d0/onnxruntime-1.31.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:984c0a2c1ad6a41fbc101dc3949abe4a72254892d01a5e70d9b792711e0bfa54", size = 21421618 }, + { url = "https://files.pythonhosted.org/packages/30/2e/5c6ec7e26a097e97ee70f2dee68b8ca4d9d26701f2f33c3f8ab585cb89fe/onnxruntime-1.31.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:e4efa4a1a0bb0b5173c6a3292c181d518b8323f9d56e978635d0c09d38c94d1a", size = 23762993 }, + { url = "https://files.pythonhosted.org/packages/6a/66/0bf4fdb9f58efa69cf4eddde24c72aebcc628d6ff1d67c9546145c6b9922/onnxruntime-1.31.0-cp314-cp314-win_amd64.whl", hash = "sha256:83e3dbcf6abc6189c4bdf7d329c07ba1133c88172134c266d84b4409aa3b9dbf", size = 15268709 }, + { url = "https://files.pythonhosted.org/packages/af/99/75a36172c1ed1d74ac0e91c11d642548081e2c9c63f15ee796564619556f/onnxruntime-1.31.0-cp314-cp314-win_arm64.whl", hash = "sha256:d2d5ac22f896c810be2b2b171392bb908f80b6c9a7e2d592ddb7435c928044e1", size = 15153795 }, + { url = "https://files.pythonhosted.org/packages/9c/ec/23b7749edc7aad53bf4632de190399fda69a9195499426637ef1b02f06c6/onnxruntime-1.31.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:d25cd65874b75fdf16149120a04d0cd4551f860a3c8e2ecec785a1903e41d8aa", size = 21432344 }, + { url = "https://files.pythonhosted.org/packages/f2/76/155ab0b265e9ceade28a8dd3858fdfa509b039f78010042c875940e32e58/onnxruntime-1.31.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:1ecc1450af28d2cf362990e188ccc81b51388f317f641ad973ab4301473200f2", size = 23772576 }, +] + +[[package]] +name = "onnxruntime-genai-core" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/14/360581437f231bcf2f331bacfed4a571e043a7269edab99a563be1d6b0c5/onnxruntime_genai_core-0.18.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:08cb9874228ea7199befe95777617c0568c424727ca3c7db36d42475341dd30d", size = 3551018 }, + { url = "https://files.pythonhosted.org/packages/58/1f/892f0044ec1a8043a29109dac843970cf5698945f33443f40c3f12f9d4fe/onnxruntime_genai_core-0.18.0-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:2cf80c3b08cb9bf9471f75a0c528dd62ae18c1874d747ca58b13cd49bf113076", size = 10356840 }, + { url = "https://files.pythonhosted.org/packages/9c/be/d9c7b0b111e96be0ae25f42171c8606167c4c053903a160fb51bbf824a70/onnxruntime_genai_core-0.18.0-py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:8a29c0a3e517cdc7cac8809ee49c2b288f549fed6cdd31c08b53df06fd50460c", size = 10668979 }, + { url = "https://files.pythonhosted.org/packages/8e/ad/e0317650f6ce5647fadec769f18c0e159a7ce2533817ae94cd4b6fc77c68/onnxruntime_genai_core-0.18.0-py3-none-win_amd64.whl", hash = "sha256:1b2e1ede54fedcbec497ceb046994a48dd41c15d310dd63a321bff6fe31b1bec", size = 3789063 }, + { url = "https://files.pythonhosted.org/packages/56/32/3cbcc7a2e5b582b72133af8fe182f08153782103add4abf4bc17e4992cad/onnxruntime_genai_core-0.18.0-py3-none-win_arm64.whl", hash = "sha256:d4c76762203a70f90c5a55229fd5f76123631a0de76c6bf5c9090ab58142d8a7", size = 3618969 }, +] + [[package]] name = "openai" version = "3.36.1" @@ -5574,18 +5640,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/60/3f/3e3f8fd0919249b0200c80fbc4f9a1e70be19f9883da71dfb7f8b9ab8aca/tomli-2.5.0-py3-none-any.whl", hash = "sha256:32a7b79ac57a2e83670ce329ccf675798bc5a2094783a63676866b70503f2e2b", size = 14765, upload-time = "2026-10-07T12:23:36.875Z" }, ] -[[package]] -name = "tqdm" -version = "4.70.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/0d/ea/b2a5bd54b28a324dae8211928b2d730b6547500342c7e6c6dea08bd0a485/tqdm-4.70.1.tar.gz", hash = "sha256:cefd0eca11b2a37a3aee776544d4f4ae913f02688135b5556b8788dfa474afc4", size = 171846, upload-time = "2026-09-11T07:25:16.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/03/921a3d3c75785aca9ebfbfcabfbc3a1be12e2ab5265deb026d55a5a3f83e/tqdm-4.70.1-py3-none-any.whl", hash = "sha256:c293e525e6fef9c20e8728fd4612df02a0aa31bb5fe91ecd93e123b1b7bffa73", size = 80199, upload-time = "2026-09-11T07:25:14.599Z" }, -] - [[package]] name = "truststore" version = "0.10.5"