.NET: Add GA computer tool support to Foundry and Foundry hosting

dotnet/agent-framework-dotnet.slnx

@@ -27,6 +27,7 @@
     <Project Path="samples/02-agents/AgentProviders/azure/Agent_With_AzureFoundryModel/Agent_With_AzureFoundryModel.csproj" />
     <Project Path="samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIChatCompletion/Agent_With_AzureOpenAIChatCompletion.csproj" />
     <Project Path="samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIResponses/Agent_With_AzureOpenAIResponses.csproj" />
+    <Project Path="samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIResponses_ComputerUse/Agent_With_AzureOpenAIResponses_ComputerUse.csproj" />
     <Project Path="samples/02-agents/AgentProviders/custom/Agent_With_CustomImplementation/Agent_With_CustomImplementation.csproj" />
     <Project Path="samples/02-agents/AgentProviders/dapr/Agent_With_Dapr/Agent_With_Dapr.csproj" />
     <Project Path="samples/02-agents/AgentProviders/github-copilot/Agent_With_GitHubCopilot/Agent_With_GitHubCopilot.csproj" />
@@ -157,7 +158,7 @@
     <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step12_Middleware/Agent_Step12_Middleware.csproj" />
     <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step13_Plugins/Agent_Step13_Plugins.csproj" />
     <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step14_CodeInterpreter/Agent_Step14_CodeInterpreter.csproj" />
-    <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step15_ComputerUse/Agent_Step15_ComputerUse.csproj" />
+    <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step15_ComputerUsePreview/Agent_Step15_ComputerUsePreview.csproj" />
     <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step16_FileSearch/Agent_Step16_FileSearch.csproj" />
     <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step17_OpenAPITools/Agent_Step17_OpenAPITools.csproj" />
     <Project Path="samples/02-agents/AgentProviders/foundry/Agent_Step18_BingCustomSearch/Agent_Step18_BingCustomSearch.csproj" />

dotnet/eng/verify-samples/AgentsSamples.cs

@@ -50,6 +50,19 @@ internal static class AgentsSamples
             ],
         },
 
+        new SampleDefinition
+        {
+            Name = "Agent_With_AzureOpenAIResponses_ComputerUse",
+            ProjectPath = "samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIResponses_ComputerUse",
+            RequiredEnvironmentVariables = ["AZURE_OPENAI_ENDPOINT"],
+            OptionalEnvironmentVariables = ["AZURE_OPENAI_DEPLOYMENT_NAME"],
+            ExpectedOutputDescription =
+            [
+                "The output should show a computer automation session running batched actions (such as click, type or keypress) against simulated browser screenshots, followed by a final response describing search results.",
+                "The output should not contain error messages or stack traces.",
+            ],
+        },
+
         new SampleDefinition
         {
             Name = "Agent_With_AzureAIProject",
@@ -1188,8 +1201,8 @@ internal static class AgentsSamples
 
         new SampleDefinition
         {
-            Name = "FoundryAgent_Step15_ComputerUse",
-            ProjectPath = "samples/02-agents/AgentProviders/foundry/Agent_Step15_ComputerUse",
+            Name = "FoundryAgent_Step15_ComputerUsePreview",
+            ProjectPath = "samples/02-agents/AgentProviders/foundry/Agent_Step15_ComputerUsePreview",
             RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT", "AZURE_AI_COMPUTER_USE_DEPLOYMENT_NAME"],
             OptionalEnvironmentVariables = [],
             ExpectedOutputDescription = ["The output should show a computer automation session processing simulated browser screenshots with iteration steps and a final response describing search results."],

dotnet/samples/02-agents/AgentProviders/README.md

@@ -37,6 +37,7 @@ See the README.md for each sample for the prerequisites for that sample.
 | [Azure Foundry Model](./azure/Agent_With_AzureFoundryModel/) | Use any model deployed to Microsoft Foundry |
 | [Azure OpenAI ChatCompletion](./azure/Agent_With_AzureOpenAIChatCompletion/) | Create an AIAgent using Azure OpenAI ChatCompletion |
 | [Azure OpenAI Responses](./azure/Agent_With_AzureOpenAIResponses/) | Create an AIAgent using Azure OpenAI Responses |
+| [Microsoft Foundry Responses computer use](./azure/Agent_With_AzureOpenAIResponses_ComputerUse/) | Use the GA computer tool with a Microsoft Foundry project Responses agent |
 
 ### [Custom](./custom/)
 

dotnet/samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIResponses_ComputerUse/Agent_With_AzureOpenAIResponses_ComputerUse.csproj

@@ -0,0 +1,25 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFrameworks>net10.0</TargetFrameworks>
+
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <NoWarn>$(NoWarn);MEAI001</NoWarn>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Identity" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <!-- Provides the Foundry project client and FoundryAITool.CreateComputerTool(). -->
+    <ProjectReference Include="..\..\..\..\..\src\Microsoft.Agents.AI.Foundry\Microsoft.Agents.AI.Foundry.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="Assets\*.jpg" CopyToOutputDirectory="Always" />
+  </ItemGroup>
+
+</Project>

dotnet/samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIResponses_ComputerUse/ComputerUseUtil.cs

@@ -0,0 +1,136 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ClientModel.Primitives;
+using System.Text.Json;
+using Microsoft.Extensions.AI;
+using OpenAI.Responses;
+
+namespace Demo.ComputerUse;
+
+/// <summary>
+/// Enum for tracking the state of the simulated web search flow.
+/// </summary>
+internal enum SearchState
+{
+    Initial,        // Browser search page
+    Typed,          // Text entered in search box
+    PressedEnter    // Search submitted, showing results
+}
+
+/// <summary>
+/// One action of a computer call batch, reduced to the fields this simulation needs.
+/// </summary>
+internal sealed record ComputerActionRequest(string Type, string? Text, IReadOnlyList<string> Keys);
+
+internal static class ComputerUseUtil
+{
+    internal static async Task<Dictionary<string, string>> UploadScreenshotAssetsAsync(IHostedFileClient fileClient)
+    {
+        string assetsDir = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Assets");
+
+        (string Key, string FileName)[] files =
+        [
+            ("browser_search", "cua_browser_search.jpg"),
+            ("search_typed", "cua_search_typed.jpg"),
+            ("search_results", "cua_search_results.jpg")
+        ];
+
+        Dictionary<string, string> screenshots = [];
+
+        foreach ((string key, string fileName) in files)
+        {
+            HostedFileContent result = await fileClient.UploadAsync(
+                Path.Combine(assetsDir, fileName),
+                new HostedFileClientOptions { Purpose = "assistants" });
+            screenshots[key] = result.FileId;
+        }
+
+        return screenshots;
+    }
+
+    internal static async Task DeleteScreenshotAssetsAsync(
+        IHostedFileClient fileClient,
+        Dictionary<string, string> screenshots)
+    {
+        foreach ((string key, string fileId) in screenshots)
+        {
+            try
+            {
+                await fileClient.DeleteAsync(fileId);
+            }
+            catch (Exception exception)
+            {
+                // Continue so one failed deletion does not leave every remaining sample asset behind.
+                Console.Error.WriteLine($"Failed to delete screenshot '{key}' ({fileId}): {exception.Message}");
+            }
+        }
+    }
+
+    /// <summary>
+    /// Reads the ordered actions of a computer call.
+    /// </summary>
+    /// <remarks>
+    /// The OpenAI .NET SDK does not expose the GA <c>actions</c> batch as a typed property yet, so the call is
+    /// re-serialized and the batch is read from its JSON. A preview call carries a single <c>action</c> instead.
+    /// </remarks>
+    internal static IReadOnlyList<ComputerActionRequest> GetActions(ComputerCallResponseItem computerCall)
+    {
+        using JsonDocument json = JsonDocument.Parse(ModelReaderWriter.Write(computerCall, ModelReaderWriterOptions.Json));
+
+        if (json.RootElement.TryGetProperty("actions", out JsonElement actions) && actions.ValueKind == JsonValueKind.Array)
+        {
+            return [.. actions.EnumerateArray().Select(ToActionRequest)];
+        }
+
+        if (json.RootElement.TryGetProperty("action", out JsonElement action) && action.ValueKind == JsonValueKind.Object)
+        {
+            return [ToActionRequest(action)];
+        }
+
+        return [];
+    }
+
+    /// <summary>
+    /// Simulates executing a computer action by advancing the state.
+    /// </summary>
+    internal static async Task<SearchState> ApplyAsync(ComputerActionRequest action, SearchState currentState)
+    {
+        if (action.Type == "wait")
+        {
+            await Task.Delay(TimeSpan.FromSeconds(1));
+        }
+
+        return action.Type switch
+        {
+            "click" when currentState == SearchState.Typed => SearchState.PressedEnter,
+            "type" when action.Text is not null => SearchState.Typed,
+            "keypress" when IsEnterKey(action) => SearchState.PressedEnter,
+            _ => currentState
+        };
+    }
+
+    /// <summary>
+    /// Returns the key of the screenshot that shows the given state.
+    /// </summary>
+    internal static string GetScreenshotKey(SearchState state) => state switch
+    {
+        SearchState.PressedEnter => "search_results",
+        SearchState.Typed => "search_typed",
+        _ => "browser_search"
+    };
+
+    private static ComputerActionRequest ToActionRequest(JsonElement action)
+    {
+        string type = action.TryGetProperty("type", out JsonElement typeElement) ? typeElement.GetString() ?? string.Empty : string.Empty;
+        string? text = action.TryGetProperty("text", out JsonElement textElement) ? textElement.GetString() : null;
+        IReadOnlyList<string> keys = action.TryGetProperty("keys", out JsonElement keysElement) && keysElement.ValueKind == JsonValueKind.Array
+            ? [.. keysElement.EnumerateArray().Select(k => k.GetString() ?? string.Empty)]
+            : [];
+
+        return new ComputerActionRequest(type, text, keys);
+    }
+
+    private static bool IsEnterKey(ComputerActionRequest action) =>
+        action.Keys.Contains("Return", StringComparer.OrdinalIgnoreCase) ||
+        action.Keys.Contains("Enter", StringComparer.OrdinalIgnoreCase);
+}

dotnet/samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIResponses_ComputerUse/Program.cs

@@ -0,0 +1,117 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+// This sample shows how to use the generally available (GA) computer tool, created with the parameterless
+// FoundryAITool.CreateComputerTool(), with an AIAgent backed by the Microsoft Foundry project Responses endpoint.
+// Each computer call returned by the model carries an ordered batch of actions; the application runs all of them and
+// then sends back a single screenshot.
+
+using Azure.AI.Projects;
+using Azure.Identity;
+using Demo.ComputerUse;
+using Microsoft.Agents.AI;
+using Microsoft.Agents.AI.Foundry;
+using Microsoft.Extensions.AI;
+using OpenAI.Responses;
+
+string endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT")
+    ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.");
+string deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4";
+
+// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production.
+// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid
+// latency issues, unintended credential probing, and potential security risks from fallback mechanisms.
+AIProjectClient projectClient = new(new Uri(endpoint), new DefaultAzureCredential());
+using IHostedFileClient fileClient = projectClient.GetProjectOpenAIClient().AsIHostedFileClient();
+
+AIAgent agent = projectClient
+    .GetProjectOpenAIClient()
+    .GetResponsesClient()
+    .AsIChatClient(deploymentName)
+    .AsAIAgent(
+        name: "ComputerAgent",
+        instructions: "You are a computer automation assistant.",
+        tools: [FoundryAITool.CreateComputerTool()]);
+
+Dictionary<string, string> screenshots = [];
+
+try
+{
+    // Foundry rejects large inline data URIs in computer_call_output, so upload the simulated screenshots once and
+    // reference their file IDs throughout the loop.
+    screenshots = await ComputerUseUtil.UploadScreenshotAssetsAsync(fileClient);
+
+    // Send the initial request with a screenshot of the browser.
+    ChatMessage message = new(ChatRole.User, [
+        new TextContent("Search for 'OpenAI news'. Type it and submit. Once you see results, the task is complete."),
+        new AIContent
+        {
+            RawRepresentation = ResponseContentPart.CreateInputImagePart(
+                imageFileId: screenshots["browser_search"],
+                imageDetailLevel: ResponseImageDetailLevel.High)
+        }
+    ]);
+
+    Console.WriteLine("Starting computer use session...");
+
+    AgentSession session = await agent.CreateSessionAsync();
+    AgentResponse response = await agent.RunAsync(message, session: session);
+
+    SearchState currentState = SearchState.Initial;
+
+    for (int i = 0; i < 10; i++)
+    {
+        // Find the next computer call.
+        ComputerCallResponseItem? computerCall = response.Messages
+            .SelectMany(m => m.Contents)
+            .Select(c => c.RawRepresentation as ComputerCallResponseItem)
+            .FirstOrDefault(item => item is not null);
+
+        if (computerCall is null)
+        {
+            if (currentState == SearchState.PressedEnter)
+            {
+                Console.WriteLine("No more computer actions. Done.");
+                Console.WriteLine(response);
+                break;
+            }
+
+            // The model may ask for confirmation before acting. Answer and continue.
+            if (response.Text.Contains('?'))
+            {
+                Console.WriteLine($"Model asked: {response.Text}");
+                response = await agent.RunAsync("Please proceed.", session);
+                continue;
+            }
+
+            Console.WriteLine(response);
+            break;
+        }
+
+        // Run every action of the batch in order, then capture a single screenshot.
+        foreach (ComputerActionRequest action in ComputerUseUtil.GetActions(computerCall))
+        {
+            Console.WriteLine($"[{i + 1}] Action: {action.Type}");
+            currentState = await ComputerUseUtil.ApplyAsync(action, currentState);
+        }
+
+        ComputerCallOutputResponseItem callOutput = new(
+            computerCall.CallId,
+            ComputerCallOutput.CreateScreenshotOutput(
+                screenshotImageFileId: screenshots[ComputerUseUtil.GetScreenshotKey(currentState)]));
+
+        // The model can flag an action as risky with pending safety checks. A real application must show them to the
+        // user and acknowledge them only after the user confirms; this simulation acknowledges them automatically.
+        foreach (ComputerCallSafetyCheck safetyCheck in computerCall.PendingSafetyChecks)
+        {
+            Console.WriteLine($"Acknowledging safety check: {safetyCheck.Code}");
+            callOutput.AcknowledgedSafetyChecks.Add(safetyCheck);
+        }
+
+        // Send the screenshot back as the computer call output.
+        response = await agent.RunAsync([new ChatMessage(ChatRole.User, [new AIContent { RawRepresentation = callOutput }])], session);
+    }
+}
+finally
+{
+    await ComputerUseUtil.DeleteScreenshotAssetsAsync(fileClient, screenshots);
+}

dotnet/samples/02-agents/AgentProviders/azure/Agent_With_AzureOpenAIResponses_ComputerUse/README.md

@@ -0,0 +1,55 @@
+# Computer use with Microsoft Foundry Responses
+
+This sample shows how to use the generally available (GA) computer tool with an `AIAgent` backed by a Microsoft Foundry project Responses endpoint.
+
+## What this sample demonstrates
+
+- Using the parameterless `FoundryAITool.CreateComputerTool()` to add the GA `computer` tool (`{"type":"computer"}`)
+- Running every action of a batched `computer_call` (the GA call returns an ordered `actions` list) before sending back one screenshot
+- Uploading screenshots to Foundry Files and returning file IDs from the computer loop
+
+For the preview `computer_use_preview` tool with a Microsoft Foundry project, which takes an environment and display size and returns a single `action` per call, see [Agent_Step15_ComputerUsePreview](../../foundry/Agent_Step15_ComputerUsePreview/).
+
+| | GA tool (this sample) | Preview tool (Agent_Step15_ComputerUsePreview) |
+| --- | --- | --- |
+| Factory | `FoundryAITool.CreateComputerTool()` | `FoundryAITool.CreateComputerTool(environment, width, height)` |
+| Wire tool | `{"type":"computer"}` | `{"type":"computer_use_preview", ...}` |
+| Actions per call | ordered `actions` batch | single `action` |
+
+For more information, see the OpenAI [computer use guide](https://developers.openai.com/api/docs/guides/tools-computer-use).
+
+## Known limitation
+
+The sample continues each turn from the stored response, so only the new `computer_call_output` is sent. Sending a GA `computer_call` back to the model instead (for example with stored responses disabled and the history kept locally) currently fails. The OpenAI .NET SDK models only the preview item and adds `"action": null`, which the Responses API rejects for the GA tool.
+
+## How the simulation works
+
+**This sample does not connect to a real browser.** It intercepts the model's actions and returns pre-captured screenshots as if the actions were performed:
+
+| State | Reached by | Screenshot sent back |
+| --- | --- | --- |
+| Initial | Session start | `cua_browser_search.jpg`, an empty search page |
+| Typed | A `type` action | `cua_search_typed.jpg`, search text in the box |
+| Search submitted | A `keypress` of Enter, or a `click` after typing | `cua_search_results.jpg`, the results page |
+
+Each loop iteration runs all actions of the call in order, then sends the screenshot for the resulting state. The loop ends when the model stops returning computer calls after the search was submitted, or after 10 iterations.
+
+## Prerequisites
+
+- .NET 10 SDK or later
+- A Microsoft Foundry project with a deployment of a model that supports the GA computer tool (for example `gpt-5.4`)
+- Foundry Files configured for the project, including storage access for the project's managed identity
+- An authenticated Azure identity with access to the project (for example, sign in with `az login`)
+
+Set the following environment variables:
+
+```powershell
+$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project"
+$env:FOUNDRY_MODEL="gpt-5.4" # Optional, defaults to gpt-5.4
+```
+
+## Run the sample
+
+```powershell
+dotnet run
+```