From b086ea734e3b20451a69418adc96141b51d93e04 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+rogerbarreto@users.noreply.github.com> Date: Fri, 23 Oct 2026 17:51:16 -0700 Subject: [PATCH] fix(dotnet): apply toolbox approval policy Apply discovered tool configuration through the existing approval pipeline while retaining OAuth consent handling and per-call state. Keep unsupported explicit configurations on the approval path. --- .../FoundryToolboxService.cs | 55 ++- .../FoundryToolboxApprovalTests.cs | 379 ++++++++++++++++++ 2 files changed, 433 insertions(+), 1 deletion(-) create mode 100644 dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryToolboxApprovalTests.cs diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryToolboxService.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryToolboxService.cs index 868bc752e8e..c6f759f839d 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryToolboxService.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryToolboxService.cs @@ -6,6 +6,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Net.Http; +using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using Azure.Core; @@ -44,6 +45,14 @@ namespace Microsoft.Agents.AI.Foundry.Hosting; /// rejected when is and /// lazily connected otherwise. /// +/// +/// Discovered tools with _meta.tool_configuration.require_approval set to always +/// require approval for each call through . An explicit +/// unsupported or malformed configuration also requires approval and is logged as a warning. +/// Existing approval requirements exposed by the function's services take precedence over metadata. +/// Tools with no approval configuration, or with require_approval set to never, +/// retain their normal invocation behavior. +/// /// [Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] public sealed class FoundryToolboxService : IHostedService, IAsyncDisposable @@ -893,7 +902,10 @@ private async Task OpenToolboxAsync( var wrapped = new List(mcpTools.Count); foreach (var tool in mcpTools) { - wrapped.Add(new ConsentAwareMcpClientAIFunction(tool, toolboxName)); + AIFunction function = new ConsentAwareMcpClientAIFunction(tool, toolboxName); + wrapped.Add(this.RequiresToolApproval(tool, toolboxName) + ? new ApprovalRequiredAIFunction(function) + : function); } _ = version; // reserved for future version-specific routing; currently handled server-side by the proxy. @@ -901,6 +913,47 @@ private async Task OpenToolboxAsync( return new ToolboxOpenResult(new CachedToolbox(client!, httpClient, wrapped), Consents: null); } + internal bool RequiresToolApproval(AIFunction tool, string toolboxName) + { + if (tool.GetService() is not null) + { + return true; + } + + if (tool.GetService()?.ProtocolTool.Meta is not { } meta + || !meta.TryGetPropertyValue("tool_configuration", out var configuration)) + { + return false; + } + + if (configuration is JsonObject toolConfiguration) + { + if (!toolConfiguration.TryGetPropertyValue("require_approval", out var approval)) + { + return false; + } + + if (approval is JsonValue value && value.TryGetValue(out var policy)) + { + if (string.Equals(policy, "always", StringComparison.Ordinal)) + { + return true; + } + + if (string.Equals(policy, "never", StringComparison.Ordinal)) + { + return false; + } + } + } + + this._logger.LogWarning( + "Tool '{ToolName}' in toolbox '{ToolboxName}' has an unsupported approval configuration; requiring approval for every call.", + tool.Name, + toolboxName); + return true; + } + /// public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryToolboxApprovalTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryToolboxApprovalTests.cs new file mode 100644 index 00000000000..1423494d2ee --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryToolboxApprovalTests.cs @@ -0,0 +1,379 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Azure.Core; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Moq; + +namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; + +public sealed class FoundryToolboxApprovalTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RequiresToolApproval_ExistingApprovalService_ReturnsTrueAsync(bool delegated) + { + // Arrange + AIFunction tool = new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => "Sunny", "get_weather")); + if (delegated) + { + var wrapper = new Mock(); + wrapper.Setup(f => f.GetService(typeof(ApprovalRequiredAIFunction), null)).Returns(tool); + tool = wrapper.Object; + } + + await using var service = new FoundryToolboxService( + Options.Create(new FoundryToolboxOptions()), + Mock.Of()); + + // Act + var requiresApproval = service.RequiresToolApproval(tool, "weather"); + + // Assert + Assert.True(requiresApproval); + } + + [Theory] + [InlineData(false, true)] + [InlineData(false, false)] + [InlineData(true, true)] + [InlineData(true, false)] + public async Task DiscoveredTool_RequiresDecisionForEachCallAsync(bool streaming, bool approved) + { + // Arrange + await using var toolbox = await ToolboxServer.CreateAsync("""{"tool_configuration":{"require_approval":"always"}}"""); + await using var service = toolbox.CreateService(); + await service.StartAsync(CancellationToken.None); + var tool = Assert.IsAssignableFrom(Assert.Single(service.Tools)); + var calls = 0; + ChatResponse NextResponse() => ++calls == 1 || calls == 3 + ? new(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent( + $"call-{calls}", tool.Name, new Dictionary { ["city"] = "Amsterdam" })])) + : new(new ChatMessage(ChatRole.Assistant, "Done")); + var client = new Mock(); + client.Setup(c => c.GetResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(() => Task.FromResult(NextResponse())); + client.Setup(c => c.GetStreamingResponseAsync(It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(() => ToUpdatesAsync(NextResponse())); + var agent = new ChatClientAgent(client.Object, new ChatClientAgentOptions + { + ChatOptions = new() { Tools = [tool] }, + }); + var session = await agent.CreateSessionAsync(); + + // Act + var response = await RunAsync(agent, session, [new(ChatRole.User, "Get the weather")], streaming); + + // Assert + Assert.Equal(0, toolbox.InvocationCount); + var request = Assert.Single(response.Messages.SelectMany(m => m.Contents).OfType()); + var call = Assert.IsType(request.ToolCall); + Assert.Equal(tool.Name, call.Name); + Assert.Equal("call-1", call.CallId); + Assert.Equal("Amsterdam", Assert.IsType(call.Arguments!["city"])); + + // Approval remains bound to the originally proposed arguments. + call.Arguments["city"] = "Rotterdam"; + await RunAsync(agent, session, [new(ChatRole.User, [request.CreateResponse(approved)])], streaming); + Assert.Equal(approved ? 1 : 0, toolbox.InvocationCount); + if (approved) + { + Assert.Equal("Amsterdam", toolbox.LastArguments!["city"]!.GetValue()); + } + + var next = await RunAsync(agent, session, [new(ChatRole.User, "Get the weather again")], streaming); + Assert.Single(next.Messages.SelectMany(m => m.Contents).OfType()); + Assert.Equal(approved ? 1 : 0, toolbox.InvocationCount); + } + + [Theory] + [InlineData(null, false)] + [InlineData("{}", false)] + [InlineData("""{"unrelated":true}""", false)] + [InlineData("""{"tool_configuration":{}}""", false)] + [InlineData("""{"tool_configuration":{"require_approval":"never"}}""", false)] + [InlineData("""{"tool_configuration":{"require_approval":"always"}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":"unknown"}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":""}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":"ALWAYS"}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":null}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":false}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":1}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":[]}}""", true)] + [InlineData("""{"tool_configuration":{"require_approval":{}}}""", true)] + [InlineData("""{"tool_configuration":null}""", true)] + [InlineData("""{"tool_configuration":[]}""", true)] + [InlineData("""{"tool_configuration":"never"}""", true)] + public async Task DiscoveredTool_ApprovalMetadataIsAppliedAsync(string? metadata, bool requiresApproval) + { + // Arrange + await using var toolbox = await ToolboxServer.CreateAsync(metadata); + await using var service = toolbox.CreateService(); + + // Act + await service.StartAsync(CancellationToken.None); + var tool = Assert.IsAssignableFrom(Assert.Single(service.Tools)); + + // Assert + Assert.Equal(requiresApproval, tool.GetService() is not null); + Assert.Equal("get_weather", tool.Name); + Assert.Equal("Gets the weather", tool.Description); + Assert.True(tool.JsonSchema.GetProperty("properties").TryGetProperty("city", out _)); + Assert.Equal(0, toolbox.InvocationCount); + if (!requiresApproval) + { + await tool.InvokeAsync(new AIFunctionArguments { ["city"] = "Amsterdam" }); + Assert.Equal(1, toolbox.InvocationCount); + } + } + + [Theory] + [InlineData("lazy")] + [InlineData("deferred")] + [InlineData("consent")] + public async Task RequestDiscoveredTool_RequiresApprovalAsync(string discoveryPath) + { + // Arrange + await using var toolbox = await ToolboxServer.CreateAsync("""{"tool_configuration":{"require_approval":"always"}}"""); + if (discoveryPath is "deferred") + { + toolbox.ListError = new JsonObject { ["code"] = -32603, ["message"] = "Try again" }; + } + else if (discoveryPath is "consent") + { + toolbox.ListError = new JsonObject + { + ["code"] = -32006, + ["message"] = """tools/list failed {"errors":[{"name":"get_weather","type":"mcp","error":{"code":"CONSENT_REQUIRED","message":"https://auth.example.com/authorize"}}]}""", + }; + } + + await using var service = toolbox.CreateService(preRegistered: discoveryPath is not "lazy"); + await service.StartAsync(CancellationToken.None); + Assert.Empty(service.Tools); + Assert.Equal( + discoveryPath is "deferred" ? FoundryToolboxStartupStatus.Degraded : + discoveryPath is "consent" ? FoundryToolboxStartupStatus.ConsentRequired : FoundryToolboxStartupStatus.Healthy, + service.StartupStatus); + toolbox.ListError = null; + await using var scope = service.CreateRequestScope(); + var previousScope = HostedCallContext.ToolboxCacheScopeId; + HostedCallContext.ToolboxCacheScopeId = scope.Id; + try + { + // Act + IReadOnlyList tools; + if (discoveryPath is "lazy") + { + tools = (await service.GetToolboxToolsAsync("weather", version: null, CancellationToken.None)).Tools; + } + else + { + if (discoveryPath is "deferred") + { + await service.RetryDeferredToolboxesAsync(CancellationToken.None); + } + else + { + Assert.Empty(await service.ResolvePendingConsentsAsync(CancellationToken.None)); + } + + tools = service.GetCurrentRequestTools(); + } + + // Assert + Assert.NotNull(Assert.IsAssignableFrom(Assert.Single(tools)).GetService()); + Assert.Empty(service.Tools); + Assert.Equal(0, toolbox.InvocationCount); + } + finally + { + HostedCallContext.ToolboxCacheScopeId = previousScope; + } + } + + [Fact] + public async Task DiscoveredTool_ApprovalWrapperPreservesConsentAsync() + { + // Arrange + const string ConsentUrl = "https://auth.example.com/authorize"; + await using var toolbox = await ToolboxServer.CreateAsync("""{"tool_configuration":{"require_approval":"always"}}"""); + toolbox.CallError = new JsonObject { ["code"] = -32006, ["message"] = ConsentUrl }; + await using var service = toolbox.CreateService(); + await service.StartAsync(CancellationToken.None); + var tool = Assert.IsType(Assert.Single(service.Tools)); + using var cancellation = new CancellationTokenSource(); + var state = new RequestConsentState { CancellationSource = cancellation }; + var previous = McpConsentContext.Current.Value; + McpConsentContext.Current.Value = state; + try + { + // Act + await Assert.ThrowsAnyAsync( + async () => await tool.InvokeAsync(new AIFunctionArguments { ["city"] = "Amsterdam" }, cancellation.Token)); + + // Assert + Assert.True(cancellation.IsCancellationRequested); + Assert.NotNull(state.Pending); + Assert.Equal("weather", state.Pending.ToolboxName); + Assert.Equal("get_weather", state.Pending.ToolName); + Assert.EndsWith(ConsentUrl, state.Pending.ConsentUrl, StringComparison.Ordinal); + Assert.Equal(1, toolbox.InvocationCount); + } + finally + { + McpConsentContext.Current.Value = previous; + } + } + + private static async Task RunAsync( + AIAgent agent, AgentSession session, IEnumerable messages, bool streaming) + { + if (!streaming) + { + return await agent.RunAsync(messages, session); + } + + return await agent.RunStreamingAsync(messages, session).ToAgentResponseAsync(); + } + + private static async IAsyncEnumerable ToUpdatesAsync(ChatResponse response) + { + await Task.CompletedTask; + foreach (var update in response.ToChatResponseUpdates()) + { + yield return update; + } + } + + private sealed class ToolboxServer(WebApplication app) : IAsyncDisposable + { + internal int InvocationCount { get; private set; } + internal JsonNode? LastArguments { get; private set; } + internal JsonObject? ListError { get; set; } + internal JsonObject? CallError { get; set; } + + internal static async Task CreateAsync(string? metadata) + { + var builder = WebApplication.CreateBuilder(); + builder.WebHost.UseUrls("http://127.0.0.1:0"); + builder.Logging.ClearProviders(); + builder.Logging.AddConsole(); + builder.Logging.SetMinimumLevel(LogLevel.Warning); + var app = builder.Build(); + var server = new ToolboxServer(app); + app.Run(async context => + { + if (!HttpMethods.IsPost(context.Request.Method)) + { + context.Response.StatusCode = StatusCodes.Status405MethodNotAllowed; + return; + } + + var request = (await JsonNode.ParseAsync(context.Request.Body, cancellationToken: context.RequestAborted))!; + if (request["id"] is null) + { + context.Response.StatusCode = StatusCodes.Status202Accepted; + return; + } + + JsonNode result; + switch (request["method"]!.GetValue()) + { + case "initialize": + result = new JsonObject + { + ["protocolVersion"] = request["params"]!["protocolVersion"]!.DeepClone(), + ["capabilities"] = new JsonObject { ["tools"] = new JsonObject() }, + ["serverInfo"] = new JsonObject { ["name"] = "test-toolbox", ["version"] = "1.0" }, + }; + break; + case "tools/list": + if (server.ListError is { } listError) + { + await WriteErrorAsync(context, request["id"]!, listError); + return; + } + + var tool = JsonNode.Parse(""" + {"name":"get_weather","description":"Gets the weather", + "inputSchema":{"type":"object","properties":{"city":{"type":"string"}},"required":["city"]}} + """)!; + if (metadata is not null) + { + tool["_meta"] = JsonNode.Parse(metadata); + } + + result = new JsonObject { ["tools"] = new JsonArray(tool) }; + break; + case "tools/call": + server.InvocationCount++; + server.LastArguments = request["params"]!["arguments"]!.DeepClone(); + if (server.CallError is { } callError) + { + await WriteErrorAsync(context, request["id"]!, callError); + return; + } + + result = JsonNode.Parse("""{"content":[{"type":"text","text":"Sunny"}]}""")!; + break; + default: + await WriteErrorAsync(context, request["id"]!, new JsonObject { ["code"] = -32601, ["message"] = "Method not found" }); + return; + } + + await context.Response.WriteAsJsonAsync(new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = request["id"]!.DeepClone(), + ["result"] = result, + }, context.RequestAborted); + }); + await app.StartAsync(); + return server; + } + + private static Task WriteErrorAsync(HttpContext context, JsonNode id, JsonObject error) => + context.Response.WriteAsJsonAsync(new JsonObject + { + ["jsonrpc"] = "2.0", + ["id"] = id.DeepClone(), + ["error"] = error.DeepClone(), + }, context.RequestAborted); + + internal FoundryToolboxService CreateService(bool preRegistered = true) + { + var credential = new Mock(); + credential.Setup(c => c.GetTokenAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(new AccessToken("test-token", DateTimeOffset.UtcNow.AddHours(1))); + var options = new FoundryToolboxOptions + { + EndpointOverride = app.Urls.Single(), + StrictMode = preRegistered, + }; + if (preRegistered) + { + options.ToolboxNames.Add("weather"); + } + + return new FoundryToolboxService( + Options.Create(options), + credential.Object, + app.Services.GetRequiredService>()); + } + + public ValueTask DisposeAsync() => app.DisposeAsync(); + } +}