From f3db78f43067c94443cee30e9ee669d22437c841 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:11:59 +0100 Subject: [PATCH 1/6] .NET: Add Microsoft.Agents.AI.TypeSafe Jev AIFunction tool (alpha) Adds an alpha package that exposes Jev, the TypeSafe System One classification model, to agents as an AIFunction built with JevAIToolBuilder. The function arguments are a JevRequest (state plus typed choice, score, and noul questions) and the result is the evaluator's JevResponse serialized as JSON. - JevAIToolOptions derives from ClientPipelineOptions; the TypeSafe API is called through a System.ClientModel pipeline authenticated with an ApiKeyCredential, retrying 429/529 and capping Retry-After at one minute. - UseEvaluator and UseClient let any other client answer requests, with input and output mappers; UseClient resolves the client from the agent's services in a new scope per call. - Requests from the model and evaluator responses are validated against the Jev contract before they reach the API or the model. - Adds unit tests and the Agent_Step25_ClassificationWithJev sample. --- dotnet/agent-framework-dotnet.slnx | 3 + dotnet/agent-framework-src-and-tests.slnf | 2 + .../Agent_Step25_ClassificationWithJev.csproj | 20 + .../Program.cs | 63 +++ .../README.md | 100 ++++ dotnet/samples/02-agents/Agents/README.md | 1 + .../Core/JevAIFunction.cs | 89 ++++ .../Core/JevApiRequest.cs | 22 + .../Core/JevContractValidator.cs | 214 ++++++++ .../Core/JevHttpEvaluator.cs | 88 ++++ .../Core/JevJsonContext.cs | 20 + .../Core/JevJsonUtilities.cs | 60 +++ .../Core/JevResponseClassifier.cs | 39 ++ .../Core/JevRetryPolicy.cs | 87 ++++ .../JevAIToolBuilder.cs | 231 +++++++++ .../JevAIToolOptions.cs | 112 +++++ .../Microsoft.Agents.AI.TypeSafe/JevAnswer.cs | 24 + .../JevChoiceAnswer.cs | 35 ++ .../JevChoiceQuestion.cs | 28 ++ .../JevNoulAnswer.cs | 20 + .../JevNoulCriteria.cs | 28 ++ .../JevNoulQuestion.cs | 24 + .../JevQuestion.cs | 32 ++ .../JevRequest.cs | 41 ++ .../JevResponse.cs | 41 ++ .../JevScoreAnswer.cs | 42 ++ .../JevScoreQuestion.cs | 27 + .../Microsoft.Agents.AI.TypeSafe/JevUsage.cs | 26 + .../Microsoft.Agents.AI.TypeSafe.csproj | 31 ++ .../JevAIFunctionTests.cs | 463 ++++++++++++++++++ .../JevAIToolBuilderTests.cs | 172 +++++++ .../JevAgentIntegrationTests.cs | 114 +++++ .../JevHttpEvaluatorTests.cs | 291 +++++++++++ .../JevTestData.cs | 105 ++++ .../JevTestHttpHandler.cs | 55 +++ ...rosoft.Agents.AI.TypeSafe.UnitTests.csproj | 17 + .../ScriptedChatClient.cs | 52 ++ 37 files changed, 2819 insertions(+) create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index 77e772cf214..ec1b47fdcbe 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -69,6 +69,7 @@ + @@ -631,6 +632,7 @@ + @@ -689,6 +691,7 @@ + diff --git a/dotnet/agent-framework-src-and-tests.slnf b/dotnet/agent-framework-src-and-tests.slnf index b6639c7a09b..bf22d4be168 100644 --- a/dotnet/agent-framework-src-and-tests.slnf +++ b/dotnet/agent-framework-src-and-tests.slnf @@ -29,6 +29,7 @@ "src\\Microsoft.Agents.AI.OpenAI\\Microsoft.Agents.AI.OpenAI.csproj", "src\\Microsoft.Agents.AI.Purview\\Microsoft.Agents.AI.Purview.csproj", "src\\Microsoft.Agents.AI.Tools.Shell\\Microsoft.Agents.AI.Tools.Shell.csproj", + "src\\Microsoft.Agents.AI.TypeSafe\\Microsoft.Agents.AI.TypeSafe.csproj", "src\\Microsoft.Agents.AI.Valkey\\Microsoft.Agents.AI.Valkey.csproj", "src\\Microsoft.Agents.AI.Workflows.Declarative.Foundry\\Microsoft.Agents.AI.Workflows.Declarative.Foundry.csproj", "src\\Microsoft.Agents.AI.Workflows.Declarative.Mcp\\Microsoft.Agents.AI.Workflows.Declarative.Mcp.csproj", @@ -64,6 +65,7 @@ "tests\\Microsoft.Agents.AI.OpenAI.UnitTests\\Microsoft.Agents.AI.OpenAI.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Purview.UnitTests\\Microsoft.Agents.AI.Purview.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Tools.Shell.UnitTests\\Microsoft.Agents.AI.Tools.Shell.UnitTests.csproj", + "tests\\Microsoft.Agents.AI.TypeSafe.UnitTests\\Microsoft.Agents.AI.TypeSafe.UnitTests.csproj", "tests\\Microsoft.Agents.AI.UnitTests\\Microsoft.Agents.AI.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Valkey.UnitTests\\Microsoft.Agents.AI.Valkey.UnitTests.csproj", "tests\\Microsoft.Agents.AI.Workflows.Declarative.Mcp.UnitTests\\Microsoft.Agents.AI.Workflows.Declarative.Mcp.UnitTests.csproj", diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj new file mode 100644 index 00000000000..26193187de0 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs new file mode 100644 index 00000000000..c99b7db4f8f --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Classification with Jev — Give an agent a fast, calibrated classifier as a tool +// +// Jev, the TypeSafe System One model, does not write text: it answers typed questions +// (choice, score, noul) about a state and returns calibrated probabilities. This sample gives +// a support agent a Jev tool built with JevAIToolBuilder. The agent's model decides which +// questions to ask, Jev answers them, and the model writes the routing decision, sending +// uncertain tickets to a human. + +using System.ClientModel; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; +var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// +// The tool's arguments are a Jev request (the state plus typed questions) and its result is the Jev response. +AIFunction jev = new JevAIToolBuilder(new ApiKeyCredential(typeSafeApiKey)).Build(); +// + +const string Instructions = """ + You triage customer support tickets. For every ticket, call evaluate_with_jev once with the ticket text as the + state and these questions: + - "department": a choice between billing (payments, invoices, refunds), technical (bugs, outages, integrations), + and sales (pricing, upgrades, new accounts). + - "urgent": a noul asking whether the customer needs help urgently. + - "frustration": a score with the levels calm, annoyed, and angry. + Then reply in one line with the department, whether it is urgent, and the frustration level. When the department + confidence is below 0.6, say the ticket needs human review instead of naming a department. + """; + +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) + .AsAIAgent(model: deploymentName, instructions: Instructions, name: "TicketTriage", tools: [jev]); + +string[] tickets = +[ + "I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales. Please help ASAP.", + "Hi, could you send me a copy of last month's invoice when you get a chance? Thanks!", + "I'd like to talk to someone about my account.", +]; + +foreach (string ticket in tickets) +{ + AgentResponse response = await agent.RunAsync(ticket); + + Console.WriteLine($"Ticket: {ticket}"); + foreach (FunctionResultContent result in response.Messages.SelectMany(static m => m.Contents).OfType()) + { + Console.WriteLine($"Jev answers: {result.Result}"); + } + + Console.WriteLine($"Agent: {response.Text}"); + Console.WriteLine(); +} diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md new file mode 100644 index 00000000000..21a96610475 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md @@ -0,0 +1,100 @@ +# Classification with Jev + +This sample gives an agent a classifier tool built with `JevAIToolBuilder` from `Microsoft.Agents.AI.TypeSafe`. + +Jev, the TypeSafe System One model, does not write text. It answers typed questions about a state and returns +calibrated probabilities. An agent's model writes the replies; Jev makes the fast, structured decisions: + +1. The agent's model calls `evaluate_with_jev` with a Jev request: the `state` to evaluate and the `questions` to + answer about it. +2. Jev answers each question: + + | Question type | What it answers | Answer | + | --- | --- | --- | + | `choice` | Which of these options? | The chosen option, the probability of each option, and a confidence | + | `score` | Which level of an ordered scale (2 to 10 levels)? | A score that can fall between levels, the probability of each level, and a confidence | + | `noul` | Is this yes or no statement true? | The probability, from 0 to 1, that it is true | + +3. The model reads the answers and writes the reply. Here it routes each support ticket and sends uncertain tickets to + human review. + +## Configuring the client + +`JevAIToolBuilder` hides the HTTP client. The API key is an `ApiKeyCredential`, and `JevAIToolOptions` follows the +`ClientPipelineOptions` pattern of the OpenAI and Azure AI client libraries: + +```csharp +var options = new JevAIToolOptions +{ + ModelId = "jev-latest", // default + Transport = new HttpClientPipelineTransport(httpClient), // for example from IHttpClientFactory + NetworkTimeout = TimeSpan.FromSeconds(30), +}; +options.AddPolicy(myPolicy, PipelinePosition.PerCall); // custom pipeline policies + +AIFunction jev = new JevAIToolBuilder(new ApiKeyCredential(apiKey), options).Build(); +``` + +The default retry policy retries rate-limited (429) and overloaded (529) requests, and fails at once when the API asks +for a wait longer than one minute, so the agent run is never blocked for long. `Name` and `Description` set what the +agent's model sees. + +## Using another client + +Any other client can answer the requests instead of the TypeSafe API. Pass a delegate that takes a `JevRequest` and +returns a `JevResponse`, or map between the Jev types and the client's own types: + +```csharp +AIFunction jev = new JevAIToolBuilder() + .UseClient( + myClient.EvaluateAsync, + inputMapper: request => ToMyClientRequest(request), + outputMapper: response => ToJevResponse(response)) + .Build(); +``` + +The tool checks the mapped response before the model sees it: every question needs an answer of the same type, and a +`choice` answer and its probabilities may only use the options the question offered. + +## Dependency injection + +Every evaluator receives the `IServiceProvider` of the agent that calls the tool. `FunctionInvokingChatClient` passes +it in `AIFunctionArguments.Services`, from the services the agent was created with. `UseClient` resolves +a registered client from a new scope on every call and disposes the scope afterwards, so scoped and disposable clients +work even though hosted agents are singletons that receive the root provider: + +```csharp +builder.Services.AddScoped(); +builder.Services.AddAIAgent("triage", Instructions) + .WithAITool(sp => new JevAIToolBuilder() + .UseClient( + (client, request, cancellationToken) => client.EvaluateAsync(request, cancellationToken), + inputMapper: ToMyClientRequest, + outputMapper: ToJevResponse) + .Build()); +``` + +With the TypeSafe API, build the tool in the same `WithAITool` factory and take the credential and the transport from +the container or configuration. + +## Prerequisites + +- .NET 10 SDK or later +- A Microsoft Foundry project with a model deployment +- Azure CLI installed and authenticated (for Azure credential authentication) +- A TypeSafe API key from the [TypeSafe console](https://console.typesafe.ai/keys) + +Set the following environment variables: + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" # Replace with your Foundry project endpoint +$env:FOUNDRY_MODEL="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini +$env:TYPESAFE_API_KEY="" +``` + +## Run the sample + +```powershell +cd dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev +dotnet run +``` diff --git a/dotnet/samples/02-agents/Agents/README.md b/dotnet/samples/02-agents/Agents/README.md index 487e7f269a2..356cf1116bf 100644 --- a/dotnet/samples/02-agents/Agents/README.md +++ b/dotnet/samples/02-agents/Agents/README.md @@ -51,6 +51,7 @@ Before you begin, ensure you have the following prerequisites: |[Switching agent operating mode](./Agent_Step22_AgentMode/)|This sample demonstrates how to use the AgentModeProvider to track and switch an agent's operating mode at runtime, including the built-in plan/execute modes and custom modes, with a simple input loop that switches mode using a slash command.| |[Tracking work with a todo list](./Agent_Step23_TodoList/)|This sample demonstrates how to use the TodoProvider to let an agent plan and track multi-step work using a todo list that persists across turns, printing the evolving todo list after each turn.| |[Routing turns across multiple models](./Agent_Step24_MultiModelRouting/)|This sample demonstrates how to use the RoutePersistingRoutingChatClient to route each agent turn to one of several named chat clients, switching the active model mid-conversation while preserving the conversation history.| +|[Classification with Jev](./Agent_Step25_ClassificationWithJev/)|This sample demonstrates how to give an agent a Jev tool built with JevAIToolBuilder, so the agent's model asks Jev typed choice, score, and noul questions and uses the calibrated answers to route support tickets.| ## Running the samples from the console diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs new file mode 100644 index 00000000000..8001bbd4f59 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The created by : its arguments are a +/// and its result is the of the configured evaluator. +/// +internal sealed class JevAIFunction : AIFunction +{ + private readonly Func> _evaluateAsync; + + public JevAIFunction(string name, string description, Func> evaluateAsync) + { + this.Name = name; + this.Description = description; + this._evaluateAsync = evaluateAsync; + } + + public override string Name { get; } + + public override string Description { get; } + + public override JsonElement JsonSchema => JevJsonUtilities.RequestSchema; + + public override JsonElement? ReturnJsonSchema => JevJsonUtilities.ResponseSchema; + + public override JsonSerializerOptions JsonSerializerOptions => JevJsonUtilities.Options; + + protected override async ValueTask InvokeCoreAsync(AIFunctionArguments arguments, CancellationToken cancellationToken) + { + JevRequest request = ReadRequest(arguments); + JevContractValidator.ValidateRequest(request); + + // FunctionInvokingChatClient sets Services to the IServiceProvider of the agent (ChatClientAgent passes its + // services through), so evaluators can resolve per-call dependencies such as a registered client. + JevResponse? response = await this._evaluateAsync(request, arguments.Services, cancellationToken).ConfigureAwait(false); + if (response is null) + { + Throw.InvalidOperationException("The Jev evaluator returned no response."); + } + + JevContractValidator.ValidateResponse(request, response); + + // The result is returned as JSON, as AIFunctionFactory does. Chat clients serialize any other result object + // with AIJsonUtilities.DefaultOptions, which has no metadata for these types when reflection is disabled + // (for example under native AOT); the model would then receive an empty tool output. + try + { + return JsonSerializer.SerializeToElement(response, JevJsonContext.Default.JevResponse); + } + catch (Exception ex) when (ex is ArgumentException or JsonException or NotSupportedException) + { + // A response that cannot be written is an evaluator fault. Reported as ArgumentException, it would look + // like a mistake in the model's arguments and invite the model to retry the same call. + throw new InvalidOperationException($"The Jev response could not be serialized: {ex.Message}", ex); + } + } + + private static JevRequest ReadRequest(AIFunctionArguments arguments) + { + // The arguments are the top-level properties of the request, so they are reassembled into one object. + var json = new JsonObject(); + foreach (KeyValuePair argument in arguments) + { + json[argument.Key] = JevJsonUtilities.ToNode(argument.Value); + } + + try + { + return json.Deserialize(JevJsonContext.Default.JevRequest) + ?? throw new JsonException("The arguments are null."); + } + catch (Exception ex) when (ex is JsonException or NotSupportedException) + { + // An unknown or missing question type surfaces as NotSupportedException rather than JsonException. + throw new ArgumentException($"The arguments are not a valid Jev request: {ex.Message}", nameof(arguments), ex); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs new file mode 100644 index 00000000000..718e446afe3 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevApiRequest.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The body of a TypeSafe System One API request: a plus the model that answers it. +/// +internal sealed class JevApiRequest +{ + [JsonPropertyName("state")] + public required JsonElement State { get; init; } + + [JsonPropertyName("model")] + public required string Model { get; init; } + + [JsonPropertyName("questions")] + public required IReadOnlyDictionary Questions { get; init; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs new file mode 100644 index 00000000000..3188ff4333c --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs @@ -0,0 +1,214 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Checks requests before they are evaluated and responses before they are returned to the caller. +/// +/// +/// Requests come from a model, so their errors are reported as messages that say how to +/// fix the arguments. Responses come from an evaluator, possibly a custom client with its own mappers, so their errors +/// are reported as ; they point at an evaluator or mapper bug, not at the model. +/// +internal static class JevContractValidator +{ + /// The most options that the Jev API accepts in one Choice question. + public const int MaxChoiceOptions = 255; + + /// + /// The fewest levels of a Score question. The API also accepts a single level, but the documentation requires two, + /// and a one-level score always returns that level, so it is rejected as a request mistake. + /// + public const int MinScoreLevels = 2; + + /// The most levels that the Jev API accepts in one Score question. + public const int MaxScoreLevels = 10; + + // Jev computes scores and probabilities in floating point, so values may exceed their bounds by rounding error. + private const double Tolerance = 1e-6; + + public static void ValidateRequest(JevRequest request) + { + if (request.State.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array)) + { + Throw.ArgumentException(nameof(request), "The state must be a string, a JSON object, or a JSON array."); + } + + if (request.Questions is null || request.Questions.Count == 0) + { + Throw.ArgumentException(nameof(request), "The request must contain at least one question."); + } + + foreach (KeyValuePair entry in request.Questions) + { + string id = entry.Key; + if (string.IsNullOrWhiteSpace(id)) + { + Throw.ArgumentException(nameof(request), "Question IDs must not be empty."); + } + + JevQuestion question = entry.Value; + if (question is null) + { + Throw.ArgumentException(nameof(request), $"Question '{id}' is null."); + } + + if (string.IsNullOrWhiteSpace(question.Instructions)) + { + Throw.ArgumentException(nameof(request), $"Question '{id}' must have instructions."); + } + + switch (question) + { + case JevChoiceQuestion choice: + int options = choice.Criteria?.Count ?? 0; + if (options is < 1 or > MaxChoiceOptions) + { + Throw.ArgumentException(nameof(request), $"Choice question '{id}' has {Count(options, "option")}; it needs from 1 to {MaxChoiceOptions}."); + } + + foreach (string option in choice.Criteria!.Keys) + { + if (string.IsNullOrWhiteSpace(option)) + { + Throw.ArgumentException(nameof(request), $"Choice question '{id}' has an empty option name."); + } + } + + break; + + case JevScoreQuestion score: + int levels = score.Criteria?.Count ?? 0; + if (levels is < MinScoreLevels or > MaxScoreLevels) + { + Throw.ArgumentException(nameof(request), $"Score question '{id}' has {Count(levels, "level")}; it needs from {MinScoreLevels} to {MaxScoreLevels}."); + } + + foreach (string level in score.Criteria!) + { + if (string.IsNullOrWhiteSpace(level)) + { + Throw.ArgumentException(nameof(request), $"Score question '{id}' has an empty level description."); + } + } + + break; + } + } + } + + public static void ValidateResponse(JevRequest request, JevResponse response) + { + if (response.Answers is null) + { + Throw.InvalidOperationException("The Jev response has no answers."); + } + + foreach (KeyValuePair entry in request.Questions) + { + string id = entry.Key; + if (!response.Answers.TryGetValue(id, out JevAnswer? answer) || answer is null) + { + Throw.InvalidOperationException($"The Jev response has no answer for question '{id}'."); + } + + switch (entry.Value, answer) + { + case (JevChoiceQuestion question, JevChoiceAnswer choice): + // A choice outside the options would reach the model as a value it never offered, which is how a + // mapper that converts between option names and a client's own values would fail silently. + if (choice.Choice is null || !question.Criteria.ContainsKey(choice.Choice)) + { + Throw.InvalidOperationException($"The answer to question '{id}' chose '{choice.Choice}', which is not one of its options."); + } + + EnsureInRange(choice.Confidence, 0, 1, id, "confidence"); + EnsureProbabilities(choice.Probabilities, id, key => question.Criteria.ContainsKey(key), "one of its options"); + break; + + case (JevScoreQuestion question, JevScoreAnswer score): + EnsureInRange(score.Score, 0, question.Criteria.Count - 1, id, "score"); + EnsureInRange(score.Confidence, 0, 1, id, "confidence"); + + // Score answers key their probabilities and legend by level number ("0", "1", ...). + int levels = question.Criteria.Count; + bool IsLevel(string key) => int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out int level) && level < levels; + string levelRange = $"a level number from 0 to {(levels - 1).ToString(CultureInfo.InvariantCulture)}"; + EnsureProbabilities(score.Probabilities, id, IsLevel, levelRange); + EnsureKeys(score.Legend?.Keys, id, "legend", IsLevel, levelRange); + break; + + case (JevNoulQuestion, JevNoulAnswer noul): + EnsureInRange(noul.Noul, 0, 1, id, "noul"); + break; + + default: + Throw.InvalidOperationException($"The answer to question '{id}' is a {KindOf(answer)} answer, but the question is a {KindOf(entry.Value)} question."); + break; + } + } + } + + /// + /// Checks that every probability is in range and is keyed by something the question offered. A mapper that fills + /// probabilities with a client's own labels would otherwise show the model options it never asked about. + /// + private static void EnsureProbabilities(IReadOnlyDictionary? probabilities, string id, Func isValidKey, string expected) + { + if (probabilities is null) + { + Throw.InvalidOperationException($"The answer to question '{id}' has no probabilities."); + } + + EnsureKeys(probabilities.Keys, id, "probabilities", isValidKey, expected); + foreach (KeyValuePair probability in probabilities) + { + EnsureInRange(probability.Value, 0, 1, id, $"probability '{probability.Key}'"); + } + } + + private static void EnsureKeys(IEnumerable? keys, string id, string field, Func isValidKey, string expected) + { + foreach (string key in keys ?? []) + { + if (!isValidKey(key)) + { + Throw.InvalidOperationException($"The answer to question '{id}' has {field} for '{key}', which is not {expected}."); + } + } + } + + private static void EnsureInRange(double value, double min, double max, string id, string field) + { + // NaN and infinity fail these comparisons too; they cannot be written as JSON at all. + if (!(value >= min - Tolerance && value <= max + Tolerance)) + { + Throw.InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "The answer to question '{0}' has {1} {2}, which is outside the range {3} to {4}.", + id, + field, + value, + min, + max)); + } + } + + private static string Count(int count, string noun) => + count == 1 ? $"1 {noun}" : $"{count.ToString(CultureInfo.InvariantCulture)} {noun}s"; + + private static string KindOf(object value) => + value switch + { + JevChoiceQuestion or JevChoiceAnswer => "Choice", + JevScoreQuestion or JevScoreAnswer => "Score", + JevNoulQuestion or JevNoulAnswer => "Noul", + _ => value.GetType().Name, + }; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs new file mode 100644 index 00000000000..02104e5fed6 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Globalization; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Evaluates a with the TypeSafe System One API through a System.ClientModel pipeline. +/// +internal sealed class JevHttpEvaluator +{ + private const string DefaultEndpoint = "https://api.typesafe.ai/v1/systemone"; + private const int MaxErrorDetailLength = 500; + + private readonly ClientPipeline _pipeline; + private readonly Uri _endpoint; + private readonly string _modelId; + + public JevHttpEvaluator(ApiKeyCredential credential, JevAIToolOptions options) + { + this._endpoint = options.Endpoint ?? new Uri(DefaultEndpoint); + this._modelId = options.ModelId; + + // The credential is applied per try, like the OpenAI client does, so a retried request is signed again. + // Creating the pipeline freezes the options, as with any other client built from them. + this._pipeline = ClientPipeline.Create( + options, + perCallPolicies: [], + perTryPolicies: [ApiKeyAuthenticationPolicy.CreateBearerAuthorizationPolicy(credential)], + beforeTransportPolicies: []); + } + + public async Task EvaluateAsync(JevRequest request, IServiceProvider? services, CancellationToken cancellationToken) + { + using PipelineMessage message = this._pipeline.CreateMessage(); + message.ResponseClassifier = JevResponseClassifier.Instance; + message.Apply(new RequestOptions { CancellationToken = cancellationToken }); + + PipelineRequest httpRequest = message.Request; + httpRequest.Method = "POST"; + httpRequest.Uri = this._endpoint; + httpRequest.Headers.Set("Accept", "application/json"); + httpRequest.Headers.Set("Content-Type", "application/json"); + httpRequest.Content = BinaryContent.Create(BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes( + new JevApiRequest { State = request.State, Model = this._modelId, Questions = request.Questions }, + JevJsonContext.Default.JevApiRequest))); + + // Transports leave the cancellation check to the HTTP handler, so a canceled call is stopped here before + // anything is sent. + cancellationToken.ThrowIfCancellationRequested(); + await this._pipeline.SendAsync(message).ConfigureAwait(false); + + PipelineResponse response = message.Response!; + if (response.IsError) + { + // The service detail says which field was rejected, which is what the model needs to correct its + // arguments, but the default ClientResultException message has only the status. The exception still + // carries the response for GetRawResponse, and never the credential. + string body = response.Content.ToString(); + string detail = body.Length > MaxErrorDetailLength ? string.Concat(body.AsSpan(0, MaxErrorDetailLength), "...") : body; + throw new ClientResultException( + string.Format(CultureInfo.InvariantCulture, "The Jev request failed with status {0} ({1}): {2}", response.Status, response.ReasonPhrase, detail), + response); + } + + return ReadResponse(response.Content); + } + + private static JevResponse ReadResponse(BinaryData content) + { + try + { + return JsonSerializer.Deserialize(content.ToMemory().Span, JevJsonContext.Default.JevResponse) + ?? throw new JsonException("The response body is null."); + } + catch (Exception ex) when (ex is JsonException or NotSupportedException) + { + // An answer with an unknown or missing type surfaces as NotSupportedException rather than JsonException. + throw new JsonException($"The Jev API response could not be read: {ex.Message}", ex); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs new file mode 100644 index 00000000000..4eb54bccb97 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Source-generated serialization metadata for the Jev contract types. +/// +/// +/// Models often write the type discriminator after other properties of a question, so out-of-order metadata +/// is allowed; without it, a valid question such as {"instructions": "...", "type": "noul"} would be rejected. +/// +[JsonSourceGenerationOptions(AllowOutOfOrderMetadataProperties = true)] +[JsonSerializable(typeof(JevRequest))] +[JsonSerializable(typeof(JevResponse))] +[JsonSerializable(typeof(JevApiRequest))] +[JsonSerializable(typeof(Dictionary))] +internal sealed partial class JevJsonContext : JsonSerializerContext; diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs new file mode 100644 index 00000000000..8e8ff710492 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization.Metadata; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Serializer options and JSON schemas shared by the Jev tool. +/// +internal static class JevJsonUtilities +{ + private static readonly Lazy s_requestSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevRequest), serializerOptions: Options)); + private static readonly Lazy s_responseSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevResponse), serializerOptions: Options)); + + /// + /// Gets the options that resolve the Jev contract types first and every other type through + /// . + /// + public static JsonSerializerOptions Options { get; } = CreateOptions(); + + /// Gets the JSON schema of the tool arguments, which is the schema of . + public static JsonElement RequestSchema => s_requestSchema.Value; + + /// Gets the JSON schema of the tool result, which is the schema of . + public static JsonElement ResponseSchema => s_responseSchema.Value; + + /// + /// Converts one tool argument into a JSON node. Chat clients pass arguments as values; + /// callers that invoke the function directly may pass .NET objects instead. + /// + public static JsonNode? ToNode(object? value) => + value switch + { + null => null, + JsonNode node => node.DeepClone(), + JsonElement element => element.ValueKind switch + { + JsonValueKind.Object => JsonObject.Create(element), + JsonValueKind.Array => JsonArray.Create(element), + JsonValueKind.Null or JsonValueKind.Undefined => null, + _ => JsonValue.Create(element), + }, + _ => JsonSerializer.SerializeToNode(value, Options.GetTypeInfo(value.GetType())), + }; + + private static JsonSerializerOptions CreateOptions() + { + var options = new JsonSerializerOptions(JevJsonContext.Default.Options) + { + TypeInfoResolver = JsonTypeInfoResolver.Combine(JevJsonContext.Default, AIJsonUtilities.DefaultOptions.TypeInfoResolver), + }; + + options.MakeReadOnly(); + return options; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs new file mode 100644 index 00000000000..adf42d6ac73 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Classifies System One responses: any 2xx status is a success, and 529 is retriable in addition to the statuses +/// that retries (408, 429, 500, 502, 503, and 504). +/// +/// +/// TypeSafe answers 529 when it is overloaded and documents it as transient, but the default classifier treats it as a +/// permanent error. Every other case, including network failures, is left to the default classifier. +/// +internal sealed class JevResponseClassifier : PipelineMessageClassifier +{ + private const int OverloadedStatus = 529; + + public static JevResponseClassifier Instance { get; } = new(); + + public override bool TryClassify(PipelineMessage message, out bool isError) + { + if (message.Response is not { } response) + { + isError = false; + return false; + } + + isError = response.Status is < 200 or >= 300; + return true; + } + + public override bool TryClassify(PipelineMessage message, Exception? exception, out bool isRetriable) + { + isRetriable = exception is null && message.Response?.Status == OverloadedStatus; + return isRetriable; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs new file mode 100644 index 00000000000..b54b892ce09 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The default retry policy of : the standard , except +/// that it never waits longer than . +/// +/// +/// waits for whatever Retry-After the server sends. Inside a tool call that +/// wait blocks the agent run with no feedback, and a very large value makes the delay itself throw, so a longer +/// request fails at once with the service's status instead. +/// +internal sealed class JevRetryPolicy : ClientRetryPolicy +{ + public const int DefaultMaxRetries = 3; + + public static readonly TimeSpan MaxRetryAfter = TimeSpan.FromMinutes(1); + + public JevRetryPolicy(int maxRetries = DefaultMaxRetries, bool enableLogging = true, ILoggerFactory? loggerFactory = null) + : base(maxRetries, enableLogging, loggerFactory) + { + } + + /// + /// Gets or sets a list that, when set, receives each delay instead of waiting it, so tests can check the delays + /// without being slowed down by them. + /// + internal List? RecordedWaitsForTests { get; set; } + + protected override bool ShouldRetry(PipelineMessage message, Exception? exception) => + base.ShouldRetry(message, exception) && IsRetryAfterWithinLimit(message); + + protected override async ValueTask ShouldRetryAsync(PipelineMessage message, Exception? exception) => + await base.ShouldRetryAsync(message, exception).ConfigureAwait(false) && IsRetryAfterWithinLimit(message); + + protected override Task WaitAsync(TimeSpan time, CancellationToken cancellationToken) + { + if (this.RecordedWaitsForTests is { } waits) + { + waits.Add(time); + return Task.CompletedTask; + } + + return base.WaitAsync(time, cancellationToken); + } + + protected override void Wait(TimeSpan time, CancellationToken cancellationToken) + { + if (this.RecordedWaitsForTests is { } waits) + { + waits.Add(time); + return; + } + + base.Wait(time, cancellationToken); + } + + private static bool IsRetryAfterWithinLimit(PipelineMessage message) + { + if (message.Response?.Headers.TryGetValue("Retry-After", out string? value) is not true || string.IsNullOrWhiteSpace(value)) + { + return true; + } + + // Retry-After is either a number of seconds or an HTTP date. + if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds)) + { + return seconds <= MaxRetryAfter.TotalSeconds; + } + + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) + { + return date - DateTimeOffset.UtcNow <= MaxRetryAfter; + } + + return true; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs new file mode 100644 index 00000000000..d1bb421f9c7 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Builds an that lets an agent classify, route, score, and check content with Jev, the +/// TypeSafe System One model. +/// +/// +/// +/// Jev does not write text. It answers typed questions about a state and returns calibrated probabilities, which +/// makes it a fast and inexpensive classifier for an agent whose own model writes the replies. The function created by +/// is a thin wrapper around an evaluator: its arguments are a written by +/// the agent's model, and its result is the evaluator's , as JSON. +/// +/// +/// With an , the evaluator is the TypeSafe System One API, called through a +/// System.ClientModel pipeline configured by . Any other client can be used instead with +/// or one of the UseClient methods. +/// +/// +/// Every evaluator receives the of the agent that calls the tool +/// (, which FunctionInvokingChatClient sets from the services +/// the agent was created with), so it can resolve its dependencies per call. In a host, register the tool with +/// services.AddAIAgent(...).WithAITool(sp => new JevAIToolBuilder(credential, options).Build()). +/// +/// +public sealed class JevAIToolBuilder +{ + private readonly ApiKeyCredential? _credential; + private readonly JevAIToolOptions _options; + private Func>? _evaluateAsync; + + /// + /// Initializes a new instance of the class that evaluates requests with the + /// TypeSafe System One API. + /// + /// The TypeSafe API key, sent as a bearer token. + /// The tool and client pipeline options. Defaults are used when . + public JevAIToolBuilder(ApiKeyCredential credential, JevAIToolOptions? options = null) + { + this._credential = Throw.IfNull(credential); + this._options = options ?? new JevAIToolOptions(); + } + + /// + /// Initializes a new instance of the class for a custom evaluator, which must be set + /// with or a UseClient method before . + /// + /// + /// The tool options. Only and apply + /// to a custom evaluator, which uses its own client configuration. Defaults are used when . + /// + public JevAIToolBuilder(JevAIToolOptions? options = null) + { + this._options = options ?? new JevAIToolOptions(); + } + + /// + /// Sets the evaluator that answers each request, replacing the TypeSafe API. + /// + /// + /// The evaluator. It receives the written by the model and the agent's services, which are + /// when the caller provides none, and returns one answer per question. + /// + /// This builder. + /// + /// The agent's services are often the application's root provider, because hosted agents are singletons by + /// default. To use a scoped service, create a scope inside the evaluator, or use + /// , which does so for every call. + /// + public JevAIToolBuilder UseEvaluator(Func> evaluateAsync) + { + this._evaluateAsync = Throw.IfNull(evaluateAsync); + return this; + } + + /// + /// Evaluates requests with another client, converting between the Jev contract types and the client's own types. + /// + /// The request type of the client. + /// The response type of the client. + /// The client operation that evaluates a request. + /// Converts the written by the model into the client's request. + /// Converts the client's response into a . + /// This builder. + /// + /// The mapped response is checked against the request before it reaches the model: every question needs an answer + /// of the same kind, a Choice answer and its probabilities may only use the question's options, and probabilities + /// and scores must be in range. A mapper mistake therefore fails the call instead of showing the model values it + /// never offered. + /// + public JevAIToolBuilder UseClient( + Func> evaluateAsync, + Func inputMapper, + Func outputMapper) + { + _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(inputMapper); + _ = Throw.IfNull(outputMapper); + + return this.UseEvaluator(async (request, _, cancellationToken) => + { + TClientRequest clientRequest = inputMapper(request); + TClientResponse clientResponse = await evaluateAsync(clientRequest, cancellationToken).ConfigureAwait(false); + return outputMapper(clientResponse); + }); + } + + /// + /// Evaluates requests with a client registered in the agent's services, converting between the Jev contract types + /// and the client's own types. + /// + /// The client type, resolved from the agent's services on every call. + /// The request type of the client. + /// The response type of the client. + /// The client operation that evaluates a request. + /// Converts the written by the model into the client's request. + /// Converts the client's response into a . + /// This builder. + /// + /// The client is resolved from a new service scope on every call, and the scope is disposed when the call ends. + /// Scoped clients therefore live for one call, and disposable transient clients are released after it, even when + /// the agent's services are the application's root provider. The call fails when the agent has no services or + /// is not registered. The mapped response is checked as described for + /// . + /// + public JevAIToolBuilder UseClient( + Func> evaluateAsync, + Func inputMapper, + Func outputMapper) + where TClient : notnull + { + _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(inputMapper); + _ = Throw.IfNull(outputMapper); + + return this.UseEvaluator(async (request, services, cancellationToken) => + { + if (services is null) + { + Throw.InvalidOperationException( + $"The Jev tool resolves {typeof(TClient).Name} from the agent's services, but this call has none. " + + "Create the agent with an IServiceProvider, for example with AddAIAgent, or pass the client operation to UseClient directly."); + } + + // Hosted agents are singletons that receive the root provider by default. Resolving from it directly would + // reject scoped clients (or keep one forever when scope validation is off) and hold every disposable + // transient client until shutdown, so each call gets its own scope. + if (services.GetService() is not { } scopeFactory) + { + return await EvaluateWithClientAsync(services).ConfigureAwait(false); + } + + AsyncServiceScope scope = scopeFactory.CreateAsyncScope(); + await using (scope.ConfigureAwait(false)) + { + return await EvaluateWithClientAsync(scope.ServiceProvider).ConfigureAwait(false); + } + + async Task EvaluateWithClientAsync(IServiceProvider provider) + { + if (provider.GetService(typeof(TClient)) is not TClient client) + { + Throw.InvalidOperationException($"{typeof(TClient).Name} is not registered in the agent's services."); + return default!; + } + + TClientRequest clientRequest = inputMapper(request); + TClientResponse clientResponse = await evaluateAsync(client, clientRequest, cancellationToken).ConfigureAwait(false); + return outputMapper(clientResponse); + } + }); + } + + /// + /// Creates the function. + /// + /// + /// An whose JSON schema is the schema of and whose result is the + /// evaluator's serialized as JSON. + /// + /// + /// The builder has neither an nor a custom evaluator, or an option that it uses is + /// empty or invalid. + /// + /// + /// With the TypeSafe API, building creates the client pipeline, which makes the options read-only. Invalid arguments + /// from the model fail with an that says what to fix; a function-invoking chat client + /// includes that message in the tool result only when its detailed errors are enabled. + /// + public AIFunction Build() + { + if (string.IsNullOrWhiteSpace(this._options.Name) || string.IsNullOrWhiteSpace(this._options.Description)) + { + Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Name)} and {nameof(JevAIToolOptions.Description)} must not be empty."); + } + + Func>? evaluateAsync = this._evaluateAsync; + if (evaluateAsync is null) + { + if (this._credential is null) + { + Throw.InvalidOperationException($"Pass an {nameof(ApiKeyCredential)} to the constructor to use the TypeSafe API, or call {nameof(UseEvaluator)} or UseClient."); + } + + // The model and endpoint only matter to the TypeSafe API; a custom evaluator uses its own configuration. + if (string.IsNullOrWhiteSpace(this._options.ModelId)) + { + Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.ModelId)} must not be empty."); + } + + if (this._options.Endpoint?.IsAbsoluteUri is false) + { + Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Endpoint)} must be an absolute URI."); + } + + evaluateAsync = new JevHttpEvaluator(this._credential, this._options).EvaluateAsync; + } + + return new JevAIFunction(this._options.Name, this._options.Description, evaluateAsync); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs new file mode 100644 index 00000000000..7f08031b762 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Configures the built by and the +/// client pipeline that sends its requests to the TypeSafe System One API. +/// +/// +/// +/// This type follows the pattern of the OpenAI and Azure AI client libraries: +/// selects the HTTP stack (for example an +/// over an HttpClient from IHttpClientFactory), +/// controls retries, and +/// adds custom policies. The pipeline settings, +/// , and apply only when the builder uses the TypeSafe API, that is, when +/// it receives an ; and apply to every +/// evaluator. +/// +/// +/// When is , the pipeline retries up to three +/// times on the statuses that TypeSafe documents as transient (429 and 529, besides the standard 408 and 5xx ones), and +/// fails at once when the API asks for a wait longer than one minute, because the wait would block the agent run +/// without any feedback. That default follows . +/// +/// +/// Like other client options, an instance becomes read-only once a tool has been built with it. +/// +/// +public sealed class JevAIToolOptions : ClientPipelineOptions +{ + private const string DefaultDescription = + "Evaluates content with Jev, a classification model. Jev does not write text: it answers typed questions about the " + + "given state and returns calibrated probabilities. Use it to classify, route, score, or check facts about text or " + + "JSON data, and ask all related questions in one call. A 'choice' question picks exactly one option from its " + + "criteria. A 'score' question rates the state against 2 to 10 ordered levels and can land between levels. A 'noul' " + + "question returns the probability, from 0 to 1, that its yes or no statement is true. Choice and score answers " + + "include a confidence from 0 to 1; treat a low confidence as uncertain."; + + /// + public override void Freeze() + { + // The default policy is assigned here, not in the constructor, because ClientLoggingOptions may be set after + // construction; ClientPipeline.Create freezes the options before it reads RetryPolicy, so the policy is in place. + this.RetryPolicy ??= new JevRetryPolicy( + JevRetryPolicy.DefaultMaxRetries, + this.ClientLoggingOptions?.EnableLogging ?? true, + this.ClientLoggingOptions?.LoggerFactory); + + base.Freeze(); + } + + /// + /// Gets or sets the System One endpoint. When , https://api.typesafe.ai/v1/systemone is used. + /// + public Uri? Endpoint + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } + + /// + /// Gets or sets the Jev model that answers the questions. Defaults to jev-latest. + /// + public string ModelId + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = "jev-latest"; + + /// + /// Gets or sets the name of the function that the model sees. Defaults to evaluate_with_jev. + /// + public string Name + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = "evaluate_with_jev"; + + /// + /// Gets or sets the description of the function that the model sees. + /// + /// + /// The default description explains the Choice, Score, and Noul question kinds. A replacement should explain them + /// too, together with when the agent should call the function. + /// + public string Description + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = DefaultDescription; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs new file mode 100644 index 00000000000..77a1adf5e55 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents Jev's answer to one question of a . +/// +/// +/// The JSON type property matches the kind of the question: choice (), +/// score (), or noul (). +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(JevChoiceAnswer), "choice")] +[JsonDerivedType(typeof(JevScoreAnswer), "score")] +[JsonDerivedType(typeof(JevNoulAnswer), "noul")] +public abstract class JevAnswer +{ + // Only the three answer kinds that the Jev API returns can exist. + private protected JevAnswer() + { + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs new file mode 100644 index 00000000000..9af3f283071 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a . +/// +[Description("The answer to a Choice question.")] +public sealed class JevChoiceAnswer : JevAnswer +{ + /// + /// Gets or sets the option with the highest probability. + /// + [JsonPropertyName("choice")] + [Description("The option with the highest probability.")] + public required string Choice { get; set; } + + /// + /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the options. + /// + [JsonPropertyName("confidence")] + [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the probability of each option. The probabilities sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("The probability of each option. The probabilities sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs new file mode 100644 index 00000000000..7377a778036 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Choice question, which picks exactly one option from a set that the caller defines. +/// +/// +/// The answer is a . +/// +[Description("A Choice question: picks exactly one option from criteria.")] +public sealed class JevChoiceQuestion : JevQuestion +{ + /// + /// Gets or sets the options, keyed by the option name that the answer returns. + /// + /// + /// Each value describes when the option applies, or is when the name is enough. + /// Jev accepts from 1 to 255 options. + /// + [JsonPropertyName("criteria")] + [Description("The options to choose from, keyed by the option name that the answer returns. Each value describes when the option applies. From 1 to 255 options.")] + public required IReadOnlyDictionary Criteria { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs new file mode 100644 index 00000000000..b737c69c977 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a . +/// +[Description("The answer to a Noul question.")] +public sealed class JevNoulAnswer : JevAnswer +{ + /// + /// Gets or sets the probability, from 0 to 1, that the answer is yes. + /// + [JsonPropertyName("noul")] + [Description("The probability, from 0 to 1, that the answer is yes.")] + public required double Noul { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs new file mode 100644 index 00000000000..4540e047ec0 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Describes what a yes and a no mean for a . +/// +public sealed class JevNoulCriteria +{ + /// + /// Gets or sets what a yes, a probability near 1, means. + /// + [JsonPropertyName("true")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("What a yes, a probability near 1, means.")] + public string? True { get; set; } + + /// + /// Gets or sets what a no, a probability near 0, means. + /// + [JsonPropertyName("false")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("What a no, a probability near 0, means.")] + public string? False { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs new file mode 100644 index 00000000000..bee933254dd --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Noul question, which returns the probability that a yes or no statement is true. +/// +/// +/// The answer is a . +/// +[Description("A Noul question: returns the probability, from 0 to 1, that the yes or no statement in instructions is true.")] +public sealed class JevNoulQuestion : JevQuestion +{ + /// + /// Gets or sets optional descriptions of what a yes and a no mean. + /// + [JsonPropertyName("criteria")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("Optional descriptions of what a yes and a no mean.")] + public JevNoulCriteria? Criteria { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs new file mode 100644 index 00000000000..af3fe909da1 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents one typed question that Jev answers about the state of a . +/// +/// +/// The JSON type property selects the question kind: choice (), +/// score (), or noul (). +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(JevChoiceQuestion), "choice")] +[JsonDerivedType(typeof(JevScoreQuestion), "score")] +[JsonDerivedType(typeof(JevNoulQuestion), "noul")] +public abstract class JevQuestion +{ + // Only the three question kinds that the Jev API accepts can exist. + private protected JevQuestion() + { + } + + /// + /// Gets or sets what Jev should decide, rate, or check. + /// + [JsonPropertyName("instructions")] + [Description("What Jev should decide, rate, or check. Describe the idea rather than exact words, because Jev matches on meaning.")] + public required string Instructions { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs new file mode 100644 index 00000000000..2899a89ccf8 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the input of a Jev evaluation: the state to evaluate and the typed questions to answer about it. +/// +/// +/// This type mirrors the body of a TypeSafe System One API request without the model, which the tool configuration +/// selects. Its JSON form is the argument shape of the that +/// creates. +/// +public sealed class JevRequest +{ + /// + /// Gets or sets the content to evaluate. + /// + /// + /// A JSON string for plain text, or a JSON object or array for structured data, such as a conversation or + /// related records. + /// + [JsonPropertyName("state")] + [Description("The content to evaluate. Use a string for plain text, or a JSON object or array for structured data. Questions can refer to object fields by name in backticks, for example `order`.")] + public required JsonElement State { get; set; } + + /// + /// Gets or sets the questions to answer, keyed by an ID that the caller chooses. + /// + /// + /// Each answer in uses the ID of its question. + /// + [JsonPropertyName("questions")] + [Description("The questions to answer about the state, keyed by an ID you choose. Each answer comes back under the same ID. Ask all related questions in one call.")] + public required IReadOnlyDictionary Questions { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs new file mode 100644 index 00000000000..944240124d8 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the result of a Jev evaluation: one typed answer per question of the . +/// +/// +/// This type mirrors the body of a TypeSafe System One API response. Its JSON form is the result of the +/// that creates. +/// +public sealed class JevResponse +{ + /// + /// Gets or sets the model that performed the evaluation, when the evaluator reports it. + /// + [JsonPropertyName("model")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("The model that performed the evaluation.")] + public string? Model { get; set; } + + /// + /// Gets or sets the answers, keyed by the ID of the question they answer. + /// + [JsonPropertyName("answers")] + [Description("One answer per question, keyed by the question ID.")] + public required IReadOnlyDictionary Answers { get; set; } + + /// + /// Gets or sets the token usage of the evaluation, when the evaluator reports it. + /// + [JsonPropertyName("usage")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("The token usage of the evaluation.")] + public JevUsage? Usage { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs new file mode 100644 index 00000000000..a4f4be565b0 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a . +/// +[Description("The answer to a Score question.")] +public sealed class JevScoreAnswer : JevAnswer +{ + /// + /// Gets or sets the score, weighted by the probability of each level. It can fall between levels. + /// + [JsonPropertyName("score")] + [Description("The score, weighted by the probability of each level. It can fall between levels.")] + public required double Score { get; set; } + + /// + /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the levels. + /// + [JsonPropertyName("confidence")] + [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the description of each level, keyed by the level number. + /// + [JsonPropertyName("legend")] + [Description("The description of each level, keyed by the level number.")] + public required IReadOnlyDictionary Legend { get; set; } + + /// + /// Gets or sets the probability of each level, keyed by the level number. The probabilities sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("The probability of each level, keyed by the level number. The probabilities sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs new file mode 100644 index 00000000000..fa8a7954b1a --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Score question, which rates the state against ordered levels that the caller defines. +/// +/// +/// The answer is a , whose score is weighted by probability and can fall between levels. +/// +[Description("A Score question: rates the state against ordered levels in criteria.")] +public sealed class JevScoreQuestion : JevQuestion +{ + /// + /// Gets or sets the level descriptions, ordered from level 0 (the lowest) to the highest level. + /// + /// + /// A Score needs from 2 to 10 levels. + /// + [JsonPropertyName("criteria")] + [Description("The level descriptions, ordered from level 0 (the lowest) to the highest level. From 2 to 10 levels.")] + public required IReadOnlyList Criteria { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs new file mode 100644 index 00000000000..1c004164070 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the token usage of a Jev evaluation. +/// +public sealed class JevUsage +{ + /// + /// Gets or sets the number of input tokens. + /// + [JsonPropertyName("input_tokens")] + [Description("The number of input tokens.")] + public required long InputTokens { get; set; } + + /// + /// Gets or sets the number of output tokens. + /// + [JsonPropertyName("output_tokens")] + [Description("The number of output tokens.")] + public required long OutputTokens { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj new file mode 100644 index 00000000000..203d77b4310 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -0,0 +1,31 @@ + + + + $(TargetFrameworksCore) + Microsoft.Agents.AI.TypeSafe + alpha + + + + true + + + + + + + Microsoft Agent Framework - TypeSafe (Jev) integration + Provides an AIFunction that lets agents classify, route, score, and check content with Jev, the TypeSafe System One model. + + + + + + + + + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs new file mode 100644 index 00000000000..9135880f01b --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs @@ -0,0 +1,463 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevAIFunctionTests +{ + [Fact] + public async Task InvokeAsync_ModelArguments_ReachTheEvaluatorAsTypedRequestAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + { + received = request; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.NotNull(received); + Assert.Equal(JevTestData.TicketText, received.State.GetString()); + + JevChoiceQuestion department = Assert.IsType(received.Questions["department"]); + Assert.Equal("Which team should handle this", department.Instructions); + Assert.Equal(["billing", "technical", "sales"], department.Criteria.Keys); + Assert.Null(department.Criteria["sales"]); + + JevScoreQuestion frustration = Assert.IsType(received.Questions["frustration"]); + Assert.Equal(3, frustration.Criteria.Count); + Assert.Equal("Calm, just stating facts", frustration.Criteria[0]); + + JevNoulQuestion urgent = Assert.IsType(received.Questions["is_urgent"]); + Assert.Equal("Explicitly time-sensitive", urgent.Criteria!.True); + Assert.Equal("No urgency expressed", urgent.Criteria.False); + } + + [Fact] + public async Task InvokeAsync_ReturnsTheResponseAsJsonWithWireNamesAsync() + { + // Arrange + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Act + object? result = await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + JsonElement json = Assert.IsType(result); + Assert.Equal("custom-model", json.GetProperty("model").GetString()); + JsonElement answers = json.GetProperty("answers"); + Assert.Equal("choice", answers.GetProperty("department").GetProperty("type").GetString()); + Assert.Equal("technical", answers.GetProperty("department").GetProperty("choice").GetString()); + Assert.Equal(0.78, answers.GetProperty("department").GetProperty("confidence").GetDouble()); + Assert.Equal("score", answers.GetProperty("frustration").GetProperty("type").GetString()); + Assert.Equal("Frustrated", answers.GetProperty("frustration").GetProperty("legend").GetProperty("1").GetString()); + Assert.Equal(0.97, answers.GetProperty("is_urgent").GetProperty("noul").GetDouble()); + Assert.Equal(10, json.GetProperty("usage").GetProperty("input_tokens").GetInt64()); + Assert.Equal(5, json.GetProperty("usage").GetProperty("output_tokens").GetInt64()); + } + + [Fact] + public async Task InvokeAsync_TypeWrittenLastAndObjectState_AreAcceptedAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + { + received = request; + return Task.FromResult(new JevResponse { Answers = new Dictionary { ["paid"] = new JevNoulAnswer { Noul = 0.1 } } }); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(""" + { + "questions": { "paid": { "instructions": "Was `order` paid?", "type": "noul" } }, + "state": { "order": { "id": "A-104", "status": "failed" } } + } + """)); + + // Assert + Assert.Equal(JsonValueKind.Object, received!.State.ValueKind); + Assert.Equal("A-104", received.State.GetProperty("order").GetProperty("id").GetString()); + Assert.IsType(received.Questions["paid"]); + } + + [Fact] + public async Task InvokeAsync_DotNetObjectArguments_AreAcceptedAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + { + received = request; + return Task.FromResult(new JevResponse { Answers = new Dictionary { ["q"] = new JevNoulAnswer { Noul = 0.5 } } }); + }).Build(); + + // Act + await tool.InvokeAsync(new AIFunctionArguments + { + ["state"] = "The payment failed.", + ["questions"] = new Dictionary { ["q"] = new JevNoulQuestion { Instructions = "Did the payment fail?" } }, + }); + + // Assert + Assert.Equal("The payment failed.", received!.State.GetString()); + Assert.Equal("Did the payment fail?", Assert.IsType(received.Questions["q"]).Instructions); + } + + [Theory] + [InlineData("""{"questions":{"q":{"type":"noul","instructions":"x"}}}""", "state")] + [InlineData("""{"state":42,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "The state must be a string")] + [InlineData("""{"state":"x","questions":{}}""", "at least one question")] + [InlineData("""{"state":"x","questions":{"q":{"type":"rank","instructions":"x"}}}""", "not a valid Jev request")] + [InlineData("""{"state":"x","questions":{"q":{"instructions":"x"}}}""", "not a valid Jev request")] + [InlineData("""{"state":"x","questions":{"q":null}}""", "Question 'q' is null")] + [InlineData("""{"state":"x","questions":{"q":{"type":"noul","instructions":" "}}}""", "Question 'q' must have instructions")] + [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{}}}}""", "has 0 options; it needs from 1 to 255")] + [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{"":"empty"}}}}""", "empty option name")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["only"]}}}""", "has 1 level; it needs from 2 to 10")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["0","1","2","3","4","5","6","7","8","9","10"]}}}""", "has 11 levels")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["low",""]}}}""", "empty level description")] + public async Task InvokeAsync_InvalidArguments_FailBeforeEvaluatingAsync(string arguments, string expectedMessage) + { + // Arrange + int calls = 0; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => + { + calls++; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + // Act + ArgumentException exception = await Assert.ThrowsAnyAsync(() => tool.InvokeAsync(JevTestData.Arguments(arguments)).AsTask()); + + // Assert + Assert.Contains(expectedMessage, exception.Message, StringComparison.Ordinal); + Assert.Equal(0, calls); + } + + [Fact] + public async Task InvokeAsync_TooManyChoiceOptions_FailBeforeEvaluatingAsync() + { + // Arrange + string options = string.Join(",", Enumerable.Range(0, 256).Select(static i => $"\"o{i}\":null")); + string arguments = "{\"state\":\"x\",\"questions\":{\"q\":{\"type\":\"choice\",\"instructions\":\"x\",\"criteria\":{" + options + "}}}}"; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Act + ArgumentException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(arguments)).AsTask()); + + // Assert + Assert.Contains("has 256 options", exception.Message, StringComparison.Ordinal); + } + + public static TheoryData, string> InvalidResponses => new() + { + { "no answer", static r => Without(r, "is_urgent"), "no answer for question 'is_urgent'" }, + { "wrong kind", static r => With(r, "department", new JevNoulAnswer { Noul = 0.5 }), "is a Noul answer, but the question is a Choice question" }, + { "unknown option", static r => With(r, "department", Choice("refunds", 0.9)), "chose 'refunds', which is not one of its options" }, + { "confidence out of range", static r => With(r, "department", Choice("technical", 1.2)), "has confidence 1.2, which is outside the range 0 to 1" }, + { "score out of range", static r => With(r, "frustration", Score(3)), "has score 3, which is outside the range 0 to 2" }, + { "noul as percent", static r => With(r, "is_urgent", new JevNoulAnswer { Noul = 97 }), "has noul 97, which is outside the range 0 to 1" }, + { "no answers", static r => new JevResponse { Answers = null! }, "has no answers" }, + { "probability for an option not offered", static r => With(r, "department", ChoiceWith("technical", new() { ["TECHNICAL"] = 1 })), "has probabilities for 'TECHNICAL', which is not one of its options" }, + { "probability above one", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = 42 })), "has probability 'technical' 42, which is outside the range 0 to 1" }, + { "probability not a number", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = double.NaN })), "has probability 'technical' NaN" }, + { "score probability for a missing level", static r => With(r, "frustration", ScoreWith(new() { ["3"] = 1 }, [])), "has probabilities for '3', which is not a level number from 0 to 2" }, + { "legend for a missing level", static r => With(r, "frustration", ScoreWith([], new() { ["high"] = "Very angry" })), "has legend for 'high', which is not a level number from 0 to 2" }, + }; + + [Theory] + [MemberData(nameof(InvalidResponses))] + public async Task InvokeAsync_InvalidEvaluatorResponse_FailsAsync(string scenario, Func corrupt, string expectedMessage) + { + // Arrange + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(corrupt(JevTestData.CreateResponse()))).Build(); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.True(exception.Message.Contains(expectedMessage, StringComparison.Ordinal), $"{scenario}: {exception.Message}"); + } + + [Fact] + public async Task InvokeAsync_EvaluatorReturnsNull_FailsAsync() + { + // Arrange + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(null!)).Build(); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Contains("returned no response", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task InvokeAsync_PassesTheCancellationTokenToTheEvaluatorAsync() + { + // Arrange + using var cancellation = new CancellationTokenSource(); + CancellationToken received = default; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, cancellationToken) => + { + received = cancellationToken; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson), cancellation.Token); + + // Assert + Assert.Equal(cancellation.Token, received); + } + + [Fact] + public async Task UseClient_MapsBetweenTheContractAndTheClientTypesAsync() + { + // Arrange + var client = new LabelClient(label: "technical"); + AIFunction tool = new JevAIToolBuilder().UseClient( + client.ClassifyAsync, + inputMapper: static request => + { + (string id, JevQuestion question) = request.Questions.Single(); + return new LabelRequest(id, request.State.GetString()!, ((JevChoiceQuestion)question).Criteria.Keys.ToList()); + }, + outputMapper: static response => new JevResponse + { + Answers = new Dictionary + { + [response.QuestionId] = Choice(response.Label, response.Score), + }, + }).Build(); + + // Act + object? result = await tool.InvokeAsync(JevTestData.Arguments(""" + {"state":"My integration keeps failing","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """)); + + // Assert + Assert.Equal(new LabelRequest("team", "My integration keeps failing", ["billing", "technical"]), client.Received, LabelRequestComparer.Instance); + JsonElement answer = Assert.IsType(result).GetProperty("answers").GetProperty("team"); + Assert.Equal("technical", answer.GetProperty("choice").GetString()); + } + + [Fact] + public async Task UseClient_MapperReturnsAnOptionThatWasNotOffered_FailsAsync() + { + // Arrange + var client = new LabelClient(label: "TECHNICAL"); + AIFunction tool = new JevAIToolBuilder().UseClient( + client.ClassifyAsync, + static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), + static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(""" + {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """)).AsTask()); + + // Assert + Assert.Contains("chose 'TECHNICAL'", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task UseEvaluator_ReceivesTheServicesOfTheCallAsync() + { + // Arrange + using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); + IServiceProvider? received = null; + AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, callServices, _) => + { + received = callServices; + return Task.FromResult(JevTestData.CreateResponse()); + }).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); + arguments.Services = services; + + // Act + await tool.InvokeAsync(arguments); + + // Assert + Assert.Same(services, received); + } + + [Fact] + public async Task UseClient_RegisteredClient_IsResolvedFromTheServicesOnEveryCallAsync() + { + // Arrange + int created = 0; + using ServiceProvider services = new ServiceCollection() + .AddTransient(_ => + { + created++; + return new LabelClient(label: "technical"); + }) + .BuildServiceProvider(); + + AIFunction tool = new JevAIToolBuilder().UseClient( + static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), + static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), + static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(""" + {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """); + arguments.Services = services; + + // Act + await tool.InvokeAsync(arguments); + object? result = await tool.InvokeAsync(arguments); + + // Assert + Assert.Equal(2, created); + Assert.Equal("technical", Assert.IsType(result).GetProperty("answers").GetProperty("team").GetProperty("choice").GetString()); + } + + [Theory] + [InlineData(false, "this call has none")] + [InlineData(true, "LabelClient is not registered")] + public async Task UseClient_ClientNotAvailable_FailsWithWhatIsMissingAsync(bool withServices, string expectedMessage) + { + // Arrange + using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); + AIFunction tool = new JevAIToolBuilder().UseClient( + static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), + static request => new LabelRequest("q", "x", []), + static response => JevTestData.CreateResponse()).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); + arguments.Services = withServices ? services : null; + + // Act + InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(arguments).AsTask()); + + // Assert + Assert.Contains(expectedMessage, exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsync() + { + // Arrange + var created = new List(); + using ServiceProvider root = new ServiceCollection() + .AddScoped(_ => + { + var client = new ScopedLabelClient(); + created.Add(client); + return client; + }) + .BuildServiceProvider(new ServiceProviderOptions { ValidateScopes = true }); + + AIFunction tool = new JevAIToolBuilder().UseClient( + static (client, text, _) => Task.FromResult(client.IsDisposed ? "disposed" : "technical"), + static request => request.State.GetString()!, + static label => new JevResponse { Answers = new Dictionary { ["team"] = Choice(label, 0.9) } }).Build(); + + AIFunctionArguments arguments = JevTestData.Arguments(""" + {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} + """); + arguments.Services = root; + + // Act + await tool.InvokeAsync(arguments); + await tool.InvokeAsync(arguments); + + // Assert + Assert.Equal(2, created.Count); + Assert.All(created, static client => Assert.True(client.IsDisposed)); + } + + private static JevChoiceAnswer Choice(string choice, double confidence) => new() + { + Choice = choice, + Confidence = confidence, + Probabilities = new Dictionary { [choice] = confidence }, + }; + + private static JevScoreAnswer Score(double score) => new() + { + Score = score, + Confidence = 1, + Legend = new Dictionary(), + Probabilities = new Dictionary(), + }; + + private static JevChoiceAnswer ChoiceWith(string choice, Dictionary probabilities) => new() + { + Choice = choice, + Confidence = 0.5, + Probabilities = probabilities, + }; + + private static JevScoreAnswer ScoreWith(Dictionary probabilities, Dictionary legend) => new() + { + Score = 1, + Confidence = 1, + Legend = legend, + Probabilities = probabilities, + }; + + private static JevResponse With(JevResponse response, string id, JevAnswer answer) + { + var answers = new Dictionary(response.Answers) { [id] = answer }; + return new JevResponse { Answers = answers }; + } + + private static JevResponse Without(JevResponse response, string id) + { + var answers = new Dictionary(response.Answers); + answers.Remove(id); + return new JevResponse { Answers = answers }; + } + + /// The request type of a hypothetical third-party classification client. + private sealed record LabelRequest(string QuestionId, string Text, List Labels); + + /// The response type of a hypothetical third-party classification client. + private sealed record LabelResponse(string QuestionId, string Label, double Score); + + private sealed class LabelClient(string label) + { + public LabelRequest? Received { get; private set; } + + public Task ClassifyAsync(LabelRequest request, CancellationToken cancellationToken) + { + this.Received = request; + return Task.FromResult(new LabelResponse(request.QuestionId, label, 0.9)); + } + } + + /// A disposable client registered as scoped, as clients holding per-request resources usually are. + private sealed class ScopedLabelClient : IDisposable + { + public bool IsDisposed { get; private set; } + + public void Dispose() => this.IsDisposed = true; + } + + private sealed class LabelRequestComparer : IEqualityComparer + { + public static LabelRequestComparer Instance { get; } = new(); + + public bool Equals(LabelRequest? x, LabelRequest? y) => + x is not null && y is not null && x.QuestionId == y.QuestionId && x.Text == y.Text && x.Labels.SequenceEqual(y.Labels); + + public int GetHashCode(LabelRequest? obj) => obj?.QuestionId.GetHashCode(StringComparison.Ordinal) ?? 0; + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs new file mode 100644 index 00000000000..1fd41c9ec41 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevAIToolBuilderTests +{ + [Fact] + public void Build_Defaults_DescribeTheJevContract() + { + // Act + AIFunction tool = new JevAIToolBuilder(new ApiKeyCredential("test-key")).Build(); + + // Assert + Assert.Equal("evaluate_with_jev", tool.Name); + Assert.Contains("'choice'", tool.Description, StringComparison.Ordinal); + Assert.Contains("'score'", tool.Description, StringComparison.Ordinal); + Assert.Contains("'noul'", tool.Description, StringComparison.Ordinal); + + JsonElement schema = tool.JsonSchema; + Assert.Equal(["state", "questions"], schema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + JsonElement question = schema.GetProperty("properties").GetProperty("questions").GetProperty("additionalProperties"); + Assert.Equal( + ["choice", "score", "noul"], + question.GetProperty("anyOf").EnumerateArray().Select(static b => b.GetProperty("properties").GetProperty("type").GetProperty("const").GetString())); + + JsonElement returnSchema = tool.ReturnJsonSchema!.Value; + Assert.Equal(["answers"], returnSchema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + Assert.True(returnSchema.GetProperty("properties").GetProperty("usage").GetProperty("properties").TryGetProperty("input_tokens", out _)); + } + + [Fact] + public void Options_Defaults_MatchTheTypeSafeApi() + { + // Act + var options = new JevAIToolOptions(); + + // Assert + Assert.Null(options.Endpoint); + Assert.Equal("jev-latest", options.ModelId); + Assert.Equal("evaluate_with_jev", options.Name); + Assert.Null(options.RetryPolicy); + } + + [Fact] + public void Build_WithTheTypeSafeApi_AssignsTheDefaultRetryPolicyOnlyWhenNoneIsSet() + { + // Arrange + var defaults = new JevAIToolOptions(); + var custom = new JevAIToolOptions { RetryPolicy = new System.ClientModel.Primitives.ClientRetryPolicy(maxRetries: 1) }; + + // Act + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), defaults).Build(); + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), custom).Build(); + + // Assert + Assert.IsType(defaults.RetryPolicy); + Assert.IsType(custom.RetryPolicy); + } + + [Fact] + public void Build_WithACustomEvaluator_IgnoresTheApiOnlyOptions() + { + // Arrange + var options = new JevAIToolOptions { ModelId = " ", Endpoint = new Uri("/relative", UriKind.Relative) }; + + // Act + AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Assert + Assert.Equal("evaluate_with_jev", tool.Name); + } + + [Fact] + public void Build_WithNameAndDescription_UsesTheOptions() + { + // Arrange + var options = new JevAIToolOptions { Name = "triage_ticket", Description = "Classifies support tickets." }; + + // Act + AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Assert + Assert.Equal("triage_ticket", tool.Name); + Assert.Equal("Classifies support tickets.", tool.Description); + } + + [Fact] + public void Build_WithTheTypeSafeApi_MakesTheOptionsReadOnly() + { + // Arrange + var options = new JevAIToolOptions(); + + // Act + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).Build(); + + // Assert + Assert.Throws(() => options.Endpoint = new Uri("https://proxy.contoso.com/jev")); + Assert.Throws(() => options.ModelId = "jev-1.13"); + } + + [Fact] + public void Build_WithACustomEvaluator_LeavesTheOptionsWritable() + { + // Arrange + var options = new JevAIToolOptions(); + + // Act + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + + // Assert + options.ModelId = "jev-1.13"; + Assert.Equal("jev-1.13", options.ModelId); + } + + [Fact] + public void Build_WithoutCredentialOrEvaluator_Throws() + { + // Act + InvalidOperationException exception = Assert.Throws(() => new JevAIToolBuilder().Build()); + + // Assert + Assert.Contains("ApiKeyCredential", exception.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData("name")] + [InlineData("description")] + [InlineData("model")] + [InlineData("endpoint")] + public void Build_InvalidOptions_Throw(string invalid) + { + // Arrange + var options = new JevAIToolOptions(); + switch (invalid) + { + case "name": options.Name = " "; break; + case "description": options.Description = ""; break; + case "model": options.ModelId = " "; break; + case "endpoint": options.Endpoint = new Uri("/v1/systemone", UriKind.Relative); break; + } + + // Act & Assert + Assert.Throws(() => new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).Build()); + } + + [Fact] + public void Methods_NullArguments_Throw() + { + // Arrange + var builder = new JevAIToolBuilder(); + Task evaluateAsync(string request, CancellationToken _) => Task.FromResult(request); + string input(JevRequest _) => "request"; + JevResponse output(string _) => JevTestData.CreateResponse(); + Task evaluateWithClientAsync(object _1, string request, CancellationToken _2) => Task.FromResult(request); + + // Act & Assert + Assert.Throws(() => new JevAIToolBuilder((ApiKeyCredential)null!)); + Assert.Throws(() => builder.UseEvaluator(null!)); + Assert.Throws(() => builder.UseClient(null!, input, (Func)output)); + Assert.Throws(() => builder.UseClient((Func>)evaluateAsync, null!, output)); + Assert.Throws(() => builder.UseClient(evaluateAsync, input, null!)); + Assert.Throws(() => builder.UseClient((Func>)evaluateWithClientAsync, null!, output)); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs new file mode 100644 index 00000000000..4a3582d140f --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevAgentIntegrationTests +{ + [Fact] + public async Task RunAsync_AgentCallsTheTool_ModelReceivesTheAnswersAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + AIFunction jev = new JevAIToolBuilder( + new ApiKeyCredential("test-key"), + new JevAIToolOptions { Transport = new HttpClientPipelineTransport(httpClient) }).Build(); + + using var model = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", jev.Name, JevTestData.CallArguments(JevTestData.RequestJson))])), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Routed to the technical team as urgent."))); + AIAgent agent = model.AsAIAgent(instructions: "Triage support tickets with the Jev tool.", tools: [jev]); + + // Act + AgentResponse response = await agent.RunAsync(JevTestData.TicketText); + + // Assert + Assert.Equal("Routed to the technical team as urgent.", response.Text); + Assert.Single(handler.Requests); + + FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); + Assert.Null(result.Exception); + + // Chat clients serialize tool results with AIJsonUtilities.DefaultOptions before sending them to the model. + string sent = JsonSerializer.Serialize(result.Result, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(JevTestData.ResponseJson), JsonNode.Parse(sent)), sent); + } + + [Fact] + public async Task GetResponseAsync_InvalidArgumentsWithDetailedErrors_TellTheModelWhatToFixAsync() + { + // Arrange + AIFunction jev = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + const string InvalidArguments = """{"state":"x","questions":{"q":{"type":"score","instructions":"How bad?","criteria":["only"]}}}"""; + + using var model = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", jev.Name, JevTestData.CallArguments(InvalidArguments))])), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "done"))); + using var client = new FunctionInvokingChatClient(model) { IncludeDetailedErrors = true }; + + // Act + await client.GetResponseAsync("How bad is it?", new ChatOptions { Tools = [jev] }); + + // Assert + FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); + Assert.Contains("has 1 level; it needs from 2 to 10", result.Result?.ToString(), StringComparison.Ordinal); + } + + [Fact] + public async Task RunAsync_AgentWithServices_ToolResolvesTheClientFromThemAsync() + { + // Arrange + var classifier = new UrgencyClassifier(probability: 0.93); + using ServiceProvider services = new ServiceCollection().AddSingleton(classifier).BuildServiceProvider(); + + AIFunction jev = new JevAIToolBuilder() + .UseClient( + static (client, text, cancellationToken) => client.ScoreAsync(text, cancellationToken), + inputMapper: static request => request.State.GetString()!, + outputMapper: static probability => new JevResponse + { + Answers = new Dictionary { ["is_urgent"] = new JevNoulAnswer { Noul = probability } }, + }) + .Build(); + + const string Arguments = """{"state":"Please help ASAP","questions":{"is_urgent":{"type":"noul","instructions":"Is this urgent?"}}}"""; + using var model = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", jev.Name, JevTestData.CallArguments(Arguments))])), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Urgent."))); + AIAgent agent = model.AsAIAgent(tools: [jev], services: services); + + // Act + AgentResponse response = await agent.RunAsync("Please help ASAP"); + + // Assert + Assert.Equal("Urgent.", response.Text); + Assert.Equal("Please help ASAP", classifier.Received); + FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); + Assert.Equal(0.93, Assert.IsType(result.Result).GetProperty("answers").GetProperty("is_urgent").GetProperty("noul").GetDouble()); + } + + /// A hypothetical third-party client registered in the application's services. + private sealed class UrgencyClassifier(double probability) + { + public string? Received { get; private set; } + + public Task ScoreAsync(string text, CancellationToken cancellationToken) + { + this.Received = text; + return Task.FromResult(probability); + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs new file mode 100644 index 00000000000..7ec9e0ab56c --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevHttpEvaluatorTests +{ + private const string ApiKey = "test-key-9f2c"; + + [Fact] + public async Task InvokeAsync_PostsTheWireRequestAndReturnsTheResponseAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + AIFunction tool = CreateBuilder(httpClient).Build(); + + // Act + object? result = await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(HttpMethod.Post, request.Method); + Assert.Equal(new Uri("https://api.typesafe.ai/v1/systemone"), request.Uri); + Assert.Equal("Bearer", request.Authorization!.Scheme); + Assert.Equal(ApiKey, request.Authorization.Parameter); + + JsonObject expected = JsonNode.Parse(JevTestData.RequestJson)!.AsObject(); + expected["model"] = "jev-latest"; + Assert.True(JsonNode.DeepEquals(expected, JsonNode.Parse(request.Body)), request.Body); + + JsonElement json = Assert.IsType(result); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(JevTestData.ResponseJson), JsonNode.Parse(json.GetRawText())), json.GetRawText()); + } + + [Fact] + public async Task InvokeAsync_CustomModelAndEndpoint_AreUsedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + var options = new JevAIToolOptions { ModelId = "jev-1.13", Endpoint = new Uri("https://proxy.contoso.com/jev") }; + + // Act + await CreateBuilder(httpClient, options).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(new Uri("https://proxy.contoso.com/jev"), request.Uri); + Assert.Equal("jev-1.13", (string?)JsonNode.Parse(request.Body)!["model"]); + } + + [Fact] + public async Task InvokeAsync_CustomPolicy_RunsInThePipelineAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + var options = new JevAIToolOptions(); + options.AddPolicy(new HeaderPolicy("x-tenant", "contoso"), PipelinePosition.PerCall); + + // Act + await CreateBuilder(httpClient, options).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal("contoso", Assert.Single(handler.Requests).Headers["x-tenant"]); + } + + [Fact] + public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, "{}") + .Reply((HttpStatusCode)529, "{}") + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + object? result = await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(3, handler.Requests.Count); + Assert.Equal(2, waits.Count); + Assert.Equal("technical", Assert.IsType(result).GetProperty("answers").GetProperty("department").GetProperty("choice").GetString()); + } + + [Fact] + public async Task InvokeAsync_StillRateLimitedAfterRetries_FailsWithTheStatusAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.TooManyRequests, """{"detail":"Rate limit exceeded"}"""); + using var httpClient = new HttpClient(handler); + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal(429, exception.Status); + Assert.Equal(4, handler.Requests.Count); + } + + [Fact] + public async Task InvokeAsync_RetryAfterSeconds_IsWaitedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, "{}", new RetryConditionHeaderValue(TimeSpan.FromSeconds(5))) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(2, handler.Requests.Count); + Assert.Equal(TimeSpan.FromSeconds(5), Assert.Single(waits)); + } + + [Fact] + public async Task InvokeAsync_RetryAfterDate_IsWaitedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply((HttpStatusCode)529, "{}", new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddSeconds(20))) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(2, handler.Requests.Count); + Assert.InRange(Assert.Single(waits), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(21)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task InvokeAsync_RetryAfterBeyondTheLimit_FailsAtOnceAsync(bool asDate) + { + // Arrange + RetryConditionHeaderValue retryAfter = asDate + ? new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddHours(1)) + : new RetryConditionHeaderValue(TimeSpan.FromHours(1)); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.TooManyRequests, """{"detail":"Slow down"}""", retryAfter); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal(429, exception.Status); + Assert.Single(handler.Requests); + Assert.Empty(waits); + } + + [Fact] + public async Task InvokeAsync_ReplacedRetryPolicy_IsUsedAndStillRetriesOverloadedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply((HttpStatusCode)529, "{}") + .Reply((HttpStatusCode)529, "{}"); + using var httpClient = new HttpClient(handler); + var options = new JevAIToolOptions + { + Transport = new HttpClientPipelineTransport(httpClient), + RetryPolicy = new NoWaitRetryPolicy(maxRetries: 1), + }; + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + new JevAIToolBuilder(new ApiKeyCredential(ApiKey), options).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal(529, exception.Status); + Assert.Equal(2, handler.Requests.Count); + } + + [Theory] + [InlineData(HttpStatusCode.Unauthorized, """{"detail":"Invalid API key"}""", "Invalid API key")] + [InlineData(HttpStatusCode.BadRequest, """{"detail":"Too many score levels. Must have at most 10 levels."}""", "Too many score levels")] + [InlineData(HttpStatusCode.UnprocessableEntity, """{"detail":[{"loc":["body","questions"],"msg":"Dictionary should have at least 1 item"}]}""", "Dictionary should have at least 1 item")] + public async Task InvokeAsync_ApiError_FailsWithTheDetailButNotTheKeyAsync(HttpStatusCode status, string body, string expectedMessage) + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(status, body); + using var httpClient = new HttpClient(handler); + + // Act + ClientResultException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.Equal((int)status, exception.Status); + Assert.Contains(expectedMessage, exception.Message, StringComparison.Ordinal); + Assert.DoesNotContain(ApiKey, exception.Message, StringComparison.Ordinal); + Assert.Single(handler.Requests); + } + + [Theory] + [InlineData("choice", "\"confidence\": 0.78, ", "", "confidence")] + [InlineData("unknown type", "\"type\": \"noul\", \"noul\": 0.97", "\"type\": \"rank\", \"noul\": 0.97", "could not be read")] + [InlineData("not json", "{", "", "could not be read")] + public async Task InvokeAsync_UnreadableResponse_FailsAsync(string scenario, string find, string replace, string expectedMessage) + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, ReplaceFirst(JevTestData.ResponseJson, find, replace)); + using var httpClient = new HttpClient(handler); + + // Act + JsonException exception = await Assert.ThrowsAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + + // Assert + Assert.True(exception.Message.Contains(expectedMessage, StringComparison.Ordinal), $"{scenario}: {exception.Message}"); + } + + [Fact] + public async Task InvokeAsync_Canceled_StopsWithoutSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + // Act & Assert + await Assert.ThrowsAnyAsync(() => + CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson), cancellation.Token).AsTask()); + Assert.Empty(handler.Requests); + } + + private static JevAIToolBuilder CreateBuilder(HttpClient httpClient, JevAIToolOptions? options = null, List? waits = null) + { + options ??= new JevAIToolOptions(); + options.Transport = new HttpClientPipelineTransport(httpClient); + options.RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = waits ?? [] }; + return new JevAIToolBuilder(new ApiKeyCredential(ApiKey), options); + } + + private static string ReplaceFirst(string text, string find, string replace) + { + int index = text.IndexOf(find, StringComparison.Ordinal); + Assert.True(index >= 0, $"'{find}' is not in the test response."); + return string.Concat(text.AsSpan(0, index), replace, text.AsSpan(index + find.Length)); + } + + private sealed class HeaderPolicy(string name, string value) : PipelinePolicy + { + public override void Process(PipelineMessage message, IReadOnlyList pipeline, int currentIndex) + { + message.Request.Headers.Set(name, value); + ProcessNext(message, pipeline, currentIndex); + } + + public override ValueTask ProcessAsync(PipelineMessage message, IReadOnlyList pipeline, int currentIndex) + { + message.Request.Headers.Set(name, value); + return ProcessNextAsync(message, pipeline, currentIndex); + } + } + + private sealed class NoWaitRetryPolicy(int maxRetries) : ClientRetryPolicy(maxRetries) + { + protected override Task WaitAsync(TimeSpan time, CancellationToken cancellationToken) => Task.CompletedTask; + + protected override void Wait(TimeSpan time, CancellationToken cancellationToken) + { + } + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs new file mode 100644 index 00000000000..3afde9ae642 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A support-ticket request and response taken from the TypeSafe quick start, used across the tests. +/// +internal static class JevTestData +{ + public const string TicketText = "Hi, I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales. Please help ASAP."; + + /// The tool arguments a model would write for the ticket, with the three question kinds. + public const string RequestJson = """ + { + "state": "Hi, I've been trying to connect my Stripe account for 3 days and the integration keeps failing. I'm losing sales. Please help ASAP.", + "questions": { + "department": { + "type": "choice", + "instructions": "Which team should handle this", + "criteria": { "billing": "Payment or subscription issues", "technical": "Bugs or integration problems", "sales": null } + }, + "frustration": { + "type": "score", + "instructions": "How frustrated the customer appears", + "criteria": ["Calm, just stating facts", "Frustrated but civil", "Very angry, strong language"] + }, + "is_urgent": { + "type": "noul", + "instructions": "The message conveys urgency or time-sensitivity", + "criteria": { "true": "Explicitly time-sensitive", "false": "No urgency expressed" } + } + } + } + """; + + /// The API response for . + public const string ResponseJson = """ + { + "model": "jev-1.13.0", + "answers": { + "department": { "type": "choice", "choice": "technical", "confidence": 0.78, "probabilities": { "technical": 0.85, "sales": 0.0, "billing": 0.15 } }, + "frustration": { + "type": "score", "score": 1.0, "confidence": 1.0, + "legend": { "0": "Calm, just stating facts", "1": "Frustrated but civil", "2": "Very angry, strong language" }, + "probabilities": { "0": 0.0, "1": 1.0, "2": 0.0 } + }, + "is_urgent": { "type": "noul", "noul": 0.97 } + }, + "usage": { "input_tokens": 392, "output_tokens": 65 } + } + """; + + /// Parses tool arguments the way chat clients do: one per top-level property. + public static AIFunctionArguments Arguments(string json) + { + var arguments = new AIFunctionArguments(); + using JsonDocument document = JsonDocument.Parse(json); + foreach (JsonProperty property in document.RootElement.EnumerateObject()) + { + arguments[property.Name] = property.Value.Clone(); + } + + return arguments; + } + + /// Parses tool arguments into the dictionary shape that carries. + public static Dictionary CallArguments(string json) + { + var arguments = new Dictionary(); + foreach (KeyValuePair argument in Arguments(json)) + { + arguments[argument.Key] = argument.Value; + } + + return arguments; + } + + /// A correct response to , built from the contract types. + public static JevResponse CreateResponse() => new() + { + Model = "custom-model", + Answers = new Dictionary + { + ["department"] = new JevChoiceAnswer + { + Choice = "technical", + Confidence = 0.78, + Probabilities = new Dictionary { ["technical"] = 0.85, ["billing"] = 0.15, ["sales"] = 0 }, + }, + ["frustration"] = new JevScoreAnswer + { + Score = 1, + Confidence = 1, + Legend = new Dictionary { ["0"] = "Calm", ["1"] = "Frustrated", ["2"] = "Very angry" }, + Probabilities = new Dictionary { ["0"] = 0, ["1"] = 1, ["2"] = 0 }, + }, + ["is_urgent"] = new JevNoulAnswer { Noul = 0.97 }, + }, + Usage = new JevUsage { InputTokens = 10, OutputTokens = 5 }, + }; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs new file mode 100644 index 00000000000..bc53c152cd1 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A fake TypeSafe endpoint that records each request and replies with queued responses, repeating the last one. +/// +internal sealed class JevTestHttpHandler : HttpMessageHandler +{ + private readonly Queue<(HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter)> _responses = new(); + private (HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter) _last = (HttpStatusCode.OK, JevTestData.ResponseJson, null); + + public List Requests { get; } = []; + + public JevTestHttpHandler Reply(HttpStatusCode status, string body, RetryConditionHeaderValue? retryAfter = null) + { + this._responses.Enqueue((status, body, retryAfter)); + return this; + } + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + string body = await request.Content!.ReadAsStringAsync(cancellationToken); + var headers = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (KeyValuePair> header in request.Headers) + { + headers[header.Key] = string.Join(",", header.Value); + } + + this.Requests.Add(new CapturedRequest(request.Method, request.RequestUri!, request.Headers.Authorization, body, headers)); + + if (this._responses.Count > 0) + { + this._last = this._responses.Dequeue(); + } + + var response = new HttpResponseMessage(this._last.Status) + { + Content = new StringContent(this._last.Body, Encoding.UTF8, "application/json"), + }; + response.Headers.RetryAfter = this._last.RetryAfter; + return response; + } + + public sealed record CapturedRequest(HttpMethod Method, Uri Uri, AuthenticationHeaderValue? Authorization, string Body, IReadOnlyDictionary Headers); +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj new file mode 100644 index 00000000000..76656380a93 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/Microsoft.Agents.AI.TypeSafe.UnitTests.csproj @@ -0,0 +1,17 @@ + + + + $(TargetFrameworksCore) + + + + + + + + + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs new file mode 100644 index 00000000000..e2b12db3bf7 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ScriptedChatClient.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A chat client that stands in for the agent's model: each call returns the next scripted response and records +/// the messages it received. +/// +internal sealed class ScriptedChatClient : IChatClient +{ + private readonly Queue, ChatResponse>> _script; + + public ScriptedChatClient(params Func, ChatResponse>[] script) + { + this._script = new Queue, ChatResponse>>(script); + } + + public List> Calls { get; } = []; + + public Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + List received = messages.ToList(); + this.Calls.Add(received); + return Task.FromResult(this._script.Dequeue()(received)); + } + + public async IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, + ChatOptions? options = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + ChatResponse response = await this.GetResponseAsync(messages, options, cancellationToken); + foreach (ChatResponseUpdate update in response.ToChatResponseUpdates()) + { + yield return update; + } + } + + public object? GetService(Type serviceType, object? serviceKey = null) => null; + + public void Dispose() + { + } +} From c4a10b376555c6147b07e5eb9714ec5006a3eb07 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 29 Sep 2026 18:16:44 +0100 Subject: [PATCH 2/6] .NET: Add JevChatClient to Microsoft.Agents.AI.TypeSafe and align Jev types with the TypeSafe SDK Add an IChatClient for Jev, the TypeSafe System One model, ported from the Python TypeSafe connector (agent-framework-typesafe): - JevChatClient sends the conversation as the Jev state and answers the questions passed with ChatOptions.WithJevQuestions (carried by RawRepresentationFactory) or JevChatClientOptions.DefaultQuestions. GetJevResult reads the typed result from ChatResponse and AgentResponse. - Closed-set function calling: internal route and argument questions choose the tool and each const, enum, Boolean, or enum-array argument, with presence questions for optional arguments, previous-call hints, and the same question IDs and limits as the Python connector. MaximumToolCallsPerTurn (default 1) ends the tool loop. - Streaming, generative options, hosted tools, and non-text content are rejected with NotSupportedException. Handled tool approvals are left out of the state, and text sent with an approval continues the approved call's turn. Align the tool contract with the official TypeScript SDK (@typesafe-ai/sdk): JevEntry (text or JSON entries), JevResult and JevResponse types, SDK retry and timeout defaults, JevHttpClient, and UseMapping instead of UseEvaluator. Samples: Agent_With_TypeSafe (chat client, default questions, closed-set tools, optional arguments, approvals), Agent_TypeSafe_Step01_LoopJudge (Jev as the judge of a LoopAgent), and a local UseClient mapping in Agent_Step25_ClassificationWithJev. The three samples are registered in eng/verify-samples and run when TYPESAFE_API_KEY is set. --- .github/workflows/dotnet-verify-samples.yml | 2 + dotnet/agent-framework-dotnet.slnx | 2 + dotnet/eng/verify-samples/AgentsSamples.cs | 66 +++ .../02-agents/AgentProviders/README.md | 7 + .../Agent_TypeSafe_Step01_LoopJudge.csproj | 20 + .../Program.cs | 101 ++++ .../Agent_TypeSafe_Step01_LoopJudge/README.md | 70 +++ .../Agent_With_TypeSafe.csproj | 16 + .../typesafe/Agent_With_TypeSafe/Program.cs | 159 ++++++ .../typesafe/Agent_With_TypeSafe/README.md | 107 ++++ .../Agent_Step25_ClassificationWithJev.csproj | 1 + .../KeywordClassifier.cs | 60 +++ .../Program.cs | 96 +++- .../README.md | 76 ++- dotnet/samples/AGENTS.md | 3 +- .../Core/JevAIFunction.cs | 26 +- .../Core/JevApiRequest.cs | 6 +- .../Core/JevArgumentKind.cs | 21 + .../Core/JevArgumentPlan.cs | 79 +++ .../Core/JevChatClientLog.cs | 15 + .../Core/JevChatState.cs | 279 +++++++++++ .../Core/JevCompiledTool.cs | 36 ++ .../Core/JevContractValidator.cs | 145 +++--- .../Core/JevEntryJsonConverter.cs | 39 ++ .../{JevHttpEvaluator.cs => JevHttpClient.cs} | 41 +- .../Core/JevJsonContext.cs | 3 +- .../Core/JevJsonUtilities.cs | 130 ++++- .../Core/JevPipelineDefaults.cs | 34 ++ .../Core/JevQuestionBudget.cs | 42 ++ .../Core/JevResponseClassifier.cs | 21 +- .../Core/JevRetryPolicy.cs | 86 +++- .../Core/JevToolCall.cs | 11 + .../Core/JevToolCallCompiler.cs | 424 ++++++++++++++++ .../Core/JevToolCallPlan.cs | 65 +++ .../Core/JevToolSchema.cs | 303 ++++++++++++ .../Core/JevUnsupportedToolSchemaException.cs | 30 ++ .../JevAIToolBuilder.cs | 89 ++-- .../JevAIToolOptions.cs | 39 +- .../Microsoft.Agents.AI.TypeSafe/JevAnswer.cs | 24 - .../JevChatClient.cs | 302 ++++++++++++ .../JevChatClientOptions.cs | 104 ++++ .../JevChatOptionsExtensions.cs | 43 ++ .../JevChatRequestOptions.cs | 26 + .../JevChoiceAnswer.cs | 35 -- .../JevChoiceQuestion.cs | 16 +- .../JevChoiceResponse.cs | 38 ++ .../Microsoft.Agents.AI.TypeSafe/JevEntry.cs | 138 ++++++ .../JevNoulAnswer.cs | 20 - .../JevNoulCriteria.cs | 14 +- .../JevNoulQuestion.cs | 11 +- .../JevNoulResponse.cs | 23 + .../JevQuestion.cs | 15 +- .../JevRequest.cs | 22 +- .../JevResponse.cs | 42 +- .../Microsoft.Agents.AI.TypeSafe/JevResult.cs | 40 ++ .../JevResultExtensions.cs | 62 +++ .../JevScoreAnswer.cs | 42 -- .../JevScoreQuestion.cs | 16 +- .../JevScoreResponse.cs | 45 ++ .../Microsoft.Agents.AI.TypeSafe/JevUsage.cs | 2 +- .../Microsoft.Agents.AI.TypeSafe.csproj | 7 +- .../JevAIFunctionTests.cs | 249 ++++++---- .../JevAIToolBuilderTests.cs | 58 ++- .../JevAgentIntegrationTests.cs | 12 +- .../JevChatClientTests.cs | 386 +++++++++++++++ .../JevChatClientToolTests.cs | 463 ++++++++++++++++++ .../JevChatTestData.cs | 54 ++ .../JevEntryTests.cs | 116 +++++ ...valuatorTests.cs => JevHttpClientTests.cs} | 91 +++- .../JevLoopJudgeTests.cs | 81 +++ .../JevTestData.cs | 14 +- .../JevTestHttpHandler.cs | 19 +- .../JevToolCallCompilerTests.cs | 207 ++++++++ .../ListLoggerFactory.cs | 36 ++ 74 files changed, 5064 insertions(+), 559 deletions(-) create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs create mode 100644 dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md create mode 100644 dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs rename dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/{JevHttpEvaluator.cs => JevHttpClient.cs} (66%) create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs delete mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs rename dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/{JevHttpEvaluatorTests.cs => JevHttpClientTests.cs} (76%) create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs diff --git a/.github/workflows/dotnet-verify-samples.yml b/.github/workflows/dotnet-verify-samples.yml index 3855914f6c9..bd60fa1d34e 100644 --- a/.github/workflows/dotnet-verify-samples.yml +++ b/.github/workflows/dotnet-verify-samples.yml @@ -112,6 +112,8 @@ jobs: # Foundry FOUNDRY_PROJECT_ENDPOINT: ${{ vars.FOUNDRY_PROJECT_ENDPOINT }} FOUNDRY_MODEL: ${{ vars.FOUNDRY_MODEL }} + # TypeSafe (Jev) + TYPESAFE_API_KEY: ${{ secrets.TYPESAFE_API_KEY }} - name: Write Job Summary if: always() diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index ec1b47fdcbe..c60b7e4dded 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -36,6 +36,8 @@ + + diff --git a/dotnet/eng/verify-samples/AgentsSamples.cs b/dotnet/eng/verify-samples/AgentsSamples.cs index f8cc340e9cb..0c125b56c56 100644 --- a/dotnet/eng/verify-samples/AgentsSamples.cs +++ b/dotnet/eng/verify-samples/AgentsSamples.cs @@ -77,6 +77,49 @@ internal static class AgentsSamples ], }, + new SampleDefinition + { + Name = "Agent_With_TypeSafe", + ProjectPath = "samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe", + RequiredEnvironmentVariables = ["TYPESAFE_API_KEY"], + Inputs = ["Y"], + MustContain = + [ + "Direct JevChatClient result:", + "Agent result:", + "Default questions result:", + "Function calling result:", + "better_city:", + "Tool approval:", + "issue_refund(plan: Pro)", + ], + ExpectedOutputDescription = + [ + "The direct result should route the duplicate subscription charge to billing, and the agent result should route the failing checkout integration to technical with a high urgency probability.", + "The default questions result should route the invoice request to billing with a low urgency probability.", + "The function calling result should contain one weather line for Seattle and one for Amsterdam, with temperatures in Fahrenheit, followed by a better_city line that names one of the two cities.", + "The tool approval step should ask to approve issue_refund for the Pro plan and, after approval, show the refund result followed by a department line.", + "The output should not contain error messages or stack traces.", + ], + }, + + new SampleDefinition + { + Name = "Agent_TypeSafe_Step01_LoopJudge", + ProjectPath = "samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge", + RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT", "TYPESAFE_API_KEY"], + OptionalEnvironmentVariables = ["FOUNDRY_MODEL"], + MustContain = ["Answer 1:", "Judge:", "red_sunsets:", "Final answer:"], + ExpectedOutputDescription = + [ + "The first answer should explain only why the sky is blue, and the judge should mark red_sunsets as missing with a low probability.", + "A later answer should also explain why sunsets are red, and the judge should mark red_sunsets as met for it.", + "Every numbered answer (Answer 1, Answer 2, and so on) should be followed by a Judge block with probabilities. The Final answer section repeats the last judged answer and has no Judge block of its own.", + "The final answer should explain both why the sky is blue and why sunsets are red.", + "The output should not contain error messages or stack traces.", + ], + }, + // ── Agents ────────────────────────────────────────────────────────── new SampleDefinition @@ -403,6 +446,29 @@ internal static class AgentsSamples ], }, + new SampleDefinition + { + Name = "Agent_Step25_ClassificationWithJev", + ProjectPath = "samples/02-agents/Agents/Agent_Step25_ClassificationWithJev", + RequiredEnvironmentVariables = ["FOUNDRY_PROJECT_ENDPOINT", "TYPESAFE_API_KEY"], + OptionalEnvironmentVariables = ["FOUNDRY_MODEL"], + MustContain = + [ + "Ticket:", + "Tool result:", + "The same tool, answered by a local keyword classifier:", + "local-keyword-classifier", + ], + ExpectedOutputDescription = + [ + "Each of the three tickets should show a tool result with department, urgent, and frustration answers from a jev model, followed by a one-line agent reply.", + "The failing Stripe integration ticket should be routed to technical and marked urgent, and the invoice request should be routed to billing and not urgent.", + "The vague account ticket may be sent to human review when the department confidence is low.", + "The last part should show a tool result from the local-keyword-classifier model for the invoice ticket, followed by an agent reply.", + "The output should not contain error messages or stack traces.", + ], + }, + // ── AgentSkills ───────────────────────────────────────────────────── new SampleDefinition diff --git a/dotnet/samples/02-agents/AgentProviders/README.md b/dotnet/samples/02-agents/AgentProviders/README.md index 407c4e8e4d9..982e35874cd 100644 --- a/dotnet/samples/02-agents/AgentProviders/README.md +++ b/dotnet/samples/02-agents/AgentProviders/README.md @@ -93,6 +93,13 @@ covering basics, function tools, structured output, middleware, MCP, code interp | [Conversation](./openai/Agent_OpenAI_Step05_Conversation/) | Multi-turn conversations with OpenAI | | [Code Interpreter](./openai/Agent_OpenAI_Step06_CodeInterpreterFileDownload/) | Code interpreter with file downloads | +### [TypeSafe](./typesafe/) + +| Sample | Description | +| --- | --- | +| [Agent with TypeSafe](./typesafe/Agent_With_TypeSafe/) | Use Jev, the TypeSafe System One model, as an IChatClient that answers typed questions and calls closed-set tools | +| [Loop judge](./typesafe/Agent_TypeSafe_Step01_LoopJudge/) | Use Jev as the judge of a LoopAgent, with one yes or no question per criterion | + ## Running the samples Navigate to a sample directory and run: diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj new file mode 100644 index 00000000000..48a53996f27 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Agent_TypeSafe_Step01_LoopJudge.csproj @@ -0,0 +1,20 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs new file mode 100644 index 00000000000..4bfe9cb9c7c --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample uses Jev, the TypeSafe System One model, as the judge of a LoopAgent. A Foundry agent writes the +// answer. After each answer, Jev checks every criterion with its own yes or no question, and the loop sends the agent +// the criteria that are still missing, until the answer meets all of them. + +#pragma warning disable MAAI001 // LoopAgent and DelegateLoopEvaluator are experimental. + +using System.ClientModel; +using Azure.AI.Projects; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; +var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// 1. The criteria. Each one becomes a Noul question, whose answer is the probability that the answer meets it, so the +// judge reports exactly which criteria are missing instead of a single verdict. +Dictionary criteria = new() +{ + ["blue_sky"] = "explains why the daytime sky appears blue", + ["red_sunsets"] = "explains why sunsets appear red or orange", + ["plain_language"] = "uses clear language suitable for a general audience", +}; +Dictionary questions = criteria.ToDictionary( + criterion => criterion.Key, + criterion => (JevQuestion)new JevNoulQuestion { Instructions = $"Does the assistant's latest response {criterion.Value}?" }); + +// 2. The agent that writes the answers. Its instructions hold back part of the first answer, so the judge has a gap +// to find and the loop runs more than once. +// WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. +// In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid +// latency issues, unintended credential probing, and potential security risks from fallback mechanisms. +AIAgent answerer = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()).AsAIAgent( + model: deploymentName, + name: "Answerer", + instructions: """ + You answer science questions for a general audience in one short paragraph. + In your first answer, explain only why the daytime sky is blue, even if the user asks for more. + When feedback lists missing points, rewrite the whole answer so that it covers them too. + """); + +// 3. The judge. AIJudgeLoopEvaluator asks its chat client for JSON structured output, which Jev cannot write, so a +// DelegateLoopEvaluator asks Jev the criteria questions instead. Jev reads the original request and the latest answer +// as a conversation. +using var judge = new JevChatClient(new ApiKeyCredential(typeSafeApiKey)); +bool accepted = false; +var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => +{ + Console.WriteLine($"\nAnswer {context.Iteration}:\n{context.LastResponse.Text}"); + List missing = await JudgeAsync(context.InitialMessages, context.LastResponse.Text, cancellationToken); + accepted = missing.Count == 0; + + // The feedback names the missing criteria, which is the gap analysis that a generative judge would write. + return accepted + ? LoopEvaluation.Stop() + : LoopEvaluation.Continue($"Your answer does not yet: {string.Join("; ", missing)}. Rewrite the whole answer so that it does."); +}); + +// 4. The loop re-runs the agent until the judge stops it. MaxIterations bounds the cost if the answer never passes. +const int MaxIterations = 3; +const string Request = "Explain why the sky is blue and why sunsets are red."; +AIAgent loop = new LoopAgent(answerer, evaluator, new LoopAgentOptions { MaxIterations = MaxIterations, NonStreamingReturnsLastResponseOnly = true }); +AgentResponse response = await loop.RunAsync(Request); + +// LoopAgent stops at MaxIterations before it calls the evaluator, so an answer from the last allowed run has not been +// judged yet. Judging it here keeps the final answer from being presented as accepted when it may miss a criterion. +if (!accepted) +{ + Console.WriteLine($"\nAnswer {MaxIterations} (the last run the loop allows):\n{response.Text}"); + accepted = (await JudgeAsync([new ChatMessage(ChatRole.User, Request)], response.Text, CancellationToken.None)).Count == 0; +} + +Console.WriteLine(accepted ? "\nFinal answer:" : "\nFinal answer (some criteria are still missing):"); +Console.WriteLine(response.Text); + +// Asks Jev whether the answer meets each criterion, prints the probabilities, and returns the missing criteria. +async Task> JudgeAsync(IEnumerable request, string answer, CancellationToken cancellationToken) +{ + List conversation = [.. request, new ChatMessage(ChatRole.Assistant, answer)]; + ChatResponse verdict = await judge.GetResponseAsync(conversation, new ChatOptions().WithJevQuestions(questions), cancellationToken); + JevResult result = verdict.GetJevResult()!; + + Console.WriteLine("Judge:"); + List missing = []; + foreach ((string id, string description) in criteria) + { + double probability = ((JevNoulResponse)result.Answers[id]).Noul; + bool met = probability > 0.5; + Console.WriteLine($" {id}: {(met ? "met" : "missing")} (probability {probability:F2})"); + if (!met) + { + missing.Add(description); + } + } + + return missing; +} diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md new file mode 100644 index 00000000000..be92f66704a --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md @@ -0,0 +1,70 @@ +# Jev as the judge of a loop + +This sample uses `JevChatClient` from `Microsoft.Agents.AI.TypeSafe` as the judge of a `LoopAgent`. A Foundry agent +writes the answer, and Jev, the TypeSafe System One model, decides whether the answer is complete. + +A generative judge writes a verdict and a gap analysis. Jev writes no text, so the sample asks it one `JevNoulQuestion` +per criterion instead. Each answer is the probability, from 0 to 1, that the answer meets that criterion, so the judge +reports exactly which criteria are missing: + +```text +Judge: + blue_sky: met (probability 0.99) + red_sunsets: missing (probability 0.02) + plain_language: met (probability 0.92) +``` + +The loop turns the missing criteria into the feedback for the next run, and stops when every criterion is met or after +three runs. The agent's instructions hold back part of the first answer on purpose, so the loop visibly runs twice. + +`LoopAgent` checks `MaxIterations` before it calls the evaluator, so when the loop reaches its limit, the answer of the +last run has not been judged. The sample judges that answer after the loop and labels the final answer when some +criteria are still missing, instead of presenting it as accepted. + +## Why a delegate evaluator + +`AIJudgeLoopEvaluator` asks its chat client for a `JudgeVerdict` as JSON structured output, which Jev cannot write; +`JevChatClient` rejects the JSON response format with a `NotSupportedException`. The sample wraps the judge in a +`DelegateLoopEvaluator` instead: + +```csharp +var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => +{ + List missing = await JudgeAsync(context.InitialMessages, context.LastResponse.Text, cancellationToken); + accepted = missing.Count == 0; + return accepted + ? LoopEvaluation.Stop() + : LoopEvaluation.Continue($"Your answer does not yet: {string.Join("; ", missing)}. ..."); +}); +``` + +`JudgeAsync` sends Jev the original request and the answer as a conversation, with +`new ChatOptions().WithJevQuestions(questions)`, and returns the criteria whose Noul is 0.5 or lower. + +Jev reads the original request and the latest answer as a conversation, and each question refers to "the assistant's +latest response". The same pattern works with a `JevScoreQuestion` for a graded rubric, or a `JevChoiceQuestion` to +classify what kind of revision is needed. + +## Prerequisites + +- .NET 10 SDK or later +- A Microsoft Foundry project with a deployed chat model +- Azure CLI installed and authenticated (`az login`) +- A TypeSafe API key + +Set the environment variables: + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-foundry-service.services.ai.azure.com/api/projects/your-foundry-project" +$env:FOUNDRY_MODEL="gpt-5.4-mini" # Optional, defaults to gpt-5.4-mini +$env:TYPESAFE_API_KEY="your-typesafe-api-key" +``` + +## Running the sample + +```powershell +dotnet run +``` + +The sample prints each answer with the judge's probabilities, then the final answer, labeled when some criteria are +still missing. The wording of the answers and the exact probabilities vary between runs. diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj new file mode 100644 index 00000000000..b83c02d799f --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Agent_With_TypeSafe.csproj @@ -0,0 +1,16 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + + diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs new file mode 100644 index 00000000000..367db584b33 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/Program.cs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample shows how to use Jev, the TypeSafe System One model, as an IChatClient and as an AIAgent. +// Jev does not write text: it answers typed questions about the conversation and returns calibrated probabilities. + +using System.ClientModel; +using System.ComponentModel; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.TypeSafe; +using Microsoft.Extensions.AI; + +var apiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); + +// 1. The questions to answer. Each answer comes back under the ID of its question. +Dictionary questions = new() +{ + ["department"] = new JevChoiceQuestion + { + Instructions = "Which team should handle the support request?", + Criteria = new Dictionary + { + ["billing"] = "Payments, subscriptions, invoices, or refunds", + ["technical"] = "Bugs, outages, integrations, or account access", + ["sales"] = "Pricing, upgrades, or new account questions", + }, + }, + ["frustration"] = new JevScoreQuestion + { + Instructions = "How frustrated does the customer appear?", + Criteria = ["Calm and factual", "Frustrated but civil", "Very angry or threatening to leave"], + }, + ["is_urgent"] = new JevNoulQuestion + { + Instructions = "Does the request require urgent attention?", + Criteria = new JevNoulCriteria + { + True = "The customer describes ongoing harm, lost revenue, or an immediate deadline", + False = "The request can wait for the normal support queue", + }, + }, +}; + +// 2. Call the chat client directly. WithJevQuestions passes the questions through ChatOptions. +using IChatClient chatClient = new JevChatClient(new ApiKeyCredential(apiKey)); +ChatResponse direct = await chatClient.GetResponseAsync( + "I was charged twice for the same subscription. Please refund the duplicate charge.", + new ChatOptions().WithJevQuestions(questions)); +PrintEvaluation("Direct JevChatClient result:", direct.GetJevResult()!); + +// 3. Use the same client through an agent. ChatClientAgentRunOptions carries the questions for one run. +AIAgent agent = chatClient.AsAIAgent(name: "JevTicketEvaluator", instructions: "Evaluate the support request."); +AgentResponse agentResponse = await agent.RunAsync( + "Our checkout integration has failed for three days and we are losing sales. Please help ASAP.", + options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(questions))); +PrintEvaluation("Agent result:", agentResponse.GetJevResult()!); + +// 4. Default questions. A client created with DefaultQuestions answers requests that pass no questions, so code that +// knows nothing about Jev can use it like any other IChatClient; here, an agent run without Jev options. +using IChatClient triageClient = new JevChatClient(new ApiKeyCredential(apiKey), new JevChatClientOptions { DefaultQuestions = questions }); +AgentResponse triaged = await triageClient.AsAIAgent(name: "JevTriage").RunAsync( + "Hi, could you send me a copy of last month's invoice when you get a chance? Thanks!"); +PrintEvaluation("Default questions result:", triaged.GetJevResult()!); + +// 5. Function calling. Jev cannot write arguments, so it chooses them from closed sets: an enum, a Boolean, and an +// optional enum that is passed only when Jev decides the user asked for it. The user asks about two cities, so the +// client allows two tool calls per turn instead of the default one. Jev reads the tool's name and description when +// it chooses a tool, so the local function gets a readable name instead of its compiler-generated one. +using IChatClient weatherClient = new JevChatClient(new ApiKeyCredential(apiKey), new JevChatClientOptions { MaximumToolCallsPerTurn = 2 }); +AIAgent weatherAgent = weatherClient.AsAIAgent(name: "WeatherEvaluator", tools: [AIFunctionFactory.Create(GetWeather, "get_weather")]); +var comparison = new Dictionary +{ + ["better_city"] = new JevChoiceQuestion + { + Instructions = "Which city has better weather based on the tool results?", + Criteria = new Dictionary + { + ["Seattle"] = "Seattle has the better weather.", + ["Amsterdam"] = "Amsterdam has the better weather.", + }, + }, +}; +AgentResponse weather = await weatherAgent.RunAsync( + "Give me a detailed weather report for Seattle and Amsterdam in Fahrenheit and tell me where the weather is better.", + options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(comparison))); + +// After tool calls, the text is the tool results followed by one line per Choice and Score answer. +Console.WriteLine("\nFunction calling result:"); +Console.WriteLine(weather.Text); +Console.WriteLine($"Comparison confidence: {((JevChoiceResponse)weather.GetJevResult()!.Answers["better_city"]).Confidence:F3}"); + +// 6. Tool approval. A tool wrapped in ApprovalRequiredAIFunction runs only after the user approves the call that Jev +// chose. The session keeps the pending call, so the approval is sent in a second run with the same questions. +AIAgent refundAgent = chatClient.AsAIAgent(name: "RefundAgent", tools: [new ApprovalRequiredAIFunction(AIFunctionFactory.Create(IssueRefund, "issue_refund"))]); +AgentSession session = await refundAgent.CreateSessionAsync(); +var refundOptions = new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(questions)); +AgentResponse refund = await refundAgent.RunAsync( + "I was charged twice for my Pro plan this month. Please refund the duplicate charge.", + session, + refundOptions); + +Console.WriteLine("\nTool approval:"); +if (refund.Messages.SelectMany(message => message.Contents).OfType().FirstOrDefault() is { ToolCall: FunctionCallContent call } approval) +{ + string arguments = string.Join(", ", call.Arguments?.Select(argument => $"{argument.Key}: {argument.Value}") ?? []); + Console.Write($"Jev wants to call {call.Name}({arguments}). Approve? (Y/N): "); + bool approved = Console.ReadLine()?.Trim().Equals("Y", StringComparison.OrdinalIgnoreCase) ?? false; + refund = await refundAgent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(approved)]), session, refundOptions); +} + +Console.WriteLine(refund.Text); + +static void PrintEvaluation(string label, JevResult result) +{ + var department = (JevChoiceResponse)result.Answers["department"]; + var frustration = (JevScoreResponse)result.Answers["frustration"]; + var urgent = (JevNoulResponse)result.Answers["is_urgent"]; + + Console.WriteLine($"\n{label}"); + Console.WriteLine($"Department: {department.Choice} (confidence {department.Confidence:F3})"); + Console.WriteLine($"Frustration score: {frustration.Score:F3} of {frustration.Legend.Count - 1}"); + Console.WriteLine($"Urgency probability: {urgent.Noul:F3}"); + Console.WriteLine($"Tokens: {result.Usage.InputTokens} in, {result.Usage.OutputTokens} out"); +} + +[Description("Gets the weather forecast.")] +static string GetWeather( + [Description("The city to get the weather for")] City city, + [Description("Whether to include a detailed hourly forecast; true when the user asks for a detailed report")] bool detailed, + [Description("The temperature unit; leave it out to use Celsius")] TemperatureUnit? unit = null) +{ + int celsius = Random.Shared.Next(15, 31); + string temperature = unit == TemperatureUnit.Fahrenheit ? $"{(celsius * 9 / 5) + 32} F" : $"{celsius} C"; + string suffix = detailed ? " with a detailed hourly forecast" : string.Empty; + return $"{city} is sunny and {temperature}{suffix}."; +} + +[Description("Refunds a duplicate charge for a subscription plan.")] +static string IssueRefund([Description("The plan that was charged twice")] Plan plan) => + $"Refunded the duplicate {plan} plan charge."; + +internal enum City +{ + Seattle, + Amsterdam, + Paris, +} + +internal enum TemperatureUnit +{ + Celsius, + Fahrenheit, +} + +internal enum Plan +{ + Basic, + Pro, + Enterprise, +} diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md new file mode 100644 index 00000000000..77ac532b541 --- /dev/null +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md @@ -0,0 +1,107 @@ +# Agent with TypeSafe (Jev) + +This sample uses `JevChatClient` from `Microsoft.Agents.AI.TypeSafe`, an `IChatClient` for Jev, the TypeSafe System +One model. Jev does not write text. It answers typed questions about the conversation and returns calibrated +probabilities: + +| Question type | What it answers | Answer | +| --- | --- | --- | +| `JevChoiceQuestion` | Which of these named alternatives? | The selected label, the probability of each label, and a confidence | +| `JevScoreQuestion` | Which score on an ordered rubric (2 to 10 entries)? | An expected score that can fall between rubric entries, the probability of each score, and a confidence | +| `JevNoulQuestion` | Yes or no? | The probability, from 0 to 1, of yes | + +The client follows the TypeSafe connector for Python (`agent-framework-typesafe`), so both languages behave alike. + +## What the sample shows + +1. **The chat client directly.** `WithJevQuestions` puts the questions in `ChatOptions`, and `GetJevResult` reads the + typed answers from the response: + + ```csharp + IChatClient client = new JevChatClient(new ApiKeyCredential(apiKey)); + ChatResponse response = await client.GetResponseAsync(ticket, new ChatOptions().WithJevQuestions(questions)); + JevResult result = response.GetJevResult()!; + ``` + + The response text is the same result as JSON, so code that reads only text still gets every answer. + +2. **The client behind an agent.** A `ChatClientAgentRunOptions` carries the questions for one run. Questions in the + agent's `ChatClientAgentOptions.ChatOptions` apply to every run, and a run's questions replace them. + +3. **Default questions.** A client created with `JevChatClientOptions.DefaultQuestions` answers requests that pass no + questions, so code that knows nothing about Jev can use it like any other `IChatClient`. The sample runs an agent + without any Jev options. + +4. **Function calling.** Jev cannot write arguments, so the client adds internal questions that choose the tool and + each argument from a closed set: a constant, an enum, a Boolean, or an array of enum values. Optional arguments + are passed only when Jev decides the user asked for them; here, the temperature unit is passed because the user + asks for Fahrenheit. The agent runs the tool, and once the turn has used + `JevChatClientOptions.MaximumToolCallsPerTurn` calls (one by default, two in this sample), the client answers the + caller's questions with the tool results in the state. The response text is then the tool results followed by one + `id: answer` line per Choice and Score answer: + + ```text + Seattle is sunny and 75 F with a detailed hourly forecast. + Amsterdam is sunny and 69 F with a detailed hourly forecast. + better_city: Seattle + ``` + + Jev reads each tool's name and description, and each argument's description, to make these choices, so write them + for Jev as you would for a person. Give local functions an explicit name, because their compiler-generated names + mean nothing. In this sample, saying that `detailed` is true "when the user asks for a detailed report" is what makes + Jev choose it reliably. Tools with free-form arguments, such as text or numbers, are left out of the request with a + warning in the log. + +5. **Tool approval.** A tool wrapped in `ApprovalRequiredAIFunction` runs only after the user approves the call that + Jev chose. The first run returns a `ToolApprovalRequestContent`; the sample asks for approval in the console and + sends the answer in a second run on the same session, with the same questions. + +## Limits + +Jev answers all questions at once, so the client rejects what needs generated text: streaming (`RunStreamingAsync`), +sampling and output options such as `Temperature` or a JSON `ResponseFormat`, server-side conversations, tools other +than functions, and message content other than text, reasoning text, and function calls and results. Each fails with a +`NotSupportedException` that names the problem. Tool approval requests and responses are allowed but left out of what +Jev reads, because the call and result they lead to carry the information. + +Because streaming is not supported, run agents with `RunAsync`. A function-invoking client runs approved tools before +it calls the chat client, so a streaming run that resumes a tool approval would run the tool and then fail. + +## Configuring the client + +`JevChatClientOptions` follows the `ClientPipelineOptions` pattern of the OpenAI and Azure AI client libraries, with the +retry and timeout defaults of the official TypeSafe SDK: + +```csharp +var options = new JevChatClientOptions +{ + ModelId = "jev-latest", // ChatOptions.ModelId overrides it per request + MaximumToolCallsPerTurn = 2, + Transport = new HttpClientPipelineTransport(httpClientFactory.CreateClient("jev")), +}; +IChatClient client = new JevChatClient(new ApiKeyCredential(apiKey), options); +``` + +To use Jev as a tool of an agent whose own model writes the replies, see +[Classification with Jev](../../../Agents/Agent_Step25_ClassificationWithJev/). To use it as the judge of a +`LoopAgent`, see [Jev as the judge of a loop](../Agent_TypeSafe_Step01_LoopJudge/). + +## Prerequisites + +- .NET 10 SDK or later +- A TypeSafe API key + +Set the environment variable: + +```powershell +$env:TYPESAFE_API_KEY="your-typesafe-api-key" +``` + +## Running the sample + +```powershell +dotnet run +``` + +The tool approval step asks for `Y` or `N` in the console. Probabilities, scores, and the temperatures in the tool +results vary between runs. diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj index 26193187de0..1c976a48330 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Agent_Step25_ClassificationWithJev.csproj @@ -10,6 +10,7 @@ + diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs new file mode 100644 index 00000000000..338dd1e7dc6 --- /dev/null +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/KeywordClassifier.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +/// +/// A local classifier that stands in for another classification client. It has its own request and response types, +/// knows nothing about Jev, and needs no network: it scores each option by the words it shares with the text. +/// +[SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", Justification = "Instantiated by the service provider")] +internal sealed class KeywordClassifier +{ + private static readonly char[] s_separators = [' ', ',', '.', '!', '?', ';', ':', '\'', '"', '(', ')', '-', '\n', '\r', '\t']; + + public Task ClassifyAsync(ClassificationRequest request, CancellationToken cancellationToken) + { + HashSet words = Words(request.Text); + var results = new Dictionary(); + foreach (ClassificationTask task in request.Tasks) + { + // One point per shared word, plus one so that every option keeps some probability. + double[] points = [.. task.Options.Select(option => Words(option.Description).Count(words.Contains) + 1.0)]; + double total = points.Sum(); + results[task.Id] = new ClassificationResult( + task.Kind, + [.. task.Options.Select((option, index) => new ScoredOption(option.Key, option.Description, points[index] / total))]); + } + + return Task.FromResult(new ClassificationResponse(results)); + } + + // Words are compared in upper case and without a plural "S", so "invoice" matches "invoices". + private static HashSet Words(string text) => + [.. text.ToUpperInvariant().Split(s_separators, StringSplitOptions.RemoveEmptyEntries).Where(word => word.Length > 3).Select(word => word.TrimEnd('S'))]; +} + +/// What the classifier decides for one task. +internal enum ClassificationKind +{ + PickOne, + Rate, + YesNo, +} + +/// The classifier's request: the text and the tasks to decide about it. +internal sealed record ClassificationRequest(string Text, IReadOnlyList Tasks); + +/// One task: pick one option, rate on ordered options, or decide yes or no. +internal sealed record ClassificationTask(string Id, ClassificationKind Kind, IReadOnlyList Options); + +/// An option of a task and the words that describe it. +internal sealed record ClassificationOption(string Key, string Description); + +/// The classifier's response: one result per task, by task ID. +internal sealed record ClassificationResponse(IReadOnlyDictionary Results); + +/// The result of one task: every option with its probability. +internal sealed record ClassificationResult(ClassificationKind Kind, IReadOnlyList Options); + +/// An option with the probability that the classifier gives it. +internal sealed record ScoredOption(string Key, string Description, double Probability); diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs index c99b7db4f8f..a228d7573d2 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/Program.cs @@ -6,14 +6,16 @@ // (choice, score, noul) about a state and returns calibrated probabilities. This sample gives // a support agent a Jev tool built with JevAIToolBuilder. The agent's model decides which // questions to ask, Jev answers them, and the model writes the routing decision, sending -// uncertain tickets to a human. +// uncertain tickets to a human. The second part answers the same tool with another client. using System.ClientModel; +using System.Globalization; using Azure.AI.Projects; using Azure.Identity; using Microsoft.Agents.AI; using Microsoft.Agents.AI.TypeSafe; using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; var endpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); var deploymentName = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? "gpt-5.4-mini"; @@ -25,12 +27,12 @@ // const string Instructions = """ - You triage customer support tickets. For every ticket, call evaluate_with_jev once with the ticket text as the + You triage customer support tickets. For every ticket, call ask_system_one once with the ticket text as the state and these questions: - "department": a choice between billing (payments, invoices, refunds), technical (bugs, outages, integrations), and sales (pricing, upgrades, new accounts). - "urgent": a noul asking whether the customer needs help urgently. - - "frustration": a score with the levels calm, annoyed, and angry. + - "frustration": a score with the rubric calm, annoyed, and angry. Then reply in one line with the department, whether it is urgent, and the frustration level. When the department confidence is below 0.6, say the ticket needs human review instead of naming a department. """; @@ -38,8 +40,8 @@ // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. // In production, consider using a specific credential (e.g., ManagedIdentityCredential) to avoid // latency issues, unintended credential probing, and potential security risks from fallback mechanisms. -AIAgent agent = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()) - .AsAIAgent(model: deploymentName, instructions: Instructions, name: "TicketTriage", tools: [jev]); +var projectClient = new AIProjectClient(new Uri(endpoint), new DefaultAzureCredential()); +AIAgent agent = projectClient.AsAIAgent(model: deploymentName, instructions: Instructions, name: "TicketTriage", tools: [jev]); string[] tickets = [ @@ -49,15 +51,97 @@ ]; foreach (string ticket in tickets) +{ + await TriageAsync(agent, ticket); +} + +// +// The same tool can be answered by another client. UseClient takes the client's operation and two mappers: one from +// the JevRequest that the model writes to the client's request, and one from the client's response to a JevResult. +// The tool resolves the client from the agent's services, in a new scope on every call, and checks the mapped result +// against the request before the model sees it. +AIFunction localJev = new JevAIToolBuilder() + .UseClient( + (classifier, request, cancellationToken) => classifier.ClassifyAsync(request, cancellationToken), + inputMapper: ToClassificationRequest, + outputMapper: ToJevResult) + .Build(); + +using ServiceProvider services = new ServiceCollection() + .AddSingleton() + .BuildServiceProvider(); +AIAgent localAgent = projectClient.AsAIAgent( + model: deploymentName, + instructions: Instructions, + name: "LocalTicketTriage", + tools: [localJev], + services: services); +// + +Console.WriteLine("The same tool, answered by a local keyword classifier:"); +Console.WriteLine(); +await TriageAsync(localAgent, tickets[1]); + +static async Task TriageAsync(AIAgent agent, string ticket) { AgentResponse response = await agent.RunAsync(ticket); Console.WriteLine($"Ticket: {ticket}"); foreach (FunctionResultContent result in response.Messages.SelectMany(static m => m.Contents).OfType()) { - Console.WriteLine($"Jev answers: {result.Result}"); + Console.WriteLine($"Tool result: {result.Result}"); } Console.WriteLine($"Agent: {response.Text}"); Console.WriteLine(); } + +// +// Converts each Jev question into a classifier task. The options carry the words the classifier matches. +static ClassificationRequest ToClassificationRequest(JevRequest request) => + new( + request.State.ToString(), + [.. request.Questions.Select(question => question.Value switch + { + JevChoiceQuestion choice => new ClassificationTask( + question.Key, + ClassificationKind.PickOne, + [.. choice.Criteria.Select(criterion => new ClassificationOption(criterion.Key, $"{criterion.Key} {criterion.Value}"))]), + JevScoreQuestion score => new ClassificationTask( + question.Key, + ClassificationKind.Rate, + [.. score.Criteria.Select((level, index) => new ClassificationOption(index.ToString(CultureInfo.InvariantCulture), level.ToString()))]), + JevNoulQuestion noul => new ClassificationTask( + question.Key, + ClassificationKind.YesNo, + [new ClassificationOption("yes", $"{noul.Instructions} {noul.Criteria?.True}"), new ClassificationOption("no", $"{noul.Criteria?.False}")]), + _ => throw new NotSupportedException($"Question '{question.Key}' has an unknown type."), + })]); + +// Converts each classifier result back into the Jev answer of the same kind. +static JevResult ToJevResult(ClassificationResponse response) => new() +{ + Model = "local-keyword-classifier", + Answers = response.Results.ToDictionary(result => result.Key, result => ToJevResponse(result.Value)), + Usage = new JevUsage { InputTokens = 0, OutputTokens = 0 }, +}; + +static JevResponse ToJevResponse(ClassificationResult result) +{ + ScoredOption best = result.Options.MaxBy(option => option.Probability)!; + Dictionary probabilities = result.Options.ToDictionary(option => option.Key, option => option.Probability); + return result.Kind switch + { + ClassificationKind.PickOne => new JevChoiceResponse { Choice = best.Key, Confidence = best.Probability, Probabilities = probabilities }, + ClassificationKind.Rate => new JevScoreResponse + { + // Like Jev, the score is the expected value over the rubric, so it can fall between levels. + Score = result.Options.Sum(option => int.Parse(option.Key, CultureInfo.InvariantCulture) * option.Probability), + Confidence = best.Probability, + Legend = result.Options.ToDictionary(option => option.Key, option => (JevEntry)option.Description), + Probabilities = probabilities, + }, + _ => new JevNoulResponse { Noul = probabilities["yes"] }, + }; +} +// diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md index 21a96610475..cd289614e72 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md @@ -5,18 +5,27 @@ This sample gives an agent a classifier tool built with `JevAIToolBuilder` from Jev, the TypeSafe System One model, does not write text. It answers typed questions about a state and returns calibrated probabilities. An agent's model writes the replies; Jev makes the fast, structured decisions: -1. The agent's model calls `evaluate_with_jev` with a Jev request: the `state` to evaluate and the `questions` to +1. The agent's model calls `ask_system_one` with a Jev request: the `state` and the `questions` to answer about it. 2. Jev answers each question: | Question type | What it answers | Answer | | --- | --- | --- | - | `choice` | Which of these options? | The chosen option, the probability of each option, and a confidence | - | `score` | Which level of an ordered scale (2 to 10 levels)? | A score that can fall between levels, the probability of each level, and a confidence | - | `noul` | Is this yes or no statement true? | The probability, from 0 to 1, that it is true | + | `choice` | Which of these named alternatives? | The selected label, the probability of each label, and a confidence | + | `score` | Which score on an ordered rubric (2 to 10 entries)? | An expected score that can fall between rubric entries, the probability of each score, and a confidence | + | `noul` | Yes or no? | The probability, from 0 to 1, of yes | + + The state, the instructions, and every criterion can be text or JSON, like `EntryType` in the official TypeSafe SDK + ([`@typesafe-ai/sdk`](https://github.com/typesafe-ai/typesafe-sdk-js)). The Jev contract types mirror the SDK's type + names: `JevRequest`, `JevChoiceQuestion`, `JevResult`, `JevChoiceResponse`, and so on. 3. The model reads the answers and writes the reply. Here it routes each support ticket and sends uncertain tickets to human review. +4. The same tool is answered by a local keyword classifier instead of the TypeSafe API, through `UseClient` and the + agent's services (see [Using another client](#using-another-client)). + +To use Jev as the agent's chat client instead, answering questions about the whole conversation, see +[Agent with TypeSafe](../../AgentProviders/typesafe/Agent_With_TypeSafe/). ## Configuring the client @@ -35,42 +44,61 @@ options.AddPolicy(myPolicy, PipelinePosition.PerCall); // custom pipe AIFunction jev = new JevAIToolBuilder(new ApiKeyCredential(apiKey), options).Build(); ``` -The default retry policy retries rate-limited (429) and overloaded (529) requests, and fails at once when the API asks -for a wait longer than one minute, so the agent run is never blocked for long. `Name` and `Description` set what the -agent's model sees. +Unset pipeline settings get the defaults of the official SDK: two retries on 408, 429, and 5xx responses (529 means +overloaded), honoring `retry-after-ms` or `Retry-After` up to one minute and otherwise backing off from half a second +to five seconds, and a 10-second timeout per attempt. `Name` and `Description` set what the agent's model sees. ## Using another client -Any other client can answer the requests instead of the TypeSafe API. Pass a delegate that takes a `JevRequest` and -returns a `JevResponse`, or map between the Jev types and the client's own types: +Any other client can answer the requests instead of the TypeSafe API. The second part of the sample shows it with +`KeywordClassifier`, a local classifier in [KeywordClassifier.cs](./KeywordClassifier.cs) that has its own request and +response types, knows nothing about Jev, and needs no network. `UseClient` takes the client's operation and two +mappers between the Jev types and the client's types: ```csharp -AIFunction jev = new JevAIToolBuilder() - .UseClient( - myClient.EvaluateAsync, - inputMapper: request => ToMyClientRequest(request), - outputMapper: response => ToJevResponse(response)) +AIFunction localJev = new JevAIToolBuilder() + .UseClient( + (classifier, request, cancellationToken) => classifier.ClassifyAsync(request, cancellationToken), + inputMapper: ToClassificationRequest, + outputMapper: ToJevResult) .Build(); ``` -The tool checks the mapped response before the model sees it: every question needs an answer of the same type, and a -`choice` answer and its probabilities may only use the options the question offered. +`ToClassificationRequest` turns each Jev question into a classifier task, and `ToJevResult` turns each classifier +result back into the Jev answer of the same kind (see the `mappers` region of [Program.cs](./Program.cs)). The tool +checks the mapped result before the model sees it: every question needs an answer of the same type, and a `choice` +answer and its probabilities may only use the labels the question offered. The output mapper receives only the +client's response, so the client's result says which kind of answer it is. + +The keyword classifier is far less certain than Jev, so for the same invoice ticket the agent's confidence rule +usually sends it to human review. Use `UseMapping` instead of `UseClient` for a mapping that is a single delegate from +a `JevRequest` to a `JevResult`. ## Dependency injection -Every evaluator receives the `IServiceProvider` of the agent that calls the tool. `FunctionInvokingChatClient` passes +Every mapping receives the `IServiceProvider` of the agent that calls the tool. `FunctionInvokingChatClient` passes it in `AIFunctionArguments.Services`, from the services the agent was created with. `UseClient` resolves -a registered client from a new scope on every call and disposes the scope afterwards, so scoped and disposable clients -work even though hosted agents are singletons that receive the root provider: +the client from a new scope on every call and disposes the scope afterwards, so scoped and disposable clients work +even though hosted agents are singletons that receive the root provider. The sample registers the classifier in a +`ServiceCollection` and passes the provider to the agent: + +```csharp +using ServiceProvider services = new ServiceCollection() + .AddSingleton() + .BuildServiceProvider(); +AIAgent localAgent = projectClient.AsAIAgent(model: deploymentName, instructions: Instructions, tools: [localJev], services: services); +``` + +In a host, register the client and build the tool in the agent's tool factory: ```csharp -builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddAIAgent("triage", Instructions) .WithAITool(sp => new JevAIToolBuilder() - .UseClient( - (client, request, cancellationToken) => client.EvaluateAsync(request, cancellationToken), - inputMapper: ToMyClientRequest, - outputMapper: ToJevResponse) + .UseClient( + (classifier, request, cancellationToken) => classifier.ClassifyAsync(request, cancellationToken), + inputMapper: ToClassificationRequest, + outputMapper: ToJevResult) .Build()); ``` diff --git a/dotnet/samples/AGENTS.md b/dotnet/samples/AGENTS.md index 7c56735e4c7..417f3434bcc 100644 --- a/dotnet/samples/AGENTS.md +++ b/dotnet/samples/AGENTS.md @@ -26,7 +26,8 @@ dotnet/samples/ │ │ ├── google-gemini/ # Google Gemini provider sample │ │ ├── ollama/ # Ollama provider sample │ │ ├── onnx/ # ONNX Runtime provider sample -│ │ └── openai/ # OpenAI provider samples +│ │ ├── openai/ # OpenAI provider samples +│ │ └── typesafe/ # TypeSafe (Jev) chat client samples │ ├── AgentOpenTelemetry/ # OpenTelemetry integration │ ├── AgentSkills/ # Agent skills patterns │ ├── AgentWithMemory/ # Memory providers (chat history, Mem0, Valkey, Foundry, AgentMemory) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs index 8001bbd4f59..45fb819c118 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevAIFunction.cs @@ -13,17 +13,17 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// The created by : its arguments are a -/// and its result is the of the configured evaluator. +/// and its result is the of the configured mapping or the TypeSafe API. /// internal sealed class JevAIFunction : AIFunction { - private readonly Func> _evaluateAsync; + private readonly Func> _systemOneAsync; - public JevAIFunction(string name, string description, Func> evaluateAsync) + public JevAIFunction(string name, string description, Func> systemOneAsync) { this.Name = name; this.Description = description; - this._evaluateAsync = evaluateAsync; + this._systemOneAsync = systemOneAsync; } public override string Name { get; } @@ -32,7 +32,7 @@ public JevAIFunction(string name, string description, Func JevJsonUtilities.RequestSchema; - public override JsonElement? ReturnJsonSchema => JevJsonUtilities.ResponseSchema; + public override JsonElement? ReturnJsonSchema => JevJsonUtilities.ResultSchema; public override JsonSerializerOptions JsonSerializerOptions => JevJsonUtilities.Options; @@ -42,27 +42,27 @@ public JevAIFunction(string name, string description, Func -/// The body of a TypeSafe System One API request: a plus the model that answers it. +/// The body of POST /v1/systemone: a with the model resolved, like +/// SystemOneRequestPayload of the official TypeSafe SDK. /// internal sealed class JevApiRequest { [JsonPropertyName("state")] - public required JsonElement State { get; init; } + public required JevEntry State { get; init; } [JsonPropertyName("model")] public required string Model { get; init; } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs new file mode 100644 index 00000000000..a99e841d048 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentKind.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// How the value of one tool argument is read from Jev's answers. +/// +internal enum JevArgumentKind +{ + /// The schema allows one value, so no question is asked about it. + Const, + + /// A Choice question selects one value of an enum. + Choice, + + /// A Noul question decides a Boolean. + Boolean, + + /// One Noul question per enum member decides which members an array includes. + Set, +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs new file mode 100644 index 00000000000..825e535cf36 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevArgumentPlan.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The questions asked about one tool argument and how its value is read from the answers. +/// +internal sealed class JevArgumentPlan +{ + public required string Name { get; init; } + + public required JevArgumentKind Kind { get; init; } + + /// + /// Gets the ID of the Noul question that decides whether an optional argument is passed at all, or + /// for a required argument. + /// + public string? PresenceQuestionId { get; init; } + + /// Gets the ID of the Choice or Noul question of a or argument. + public string? ValueQuestionId { get; init; } + + /// Gets the enum values of a argument, keyed by their Choice label. + public IReadOnlyDictionary Values { get; init; } = new Dictionary(); + + /// Gets the Noul question ID and the value of each member of a argument. + public IReadOnlyList> Members { get; init; } = []; + + /// Gets the value of a argument. + public JsonElement ConstValue { get; init; } + + /// + /// Reads the argument value from Jev's answers. + /// + /// + /// when an optional argument should be left out, so that the function uses its default. + /// + public bool TryDecode(JevResult result, out JsonElement value) + { + value = default; + if (this.PresenceQuestionId is not null && !JevToolCallPlan.IsYes(result, this.PresenceQuestionId)) + { + return false; + } + + switch (this.Kind) + { + case JevArgumentKind.Const: + value = this.ConstValue; + return true; + + case JevArgumentKind.Boolean: + value = JevJsonUtilities.CreateBoolean(JevToolCallPlan.IsYes(result, this.ValueQuestionId!)); + return true; + + case JevArgumentKind.Choice: + string label = JevToolCallPlan.GetChoice(result, this.ValueQuestionId!); + if (!this.Values.TryGetValue(label, out value)) + { + Throw.InvalidOperationException($"Jev returned the unknown choice '{label}' for the tool argument '{this.Name}'."); + } + + return true; + + case JevArgumentKind.Set: + value = JevJsonUtilities.CreateArray(this.Members.Where(member => JevToolCallPlan.IsYes(result, member.Key)).Select(member => member.Value)); + return true; + + default: + Throw.InvalidOperationException($"The argument plan of '{this.Name}' has the unknown kind {this.Kind}."); + return false; + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs new file mode 100644 index 00000000000..3d093ad6a39 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatClientLog.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// Source-generated log messages of . +internal static partial class JevChatClientLog +{ + [LoggerMessage( + EventId = 1, + Level = LogLevel.Warning, + Message = "The tool '{ToolName}' is left out of the Jev request because its parameters cannot be expressed as Jev questions: {Reason}")] + public static partial void LogToolExcluded(this ILogger logger, string toolName, string reason); +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs new file mode 100644 index 00000000000..bbbc5a84582 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs @@ -0,0 +1,279 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Converts a conversation into the state that Jev answers questions about, and reads the current turn of it. +/// +/// +/// The state has the shape of the Python TypeSafe connector: {"messages": [{"role", "contents": [...]}], +/// "instructions"}, where each content is text, text_reasoning, function_call, or +/// function_result. Tool approval requests and responses are left out: they are control messages, and the call +/// and result they lead to carry the information. The current turn is everything after the last user message that +/// starts a request: the tool calls and results that answer it. +/// +internal static class JevChatState +{ + public static JevEntry Build(IReadOnlyList messages, string? instructions) + { + var stateMessages = new JsonArray(); + for (int index = 0; index < messages.Count; index++) + { + ChatMessage message = messages[index]; + List unsupported = [.. message.Contents + .Where(content => content is not (TextContent or TextReasoningContent or FunctionCallContent or FunctionResultContent or ToolApprovalRequestContent or ToolApprovalResponseContent)) + .Select(content => content.GetType().Name) + .Distinct(StringComparer.Ordinal) + .OrderBy(name => name, StringComparer.Ordinal)]; + if (unsupported.Count > 0) + { + throw new NotSupportedException( + $"Jev supports text, reasoning text, and function call and result content only; message {index} contains {string.Join(", ", unsupported)}."); + } + + // Approval content is not written: FunctionInvokingChatClient keeps handled approvals in the history of + // later turns, and rejecting them would make every turn after an approval fail. + var contents = new JsonArray(); + foreach (AIContent content in message.Contents) + { + switch (content) + { + case TextContent { Text: { Length: > 0 } text }: + contents.Add((JsonNode)new JsonObject { ["type"] = "text", ["text"] = text }); + break; + + // Only the readable reasoning text is sent; protected (encrypted) reasoning is meaningful only to + // the model that produced it. + case TextReasoningContent { Text: { Length: > 0 } reasoning }: + contents.Add((JsonNode)new JsonObject { ["type"] = "text_reasoning", ["text"] = reasoning }); + break; + + case FunctionCallContent call: + contents.Add((JsonNode)new JsonObject + { + ["type"] = "function_call", + ["call_id"] = call.CallId, + ["name"] = call.Name, + ["arguments"] = call.Arguments is null ? null : ToObject(call.Arguments), + }); + break; + + case FunctionResultContent result: + contents.Add((JsonNode)SerializeResult(result)); + break; + } + } + + if (contents.Count > 0) + { + stateMessages.Add((JsonNode)new JsonObject { ["role"] = message.Role.Value, ["contents"] = contents }); + } + } + + if (stateMessages.Count == 0) + { + Throw.ArgumentException(nameof(messages), "Jev needs at least one message with text or function call content."); + } + + var state = new JsonObject { ["messages"] = stateMessages }; + if (!string.IsNullOrWhiteSpace(instructions)) + { + state["instructions"] = instructions; + } + + return new JevEntry(JevJsonUtilities.Parse(state.ToJsonString())); + } + + /// + /// Gets the index of the first message of the current turn, which follows the last user message that starts a + /// request. + /// + /// + /// User messages that directly follow a function result continue the turn instead of starting one. + /// FunctionInvokingChatClient moves text that was sent together with an approval response, such as "Yes, go + /// ahead", after the approved call and its result. Starting the turn there would hide the approved call from the + /// tool call limit and offer the tool again with no hint that it already ran. + /// + public static int GetCurrentTurnStart(IReadOnlyList messages) + { + int index = messages.Count - 1; + while (index >= 0) + { + if (messages[index].Role != ChatRole.User) + { + index--; + continue; + } + + // Text that followed several approval responses may span consecutive user messages. + int runStart = index; + while (runStart > 0 && messages[runStart - 1].Role == ChatRole.User) + { + runStart--; + } + + if (runStart > 0 && messages[runStart - 1].Contents.Any(content => content is FunctionResultContent)) + { + index = runStart - 1; + continue; + } + + return index + 1; + } + + return 0; + } + + /// Gets the argument sets of the calls in the current turn, by tool name, oldest first. + public static Dictionary>> GetCurrentTurnCalls(IReadOnlyList messages, int turnStart) + { + var calls = new Dictionary>>(StringComparer.Ordinal); + foreach (FunctionCallContent call in CurrentTurnContents(messages, turnStart)) + { + if (string.IsNullOrEmpty(call.Name)) + { + continue; + } + + var arguments = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair argument in call.Arguments ?? new Dictionary()) + { + arguments[argument.Key] = JevJsonUtilities.ToElement(argument.Value); + } + + if (!calls.TryGetValue(call.Name, out List>? list)) + { + calls[call.Name] = list = []; + } + + list.Add(arguments); + } + + return calls; + } + + /// Counts the function calls in the current turn. + public static int CountCurrentTurnCalls(IReadOnlyList messages, int turnStart) => + CurrentTurnContents(messages, turnStart).Count(); + + /// + /// Gets the text of each function result in the current turn, oldest first, unwrapping results of the form + /// {"result": "text"}. + /// + public static List GetCurrentTurnResultTexts(IReadOnlyList messages, int turnStart) => + [.. CurrentTurnContents(messages, turnStart).Select(result => GetResultText(result.Result)).OfType()]; + + private static IEnumerable CurrentTurnContents(IReadOnlyList messages, int turnStart) + where T : AIContent + { + for (int index = turnStart; index < messages.Count; index++) + { + foreach (T content in messages[index].Contents.OfType()) + { + yield return content; + } + } + } + + private static JsonObject ToObject(IDictionary arguments) + { + var json = new JsonObject(); + foreach (KeyValuePair argument in arguments) + { + json[argument.Key] = JevJsonUtilities.ToNodeOrText(argument.Value); + } + + return json; + } + + private static JsonObject SerializeResult(FunctionResultContent result) + { + var json = new JsonObject { ["type"] = "function_result", ["call_id"] = result.CallId }; + IEnumerable? items = result.Result switch + { + AIContent content => [content], + IEnumerable contents => contents, + _ => null, + }; + + if (items is null) + { + json["result"] = JevJsonUtilities.ToNodeOrText(result.Result); + return json; + } + + // Rich results are sent as text items, like the Python connector; other media cannot be described to Jev. + var texts = new List(); + foreach (AIContent item in items) + { + if (item is not TextContent text) + { + throw new NotSupportedException($"Jev supports text function results only; a result of the call '{result.CallId}' contains {item.GetType().Name}."); + } + + texts.Add(text.Text ?? string.Empty); + } + + json["result"] = string.Concat(texts); + var textItems = new JsonArray(); + foreach (string text in texts) + { + textItems.Add((JsonNode)new JsonObject { ["type"] = "text", ["text"] = text }); + } + + json["items"] = textItems; + return json; + } + + private static string? GetResultText(object? result) => + result switch + { + null => null, + string text => Unwrap(text), + JsonElement { ValueKind: JsonValueKind.String } element => Unwrap(element.GetString()!), + JsonElement { ValueKind: JsonValueKind.Null or JsonValueKind.Undefined } => null, + JsonElement element => Unwrap(element.GetRawText()), + TextContent content => content.Text, + IEnumerable contents => string.Concat(contents.OfType().Select(content => content.Text)), + _ => JevJsonUtilities.ToNodeOrText(result)?.ToJsonString(), + }; + + /// + /// Returns the text of a result written as {"result": "text"}, such as an MCP tool result, or the result + /// unchanged. + /// + private static string Unwrap(string result) + { + if (!result.AsSpan().TrimStart().StartsWith("{".AsSpan(), StringComparison.Ordinal)) + { + return result; + } + + try + { + using JsonDocument document = JsonDocument.Parse(result); + JsonElement root = document.RootElement; + if (root.ValueKind == JsonValueKind.Object && + root.EnumerateObject().Count() == 1 && + root.TryGetProperty("result", out JsonElement inner) && + inner.ValueKind == JsonValueKind.String) + { + return inner.GetString()!; + } + } + catch (JsonException) + { + // Text that only looks like JSON is returned as it is. + } + + return result; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs new file mode 100644 index 00000000000..cd4a568618b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevCompiledTool.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// One tool that can be expressed as Jev questions: its route label, its argument questions, and their decoders. +/// +internal sealed class JevCompiledTool +{ + public required AIFunctionDeclaration Function { get; init; } + + /// Gets the label of the tool in the route question, such as t0. + public required string RouteLabel { get; init; } + + public required IReadOnlyDictionary Questions { get; init; } + + public required IReadOnlyList Arguments { get; init; } + + public Dictionary DecodeArguments(JevResult result) + { + var arguments = new Dictionary(this.Arguments.Count); + foreach (JevArgumentPlan argument in this.Arguments) + { + if (argument.TryDecode(result, out JsonElement value)) + { + arguments[argument.Name] = value; + } + } + + return arguments; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs index 3188ff4333c..c27a2db5f0d 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevContractValidator.cs @@ -9,142 +9,132 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Checks requests before they are evaluated and responses before they are returned to the caller. +/// Checks requests before they are sent and results before they are returned to the caller. /// /// -/// Requests come from a model, so their errors are reported as messages that say how to -/// fix the arguments. Responses come from an evaluator, possibly a custom client with its own mappers, so their errors -/// are reported as ; they point at an evaluator or mapper bug, not at the model. +/// +/// Request rules follow the official TypeSafe SDK (at least one question; a Score rubric of at least two entries) and +/// the limits the API rejects (at most 255 Choice alternatives and 10 Score entries; a Noul needs instructions or +/// criteria; the state and Score rubric entries may not be null, although the SDK types allow it). Checking them here +/// saves a round trip and gives the model a message that says what to fix. +/// +/// +/// Requests come from a model, so their errors are . Results come from the API or from +/// a custom mapping, so their errors are ; they point at a mapping bug, not at +/// the model. +/// /// internal static class JevContractValidator { - /// The most options that the Jev API accepts in one Choice question. - public const int MaxChoiceOptions = 255; - - /// - /// The fewest levels of a Score question. The API also accepts a single level, but the documentation requires two, - /// and a one-level score always returns that level, so it is rejected as a request mistake. - /// - public const int MinScoreLevels = 2; - - /// The most levels that the Jev API accepts in one Score question. - public const int MaxScoreLevels = 10; + public const int MaxChoiceCriteria = 255; + public const int MinScoreCriteria = 2; + public const int MaxScoreCriteria = 10; // Jev computes scores and probabilities in floating point, so values may exceed their bounds by rounding error. private const double Tolerance = 1e-6; public static void ValidateRequest(JevRequest request) { - if (request.State.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array)) + // The SDK types allow a null state, but the API answers 422 ("Field required"). + if (request.State.IsNull) { - Throw.ArgumentException(nameof(request), "The state must be a string, a JSON object, or a JSON array."); + Throw.ArgumentException(nameof(request), "The state must be text or a JSON object or array."); } if (request.Questions is null || request.Questions.Count == 0) { - Throw.ArgumentException(nameof(request), "The request must contain at least one question."); + Throw.ArgumentException(nameof(request), "At least one question is required."); } foreach (KeyValuePair entry in request.Questions) { string id = entry.Key; - if (string.IsNullOrWhiteSpace(id)) - { - Throw.ArgumentException(nameof(request), "Question IDs must not be empty."); - } - - JevQuestion question = entry.Value; - if (question is null) - { - Throw.ArgumentException(nameof(request), $"Question '{id}' is null."); - } - - if (string.IsNullOrWhiteSpace(question.Instructions)) + switch (entry.Value) { - Throw.ArgumentException(nameof(request), $"Question '{id}' must have instructions."); - } + case null: + Throw.ArgumentException(nameof(request), $"Question '{id}' is null."); + break; - switch (question) - { case JevChoiceQuestion choice: - int options = choice.Criteria?.Count ?? 0; - if (options is < 1 or > MaxChoiceOptions) - { - Throw.ArgumentException(nameof(request), $"Choice question '{id}' has {Count(options, "option")}; it needs from 1 to {MaxChoiceOptions}."); - } - - foreach (string option in choice.Criteria!.Keys) + int alternatives = choice.Criteria?.Count ?? 0; + if (alternatives is < 1 or > MaxChoiceCriteria) { - if (string.IsNullOrWhiteSpace(option)) - { - Throw.ArgumentException(nameof(request), $"Choice question '{id}' has an empty option name."); - } + Throw.ArgumentException(nameof(request), $"Choice question '{id}' has {Count(alternatives, "alternative")}; it needs from 1 to {MaxChoiceCriteria}."); } break; case JevScoreQuestion score: - int levels = score.Criteria?.Count ?? 0; - if (levels is < MinScoreLevels or > MaxScoreLevels) + int scores = score.Criteria?.Count ?? 0; + if (scores is < MinScoreCriteria or > MaxScoreCriteria) { - Throw.ArgumentException(nameof(request), $"Score question '{id}' has {Count(levels, "level")}; it needs from {MinScoreLevels} to {MaxScoreLevels}."); + Throw.ArgumentException(nameof(request), $"Score question '{id}' has {Count(scores, "criterion", "criteria")}; it needs from {MinScoreCriteria} to {MaxScoreCriteria}."); } - foreach (string level in score.Criteria!) + // The SDK types allow null rubric entries, but the API answers 422 for them. + for (int index = 0; index < scores; index++) { - if (string.IsNullOrWhiteSpace(level)) + if (score.Criteria![index].IsNull) { - Throw.ArgumentException(nameof(request), $"Score question '{id}' has an empty level description."); + Throw.ArgumentException(nameof(request), $"Score question '{id}' has a null criterion at index {index.ToString(CultureInfo.InvariantCulture)}; every score needs text or JSON."); } } + break; + + case JevNoulQuestion noul: + if (IsEmpty(noul.Instructions) && IsEmpty(noul.Criteria?.True) && IsEmpty(noul.Criteria?.False)) + { + Throw.ArgumentException(nameof(request), $"Noul question '{id}' needs instructions or criteria."); + } + break; } } } - public static void ValidateResponse(JevRequest request, JevResponse response) + public static void ValidateResult(JevRequest request, JevResult result) { - if (response.Answers is null) + if (result.Answers is null) { - Throw.InvalidOperationException("The Jev response has no answers."); + Throw.InvalidOperationException("The Jev result has no answers."); } foreach (KeyValuePair entry in request.Questions) { string id = entry.Key; - if (!response.Answers.TryGetValue(id, out JevAnswer? answer) || answer is null) + if (!result.Answers.TryGetValue(id, out JevResponse? answer) || answer is null) { - Throw.InvalidOperationException($"The Jev response has no answer for question '{id}'."); + Throw.InvalidOperationException($"The Jev result has no answer for question '{id}'."); } switch (entry.Value, answer) { - case (JevChoiceQuestion question, JevChoiceAnswer choice): - // A choice outside the options would reach the model as a value it never offered, which is how a - // mapper that converts between option names and a client's own values would fail silently. + case (JevChoiceQuestion question, JevChoiceResponse choice): + // A label outside the criteria would reach the model as a value it never offered, which is how a + // mapping that converts between labels and a client's own values would fail silently. if (choice.Choice is null || !question.Criteria.ContainsKey(choice.Choice)) { - Throw.InvalidOperationException($"The answer to question '{id}' chose '{choice.Choice}', which is not one of its options."); + Throw.InvalidOperationException($"The answer to question '{id}' chose '{choice.Choice}', which is not one of its labels."); } EnsureInRange(choice.Confidence, 0, 1, id, "confidence"); - EnsureProbabilities(choice.Probabilities, id, key => question.Criteria.ContainsKey(key), "one of its options"); + EnsureProbabilities(choice.Probabilities, id, key => question.Criteria.ContainsKey(key), "one of its labels"); break; - case (JevScoreQuestion question, JevScoreAnswer score): + case (JevScoreQuestion question, JevScoreResponse score): EnsureInRange(score.Score, 0, question.Criteria.Count - 1, id, "score"); EnsureInRange(score.Confidence, 0, 1, id, "confidence"); - // Score answers key their probabilities and legend by level number ("0", "1", ...). - int levels = question.Criteria.Count; - bool IsLevel(string key) => int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out int level) && level < levels; - string levelRange = $"a level number from 0 to {(levels - 1).ToString(CultureInfo.InvariantCulture)}"; - EnsureProbabilities(score.Probabilities, id, IsLevel, levelRange); - EnsureKeys(score.Legend?.Keys, id, "legend", IsLevel, levelRange); + // Score responses key their probabilities and legend by score ("0", "1", ...). + int scores = question.Criteria.Count; + bool IsScore(string key) => int.TryParse(key, NumberStyles.None, CultureInfo.InvariantCulture, out int value) && value < scores; + string scoreRange = $"a score from 0 to {(scores - 1).ToString(CultureInfo.InvariantCulture)}"; + EnsureProbabilities(score.Probabilities, id, IsScore, scoreRange); + EnsureKeys(score.Legend?.Keys, id, "legend", IsScore, scoreRange); break; - case (JevNoulQuestion, JevNoulAnswer noul): + case (JevNoulQuestion, JevNoulResponse noul): EnsureInRange(noul.Noul, 0, 1, id, "noul"); break; @@ -155,9 +145,12 @@ public static void ValidateResponse(JevRequest request, JevResponse response) } } + private static bool IsEmpty(JevEntry? entry) => + entry is not { } value || value.IsNull || (value.Kind == JsonValueKind.String && string.IsNullOrWhiteSpace(value.Text)); + /// - /// Checks that every probability is in range and is keyed by something the question offered. A mapper that fills - /// probabilities with a client's own labels would otherwise show the model options it never asked about. + /// Checks that every probability is in range and is keyed by something the question offered. A mapping that fills + /// probabilities with a client's own labels would otherwise show the model alternatives it never asked about. /// private static void EnsureProbabilities(IReadOnlyDictionary? probabilities, string id, Func isValidKey, string expected) { @@ -200,15 +193,15 @@ private static void EnsureInRange(double value, double min, double max, string i } } - private static string Count(int count, string noun) => - count == 1 ? $"1 {noun}" : $"{count.ToString(CultureInfo.InvariantCulture)} {noun}s"; + private static string Count(int count, string singular, string? plural = null) => + count == 1 ? $"1 {singular}" : $"{count.ToString(CultureInfo.InvariantCulture)} {plural ?? singular + "s"}"; private static string KindOf(object value) => value switch { - JevChoiceQuestion or JevChoiceAnswer => "Choice", - JevScoreQuestion or JevScoreAnswer => "Score", - JevNoulQuestion or JevNoulAnswer => "Noul", + JevChoiceQuestion or JevChoiceResponse => "Choice", + JevScoreQuestion or JevScoreResponse => "Score", + JevNoulQuestion or JevNoulResponse => "Noul", _ => value.GetType().Name, }; } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs new file mode 100644 index 00000000000..3287e38784f --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevEntryJsonConverter.cs @@ -0,0 +1,39 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Reads and writes a as the JSON value it holds. +/// +internal sealed class JevEntryJsonConverter : JsonConverter +{ + // A JSON null is a valid entry (an undescribed criterion), so the converter reads it instead of the serializer. + public override bool HandleNull => true; + + public override JevEntry Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonElement value = JsonElement.ParseValue(ref reader); + if (value.ValueKind is JsonValueKind.Number or JsonValueKind.True or JsonValueKind.False) + { + throw new JsonException("A Jev entry must be text, a JSON object or array, or null."); + } + + return new JevEntry(value); + } + + public override void Write(Utf8JsonWriter writer, JevEntry value, JsonSerializerOptions options) + { + if (value.Json is { } json) + { + json.WriteTo(writer); + } + else + { + writer.WriteNullValue(); + } + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpClient.cs similarity index 66% rename from dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs rename to dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpClient.cs index 02104e5fed6..c00c821cae2 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpEvaluator.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevHttpClient.cs @@ -11,21 +11,19 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Evaluates a with the TypeSafe System One API through a System.ClientModel pipeline. +/// Sends a to POST /v1/systemone of the TypeSafe API through a System.ClientModel pipeline, +/// like systemOne of the official TypeSafe SDK. /// -internal sealed class JevHttpEvaluator +internal sealed class JevHttpClient { - private const string DefaultEndpoint = "https://api.typesafe.ai/v1/systemone"; private const int MaxErrorDetailLength = 500; private readonly ClientPipeline _pipeline; - private readonly Uri _endpoint; - private readonly string _modelId; - public JevHttpEvaluator(ApiKeyCredential credential, JevAIToolOptions options) + public JevHttpClient(ApiKeyCredential credential, ClientPipelineOptions options, Uri? endpoint, string modelId) { - this._endpoint = options.Endpoint ?? new Uri(DefaultEndpoint); - this._modelId = options.ModelId; + this.Endpoint = endpoint ?? new Uri(JevPipelineDefaults.DefaultEndpoint); + this.ModelId = modelId; // The credential is applied per try, like the OpenAI client does, so a retried request is signed again. // Creating the pipeline freezes the options, as with any other client built from them. @@ -36,7 +34,22 @@ public JevHttpEvaluator(ApiKeyCredential credential, JevAIToolOptions options) beforeTransportPolicies: []); } - public async Task EvaluateAsync(JevRequest request, IServiceProvider? services, CancellationToken cancellationToken) + /// Gets the System One endpoint that requests are sent to. + public Uri Endpoint { get; } + + /// Gets the model that answers requests unless a call selects another. + public string ModelId { get; } + + /// + /// Answers a request with the configured model. The signature matches the mapping of . + /// + public Task SystemOneAsync(JevRequest request, IServiceProvider? services, CancellationToken cancellationToken) => + this.SystemOneAsync(request, this.ModelId, cancellationToken); + + /// + /// Answers a request with the given model. + /// + public async Task SystemOneAsync(JevRequest request, string modelId, CancellationToken cancellationToken) { using PipelineMessage message = this._pipeline.CreateMessage(); message.ResponseClassifier = JevResponseClassifier.Instance; @@ -44,11 +57,11 @@ public async Task EvaluateAsync(JevRequest request, IServiceProvide PipelineRequest httpRequest = message.Request; httpRequest.Method = "POST"; - httpRequest.Uri = this._endpoint; + httpRequest.Uri = this.Endpoint; httpRequest.Headers.Set("Accept", "application/json"); httpRequest.Headers.Set("Content-Type", "application/json"); httpRequest.Content = BinaryContent.Create(BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes( - new JevApiRequest { State = request.State, Model = this._modelId, Questions = request.Questions }, + new JevApiRequest { State = request.State, Model = modelId, Questions = request.Questions }, JevJsonContext.Default.JevApiRequest))); // Transports leave the cancellation check to the HTTP handler, so a canceled call is stopped here before @@ -69,14 +82,14 @@ public async Task EvaluateAsync(JevRequest request, IServiceProvide response); } - return ReadResponse(response.Content); + return ReadResult(response.Content); } - private static JevResponse ReadResponse(BinaryData content) + private static JevResult ReadResult(BinaryData content) { try { - return JsonSerializer.Deserialize(content.ToMemory().Span, JevJsonContext.Default.JevResponse) + return JsonSerializer.Deserialize(content.ToMemory().Span, JevJsonContext.Default.JevResult) ?? throw new JsonException("The response body is null."); } catch (Exception ex) when (ex is JsonException or NotSupportedException) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs index 4eb54bccb97..2a7c2e73b64 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonContext.cs @@ -14,7 +14,8 @@ namespace Microsoft.Agents.AI.TypeSafe; /// [JsonSourceGenerationOptions(AllowOutOfOrderMetadataProperties = true)] [JsonSerializable(typeof(JevRequest))] -[JsonSerializable(typeof(JevResponse))] +[JsonSerializable(typeof(JevResult))] [JsonSerializable(typeof(JevApiRequest))] [JsonSerializable(typeof(Dictionary))] +[JsonSerializable(typeof(string))] internal sealed partial class JevJsonContext : JsonSerializerContext; diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs index 8e8ff710492..affd4f95ec6 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevJsonUtilities.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Buffers; +using System.Collections.Generic; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; @@ -9,12 +11,16 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Serializer options and JSON schemas shared by the Jev tool. +/// Serializer options, JSON schemas, and JSON value helpers shared by the Jev tool and chat client. /// internal static class JevJsonUtilities { - private static readonly Lazy s_requestSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevRequest), serializerOptions: Options)); - private static readonly Lazy s_responseSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevResponse), serializerOptions: Options)); + private static readonly JsonElement s_true = Parse("true"); + private static readonly JsonElement s_false = Parse("false"); + private static readonly JsonElement s_null = Parse("null"); + private static readonly AIJsonSchemaCreateOptions s_schemaOptions = new() { TransformSchemaNode = DescribeEntries }; + private static readonly Lazy s_requestSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevRequest), serializerOptions: Options, inferenceOptions: s_schemaOptions)); + private static readonly Lazy s_resultSchema = new(static () => AIJsonUtilities.CreateJsonSchema(typeof(JevResult), serializerOptions: Options, inferenceOptions: s_schemaOptions)); /// /// Gets the options that resolve the Jev contract types first and every other type through @@ -25,8 +31,8 @@ internal static class JevJsonUtilities /// Gets the JSON schema of the tool arguments, which is the schema of . public static JsonElement RequestSchema => s_requestSchema.Value; - /// Gets the JSON schema of the tool result, which is the schema of . - public static JsonElement ResponseSchema => s_responseSchema.Value; + /// Gets the JSON schema of the tool result, which is the schema of . + public static JsonElement ResultSchema => s_resultSchema.Value; /// /// Converts one tool argument into a JSON node. Chat clients pass arguments as values; @@ -47,6 +53,62 @@ internal static class JevJsonUtilities _ => JsonSerializer.SerializeToNode(value, Options.GetTypeInfo(value.GetType())), }; + /// + /// Converts a value from a chat message, such as a function argument or result, into a JSON node. A value that + /// cannot be serialized, for example a type without metadata under native AOT, becomes its text. + /// + public static JsonNode? ToNodeOrText(object? value) + { + try + { + return ToNode(value); + } + catch (Exception ex) when (ex is NotSupportedException or InvalidOperationException or JsonException or ArgumentException) + { + // Chat history is informational for Jev, so an unserializable value is described rather than failing + // the whole request. + return JsonValue.Create(value?.ToString()); + } + } + + /// Converts a value from a chat message into a detached , like . + public static JsonElement ToElement(object? value) => + value switch + { + JsonElement element => element.ValueKind == JsonValueKind.Undefined ? s_null : element.Clone(), + null => s_null, + _ => Parse(ToNodeOrText(value)?.ToJsonString() ?? "null"), + }; + + /// Gets the JSON or value. + public static JsonElement CreateBoolean(bool value) => value ? s_true : s_false; + + /// Writes values into a new JSON array. + public static JsonElement CreateArray(IEnumerable values) + { + var buffer = new ArrayBufferWriter(); + using (var writer = new Utf8JsonWriter(buffer)) + { + writer.WriteStartArray(); + foreach (JsonElement value in values) + { + value.WriteTo(writer); + } + + writer.WriteEndArray(); + } + + using JsonDocument document = JsonDocument.Parse(buffer.WrittenMemory); + return document.RootElement.Clone(); + } + + /// Parses JSON text into a detached . + public static JsonElement Parse(string json) + { + using JsonDocument document = JsonDocument.Parse(json); + return document.RootElement.Clone(); + } + private static JsonSerializerOptions CreateOptions() { var options = new JsonSerializerOptions(JevJsonContext.Default.Options) @@ -57,4 +119,62 @@ private static JsonSerializerOptions CreateOptions() options.MakeReadOnly(); return options; } + + /// + /// Gives the schema of the SDK's EntryType, narrowed to what the API accepts. Its + /// custom converter hides the shape from the schema generator, which would otherwise allow any JSON value, and + /// entries inside collections are only reachable from the collection's node. + /// + /// + /// The API rejects a state and Score rubric entries even though the + /// SDK types allow them, so those schemas leave out; the size limits tell the model how many + /// alternatives and scores it may send. + /// + private static JsonNode DescribeEntries(AIJsonSchemaCreateContext context, JsonNode schema) + { + Type type = context.TypeInfo.Type; + if (type == typeof(JevEntry)) + { + // JevRequest.State is the only non-nullable single entry. + return WithEntryType(schema, allowNull: false); + } + + if (type == typeof(JevEntry?)) + { + return WithEntryType(schema, allowNull: true); + } + + if (type == typeof(IReadOnlyList) && schema is JsonObject rubric) + { + rubric["items"] = WithEntryType(new JsonObject(), allowNull: false); + rubric["minItems"] = JevContractValidator.MinScoreCriteria; + rubric["maxItems"] = JevContractValidator.MaxScoreCriteria; + return rubric; + } + + if (type == typeof(IReadOnlyDictionary) && schema is JsonObject map) + { + // Choice criteria may leave an alternative undescribed; a Score legend echoes the rubric, which may not. + bool isChoiceCriteria = context.PropertyInfo?.Name == "criteria"; + map["additionalProperties"] = WithEntryType(new JsonObject(), allowNull: isChoiceCriteria); + if (isChoiceCriteria) + { + map["minProperties"] = 1; + map["maxProperties"] = JevContractValidator.MaxChoiceCriteria; + } + + return map; + } + + return schema; + } + + private static JsonObject WithEntryType(JsonNode schema, bool allowNull) + { + JsonObject entry = schema as JsonObject ?? []; + entry["type"] = allowNull + ? new JsonArray("string", "object", "array", "null") + : new JsonArray("string", "object", "array"); + return entry; + } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs new file mode 100644 index 00000000000..06476e6cec8 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevPipelineDefaults.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Assigns the pipeline defaults of the official TypeSafe SDK to the options of every Jev client. +/// +internal static class JevPipelineDefaults +{ + public const string DefaultEndpoint = "https://api.typesafe.ai/v1/systemone"; + public const string DefaultModelId = "jev-latest"; + + private static readonly TimeSpan s_defaultNetworkTimeout = TimeSpan.FromSeconds(10); + + /// + /// Sets the SDK retry policy and network timeout where the caller left them unset. + /// + /// + /// Call this from , not from a constructor, because + /// may be set after construction. ClientPipeline.Create + /// freezes the options before it reads them, so the defaults are in place when the pipeline is built. + /// + public static void Apply(ClientPipelineOptions options) + { + options.RetryPolicy ??= new JevRetryPolicy( + JevRetryPolicy.DefaultMaxRetries, + options.ClientLoggingOptions?.EnableLogging ?? true, + options.ClientLoggingOptions?.LoggerFactory); + options.NetworkTimeout ??= s_defaultNetworkTimeout; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs new file mode 100644 index 00000000000..f6fe3f278cf --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Globalization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Counts the internal questions that tool routing adds to a request, and stops the request when they exceed +/// . +/// +/// +/// Each tool is compiled against a copy of the budget, which is committed only when the tool compiles. A tool that +/// is excluded for an unsupported schema therefore returns the questions it reserved to the tools after it. +/// +internal sealed class JevQuestionBudget +{ + public const int MaxInternalQuestions = 128; + + public JevQuestionBudget(int count = 0) + { + this.Count = count; + } + + public int Count { get; set; } + + public void Reserve(int count) + { + int next = this.Count + count; + if (next > MaxInternalQuestions) + { + // Too many questions is a property of the whole tool set, not of one tool, so it stops the request instead + // of excluding the tool that crossed the limit. + Throw.InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "The tools need more than {0} internal Jev questions. Offer fewer tools, or tools with fewer arguments.", + MaxInternalQuestions)); + } + + this.Count = next; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs index adf42d6ac73..8677e243a61 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevResponseClassifier.cs @@ -6,17 +6,16 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Classifies System One responses: any 2xx status is a success, and 529 is retriable in addition to the statuses -/// that retries (408, 429, 500, 502, 503, and 504). +/// Classifies System One responses like the official TypeSafe SDK: any 2xx status is a success, and 408, 429, and +/// every 5xx status are retriable. /// /// -/// TypeSafe answers 529 when it is overloaded and documents it as transient, but the default classifier treats it as a -/// permanent error. Every other case, including network failures, is left to the default classifier. +/// only retries 500, 502, 503, and 504 among the 5xx statuses, which +/// leaves out 529, the status TypeSafe answers when it is overloaded. Network failures are left to the default +/// classifier, which retries them, as the SDK does. /// internal sealed class JevResponseClassifier : PipelineMessageClassifier { - private const int OverloadedStatus = 529; - public static JevResponseClassifier Instance { get; } = new(); public override bool TryClassify(PipelineMessage message, out bool isError) @@ -33,7 +32,13 @@ public override bool TryClassify(PipelineMessage message, out bool isError) public override bool TryClassify(PipelineMessage message, Exception? exception, out bool isRetriable) { - isRetriable = exception is null && message.Response?.Status == OverloadedStatus; - return isRetriable; + if (exception is not null || message.Response is not { } response) + { + isRetriable = false; + return false; + } + + isRetriable = response.Status is 408 or 429 or (>= 500 and <= 599); + return true; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs index b54b892ce09..0b5725db567 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevRetryPolicy.cs @@ -11,18 +11,30 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// The default retry policy of : the standard , except -/// that it never waits longer than . +/// The default retry policy of the Jev clients, with the defaults of the official TypeSafe SDK +/// (DEFAULT_RETRY_POLICY in retry.ts). /// /// -/// waits for whatever Retry-After the server sends. Inside a tool call that -/// wait blocks the agent run with no feedback, and a very large value makes the delay itself throw, so a longer -/// request fails at once with the service's status instead. +/// +/// Up to retries after the first attempt. +/// +/// A server delay from retry-after-ms or Retry-After is honored up to . +/// A longer delay falls back to backoff, because waiting it would block the agent run with no feedback. +/// +/// +/// Otherwise the delay is an exponential backoff from , capped at , +/// minus up to of random jitter so that concurrent callers do not retry in lockstep. +/// +/// +/// Which statuses are retried is decided by . /// internal sealed class JevRetryPolicy : ClientRetryPolicy { - public const int DefaultMaxRetries = 3; + public const int DefaultMaxRetries = 2; + public const double BackoffJitter = 0.25; + public static readonly TimeSpan InitialBackoff = TimeSpan.FromMilliseconds(500); + public static readonly TimeSpan MaxBackoff = TimeSpan.FromSeconds(5); public static readonly TimeSpan MaxRetryAfter = TimeSpan.FromMinutes(1); public JevRetryPolicy(int maxRetries = DefaultMaxRetries, bool enableLogging = true, ILoggerFactory? loggerFactory = null) @@ -36,11 +48,20 @@ public JevRetryPolicy(int maxRetries = DefaultMaxRetries, bool enableLogging = t /// internal List? RecordedWaitsForTests { get; set; } - protected override bool ShouldRetry(PipelineMessage message, Exception? exception) => - base.ShouldRetry(message, exception) && IsRetryAfterWithinLimit(message); + protected override TimeSpan GetNextDelay(PipelineMessage message, int tryCount) + { + if (TryGetServerDelayMilliseconds(message.Response, out double serverDelay) && serverDelay <= MaxRetryAfter.TotalMilliseconds) + { + return TimeSpan.FromMilliseconds(serverDelay); + } - protected override async ValueTask ShouldRetryAsync(PipelineMessage message, Exception? exception) => - await base.ShouldRetryAsync(message, exception).ConfigureAwait(false) && IsRetryAfterWithinLimit(message); + // tryCount is 1 for the first retry, matching attempt 0 of the SDK's retryDelayMs. + double exponential = Math.Min(InitialBackoff.TotalMilliseconds * Math.Pow(2, tryCount - 1), MaxBackoff.TotalMilliseconds); +#pragma warning disable CA5394 // Jitter only spreads retries over time; it needs no cryptographic randomness. + double jitter = Random.Shared.NextDouble() * BackoffJitter; +#pragma warning restore CA5394 + return TimeSpan.FromMilliseconds(Math.Round(exponential * (1 - jitter))); + } protected override Task WaitAsync(TimeSpan time, CancellationToken cancellationToken) { @@ -64,24 +85,53 @@ protected override void Wait(TimeSpan time, CancellationToken cancellationToken) base.Wait(time, cancellationToken); } - private static bool IsRetryAfterWithinLimit(PipelineMessage message) + /// + /// Reads retry-after-ms, preferred, or Retry-After in seconds or as an HTTP date, like the SDK's + /// parseRetryAfter, as a number of milliseconds. + /// + /// + /// The delay stays a until it is compared with : a huge value such as + /// 1e300 or the text NaN would make throw and fail the call, where the SDK + /// simply backs off. + /// + private static bool TryGetServerDelayMilliseconds(PipelineResponse? response, out double delay) { - if (message.Response?.Headers.TryGetValue("Retry-After", out string? value) is not true || string.IsNullOrWhiteSpace(value)) + delay = 0; + if (response is null) + { + return false; + } + + if (response.Headers.TryGetValue("retry-after-ms", out string? milliseconds) && + double.TryParse(milliseconds, NumberStyles.Float, CultureInfo.InvariantCulture, out double ms) && + double.IsFinite(ms) && ms >= 0) { + delay = ms; return true; } - // Retry-After is either a number of seconds or an HTTP date. - if (double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds)) + if (!response.Headers.TryGetValue("Retry-After", out string? retryAfter) || string.IsNullOrWhiteSpace(retryAfter)) { - return seconds <= MaxRetryAfter.TotalSeconds; + return false; } - if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) + if (double.TryParse(retryAfter, NumberStyles.Float, CultureInfo.InvariantCulture, out double seconds)) { - return date - DateTimeOffset.UtcNow <= MaxRetryAfter; + if (!double.IsFinite(seconds) || seconds < 0) + { + return false; + } + + delay = seconds * 1000; + return double.IsFinite(delay); + } + + if (DateTimeOffset.TryParse(retryAfter, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset date)) + { + delay = Math.Max(0, (date - DateTimeOffset.UtcNow).TotalMilliseconds); + return true; } - return true; + return false; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs new file mode 100644 index 00000000000..ef5e2807a84 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCall.cs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The tool call that Jev selected: the function and the arguments decoded from its answers. +/// +internal readonly record struct JevToolCall(AIFunctionDeclaration Function, Dictionary Arguments); diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs new file mode 100644 index 00000000000..4661cd6e096 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs @@ -0,0 +1,424 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Turns function tools into Jev questions, so that Jev, which cannot write arguments, can still call a tool. +/// +/// +/// +/// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same +/// question IDs, limits, and wording, so both connectors ask Jev the same questions. +/// +/// +/// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum +/// (a Choice), a Boolean (a Noul), or an array of enum values (one Noul per value). An optional argument also gets a +/// presence Noul, so that it is passed only when the user asked for it. A tool with any other argument, such as free +/// text or a number, cannot be expressed this way; it is logged and left out, or fails the request when the tool mode +/// requires a call and no tool remains. +/// +/// +internal static class JevToolCallCompiler +{ + public const string QuestionPrefix = "__af_tool__"; + public const string RouteQuestionId = QuestionPrefix + ".route"; + public const string RouteNone = "none"; + public const int MaxRoutableTools = 32; + public const int MaxToolProperties = 64; + public const int MaxEnumValues = 64; + + /// + /// Rejects caller question IDs that could collide with the internal tool questions. + /// + public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) + { + List reserved = [.. questionIds.Where(id => id.StartsWith(QuestionPrefix, StringComparison.Ordinal)).OrderBy(id => id, StringComparer.Ordinal)]; + if (reserved.Count > 0) + { + Throw.ArgumentException("questions", $"Question IDs that start with '{QuestionPrefix}' are reserved for tool calls: {string.Join(", ", reserved)}."); + } + } + + /// + /// Compiles the tools that the tool mode allows into a tool call plan. + /// + /// The function tools of the request. + /// The tool mode of the request; means . + /// + /// The argument sets of the tools already called in the current turn, by tool name, oldest first. They become + /// hints that keep Jev from repeating a call. + /// + /// Receives a warning for each tool that is left out. + /// The plan, or when no tool can be called. + public static JevToolCallPlan? Compile( + IReadOnlyList tools, + ChatToolMode? toolMode, + IReadOnlyDictionary>> previousCalls, + ILogger logger) + { + if (toolMode is NoneChatToolMode) + { + return null; + } + + bool required = toolMode is RequiredChatToolMode; + string? requiredName = (toolMode as RequiredChatToolMode)?.RequiredFunctionName; + if (tools.Count == 0) + { + if (required) + { + Throw.InvalidOperationException("The tool mode requires a tool call, but the request has no tools."); + } + + return null; + } + + List selected = [.. tools.Where(tool => requiredName is null || tool.Name == requiredName)]; + if (requiredName is not null && selected.Count == 0) + { + Throw.InvalidOperationException($"The tool mode requires the tool '{requiredName}', but the request does not have it."); + } + + if (selected.Count > MaxRoutableTools) + { + Throw.InvalidOperationException(string.Format( + CultureInfo.InvariantCulture, + "Jev can choose among at most {0} tools per request, but the request has {1}. Offer fewer tools.", + MaxRoutableTools, + selected.Count)); + } + + var compiled = new List(selected.Count); + var unsupported = new List>(); + var budget = new JevQuestionBudget(); + for (int index = 0; index < selected.Count; index++) + { + AIFunctionDeclaration tool = selected[index]; + + // Each tool reserves questions from a copy of the budget, which is committed only when the tool compiles. + var toolBudget = new JevQuestionBudget(budget.Count); + try + { + List> calls = previousCalls.TryGetValue(tool.Name, out List>? found) ? found : []; + compiled.Add(CompileTool(tool, index, calls, toolBudget)); + budget.Count = toolBudget.Count; + } + catch (JevUnsupportedToolSchemaException ex) + { + unsupported.Add(new(tool.Name, ex.Message)); + } + } + + foreach (KeyValuePair tool in unsupported) + { + logger.LogToolExcluded(tool.Key, tool.Value); + } + + if (required && compiled.Count == 0) + { + string details = string.Join("; ", unsupported.Select(tool => $"{tool.Key}: {tool.Value}")); + Throw.InvalidOperationException($"The tool mode requires a tool call, but no tool can be expressed as Jev questions. {details}".TrimEnd()); + } + + if (compiled.Count == 0) + { + return null; + } + + // With a single tool that must be called, there is nothing to route. + JevCompiledTool? requiredTool = required && compiled.Count == 1 ? compiled[0] : null; + var questions = new Dictionary(StringComparer.Ordinal); + if (requiredTool is null) + { + var criteria = new Dictionary(StringComparer.Ordinal); + foreach (JevCompiledTool tool in compiled) + { + string name = tool.Function.Name; + string description = string.IsNullOrWhiteSpace(tool.Function.Description) + ? $"Call the '{name}' tool." + : $"Tool '{name}': {tool.Function.Description}"; + if (previousCalls.TryGetValue(name, out List>? calls) && calls.Count > 0) + { + description += + " Call this tool again only if the latest user request still needs a distinct execution. " + + $"Do not repeat argument sets already called this turn: {JevToolSchema.Describe(calls)}."; + } + + criteria[tool.RouteLabel] = description; + } + + if (!required) + { + criteria[RouteNone] = "Do not call a tool because all requested tool actions are already satisfied, or because no tool is needed."; + } + + budget.Reserve(1); + questions[RouteQuestionId] = new JevChoiceQuestion + { + Instructions = "Which available tool, if any, should handle the latest user request?", + Criteria = criteria, + }; + } + + foreach (JevCompiledTool tool in compiled) + { + foreach (KeyValuePair question in tool.Questions) + { + questions[question.Key] = question.Value; + } + } + + return new JevToolCallPlan + { + Questions = questions, + Tools = compiled, + RouteQuestionId = requiredTool is null ? RouteQuestionId : null, + RequiredTool = requiredTool, + }; + } + + private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolIndex, List> previousCalls, JevQuestionBudget budget) + { + JsonElement schemaElement = tool.JsonSchema; + Dictionary schema = schemaElement.ValueKind switch + { + JsonValueKind.Object => JevToolSchema.ToMap(schemaElement), + JsonValueKind.Undefined or JsonValueKind.Null => [], + _ => throw new JevUnsupportedToolSchemaException("the top-level input schema must be an object"), + }; + + JevToolSchema.RejectUnsupportedKeys(schema, JevToolSchema.RootKeys, "root-level"); + if (schema.TryGetValue("type", out JsonElement type) && type.ValueKind != JsonValueKind.Null && !(type.ValueKind == JsonValueKind.String && type.ValueEquals("object"))) + { + throw new JevUnsupportedToolSchemaException("the top-level input schema must be an object"); + } + + List properties = []; + if (schema.TryGetValue("properties", out JsonElement propertiesElement) && propertiesElement.ValueKind != JsonValueKind.Null) + { + if (propertiesElement.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException("the input schema properties must be an object"); + } + + properties.AddRange(propertiesElement.EnumerateObject()); + } + + if (properties.Count > MaxToolProperties) + { + throw new JevUnsupportedToolSchemaException(string.Format( + CultureInfo.InvariantCulture, + "the input schema defines {0} properties; the supported maximum is {1}", + properties.Count, + MaxToolProperties)); + } + + var requiredNames = new HashSet(StringComparer.Ordinal); + if (schema.TryGetValue("required", out JsonElement requiredElement)) + { + if (requiredElement.ValueKind != JsonValueKind.Array || requiredElement.EnumerateArray().Any(item => item.ValueKind != JsonValueKind.String)) + { + throw new JevUnsupportedToolSchemaException("the input schema required field must be a string array"); + } + + requiredNames.UnionWith(requiredElement.EnumerateArray().Select(item => item.GetString()!)); + } + + List missing = [.. requiredNames.Where(name => !properties.Any(property => property.Name == name)).OrderBy(name => name, StringComparer.Ordinal)]; + if (missing.Count > 0) + { + throw new JevUnsupportedToolSchemaException($"required arguments are missing from properties: {string.Join(", ", missing)}"); + } + + var questions = new Dictionary(StringComparer.Ordinal); + var arguments = new List(properties.Count); + for (int argumentIndex = 0; argumentIndex < properties.Count; argumentIndex++) + { + JsonProperty property = properties[argumentIndex]; + bool isRequired = requiredNames.Contains(property.Name); + if (property.Value.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException($"argument '{property.Name}' has an invalid schema"); + } + + List previousValues = [.. previousCalls.Where(call => call.ContainsKey(property.Name)).Select(call => call[property.Name])]; + try + { + arguments.Add(CompileArgument(tool, schema, property.Name, JevToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions)); + } + catch (JevUnsupportedToolSchemaException ex) + { + throw new JevUnsupportedToolSchemaException($"{(isRequired ? "required" : "optional")} argument '{property.Name}': {ex.Message}", ex); + } + } + + return new JevCompiledTool + { + Function = tool, + RouteLabel = string.Create(CultureInfo.InvariantCulture, $"t{toolIndex}"), + Questions = questions, + Arguments = arguments, + }; + } + + private static JevArgumentPlan CompileArgument( + AIFunctionDeclaration tool, + Dictionary root, + string name, + Dictionary rawSchema, + bool required, + int toolIndex, + int argumentIndex, + List previousValues, + JevQuestionBudget budget, + Dictionary questions) + { + (Dictionary schema, bool nullable) = JevToolSchema.Resolve(rawSchema, root); + JevToolSchema.RejectUnsupportedKeys(schema, JevToolSchema.ArgumentKeys, "argument"); + + // A required argument must be passed, and Jev cannot choose null among the closed-set values, so a required + // nullable argument would always have to be guessed. + if (nullable && required) + { + throw new JevUnsupportedToolSchemaException("required nullable arguments are not supported"); + } + + string prefix = string.Create(CultureInfo.InvariantCulture, $"{QuestionPrefix}.t{toolIndex}.a{argumentIndex}"); + string description = GetText(schema, "description") ?? GetText(rawSchema, "description") ?? GetText(schema, "title") ?? name; + string previousInstruction = previousValues.Count > 0 + ? $" Values already used for this argument in earlier calls this turn: {JevToolSchema.Describe(previousValues)}. Choose a different value when the user requested another distinct call." + : string.Empty; + string? presenceQuestionId = required ? null : prefix + ".present"; + + void AddPresenceQuestion() + { + if (presenceQuestionId is not null) + { + questions[presenceQuestionId] = new JevNoulQuestion + { + Instructions = $"For the {tool.Name} tool, did the user explicitly specify the {name} argument? Argument meaning: {description}.{previousInstruction}", + }; + } + } + + if (schema.TryGetValue("const", out JsonElement constValue)) + { + budget.Reserve(required ? 0 : 1); + AddPresenceQuestion(); + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; + } + + if (schema.TryGetValue("enum", out JsonElement enumElement) && enumElement.ValueKind == JsonValueKind.Array && enumElement.GetArrayLength() > 0) + { + List values = [.. enumElement.EnumerateArray()]; + if (values.Count > MaxEnumValues) + { + throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "enum defines {0} values; the supported maximum is {1}", values.Count, MaxEnumValues)); + } + + if (values.Count == 1) + { + budget.Reserve(required ? 0 : 1); + AddPresenceQuestion(); + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = values[0] }; + } + + var labeled = new Dictionary(StringComparer.Ordinal); + var criteria = new Dictionary(StringComparer.Ordinal); + for (int index = 0; index < values.Count; index++) + { + string label = string.Create(CultureInfo.InvariantCulture, $"v{index}"); + labeled[label] = values[index]; + criteria[label] = JevToolSchema.Describe(values[index]); + } + + string questionId = prefix + ".value"; + budget.Reserve(required ? 1 : 2); + AddPresenceQuestion(); + questions[questionId] = new JevChoiceQuestion + { + Instructions = $"For the {tool.Name} tool, choose the {name} argument. Argument meaning: {description}.{previousInstruction}", + Criteria = criteria, + }; + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Choice, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId, Values = labeled }; + } + + string? schemaType = schema.TryGetValue("type", out JsonElement typeElement) && typeElement.ValueKind == JsonValueKind.String ? typeElement.GetString() : null; + if (schemaType == "boolean") + { + string questionId = prefix + ".value"; + budget.Reserve(required ? 1 : 2); + AddPresenceQuestion(); + questions[questionId] = new JevNoulQuestion + { + Instructions = $"For the {tool.Name} tool, should the {name} argument be true? Argument meaning: {description}.{previousInstruction}", + }; + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Boolean, PresenceQuestionId = presenceQuestionId, ValueQuestionId = questionId }; + } + + if (schemaType == "array") + { + if (!schema.TryGetValue("items", out JsonElement itemsElement) || itemsElement.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException("array items must define an enum"); + } + + (Dictionary items, bool itemsNullable) = JevToolSchema.Resolve(JevToolSchema.ToMap(itemsElement), root); + JevToolSchema.RejectUnsupportedKeys(items, JevToolSchema.ArrayItemKeys, "array item"); + if (itemsNullable) + { + throw new JevUnsupportedToolSchemaException("nullable array members are not supported"); + } + + if (!items.TryGetValue("enum", out JsonElement membersElement) || membersElement.ValueKind != JsonValueKind.Array || membersElement.GetArrayLength() == 0) + { + throw new JevUnsupportedToolSchemaException("array items must define a non-empty enum"); + } + + List members = [.. membersElement.EnumerateArray()]; + if (members.Count > MaxEnumValues) + { + throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "array enum defines {0} values; the supported maximum is {1}", members.Count, MaxEnumValues)); + } + + IReadOnlyList itemTypes = JevToolSchema.GetTypes(items.TryGetValue("type", out JsonElement itemType) ? itemType : null, "array item"); + foreach (JsonElement member in members) + { + if (itemTypes.Count > 0 && !itemTypes.Any(candidate => JevToolSchema.MatchesType(member, candidate))) + { + string declared = itemTypes.Count == 1 ? $"\"{itemTypes[0]}\"" : JevToolSchema.Describe(itemTypes.Select(candidate => JevJsonUtilities.ToElement(candidate))); + throw new JevUnsupportedToolSchemaException($"array enum member {JevToolSchema.Describe(member)} does not match declared item type {declared}"); + } + } + + budget.Reserve(members.Count + (required ? 0 : 1)); + AddPresenceQuestion(); + var memberQuestions = new List>(members.Count); + for (int index = 0; index < members.Count; index++) + { + string questionId = string.Create(CultureInfo.InvariantCulture, $"{prefix}.m{index}"); + questions[questionId] = new JevNoulQuestion + { + Instructions = $"For the {tool.Name} tool, should the {name} argument include {JevToolSchema.Describe(members[index])}? Argument meaning: {description}.{previousInstruction}", + }; + memberQuestions.Add(new(questionId, members[index])); + } + + return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Set, PresenceQuestionId = presenceQuestionId, Members = memberQuestions }; + } + + throw new JevUnsupportedToolSchemaException("only const, enum, boolean, and arrays of enum values are supported"); + } + + private static string? GetText(Dictionary schema, string key) => + schema.TryGetValue(key, out JsonElement value) && value.ValueKind == JsonValueKind.String && value.GetString() is { Length: > 0 } text ? text : null; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs new file mode 100644 index 00000000000..e4f0335c806 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallPlan.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// The internal questions that let Jev choose one tool call, and the decoders that turn their answers into the call. +/// +internal sealed class JevToolCallPlan +{ + /// + /// A Noul answer above this probability means yes, for presence, Boolean, and array member questions. + /// + public const double NoulYesThreshold = 0.5; + + public required IReadOnlyDictionary Questions { get; init; } + + public required IReadOnlyList Tools { get; init; } + + /// + /// Gets the ID of the Choice question that selects the tool, or when + /// is the only possible call. + /// + public string? RouteQuestionId { get; init; } + + /// + /// Gets the tool that must be called, when the tool mode requires a call and only one tool compiled. + /// + public JevCompiledTool? RequiredTool { get; init; } + + /// + /// Reads the selected tool call from Jev's answers. + /// + /// The tool call, or when Jev chose not to call a tool. + public JevToolCall? Decode(JevResult result) + { + JevCompiledTool? selected = this.RequiredTool; + if (selected is null) + { + string route = GetChoice(result, this.RouteQuestionId!); + if (route == JevToolCallCompiler.RouteNone) + { + return null; + } + + selected = this.Tools.FirstOrDefault(tool => tool.RouteLabel == route) + ?? throw new InvalidOperationException($"Jev returned the unknown tool route '{route}'."); + } + + return new JevToolCall(selected.Function, selected.DecodeArguments(result)); + } + + public static string GetChoice(JevResult result, string questionId) => + result.Answers.TryGetValue(questionId, out JevResponse? answer) && answer is JevChoiceResponse choice + ? choice.Choice + : throw new InvalidOperationException($"The Jev result has no Choice answer for the internal question '{questionId}'."); + + public static bool IsYes(JevResult result, string questionId) => + result.Answers.TryGetValue(questionId, out JevResponse? answer) && answer is JevNoulResponse noul + ? noul.Noul > NoulYesThreshold + : throw new InvalidOperationException($"The Jev result has no Noul answer for the internal question '{questionId}'."); +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs new file mode 100644 index 00000000000..07bcbb61309 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Text.Json; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Reads the JSON schemas of tool parameters for : which keywords are allowed where, +/// how references and nullable forms resolve, and how values are described to Jev. +/// +internal static class JevToolSchema +{ + // A schema definition may refer to itself; without a limit its resolution would never end. + private const int MaxResolutionDepth = 16; + + private static readonly string[] s_annotationKeys = ["$comment", "$id", "$schema", "default", "deprecated", "description", "examples", "readOnly", "title", "writeOnly"]; + + /// Gets the keywords that do not constrain a value, so they are allowed anywhere. + public static HashSet AnnotationKeys { get; } = new(s_annotationKeys, StringComparer.Ordinal); + + /// Gets the keywords allowed on the object schema of a tool's parameters. + public static HashSet RootKeys { get; } = new(s_annotationKeys.Concat(["$defs", "additionalProperties", "properties", "required", "type"]), StringComparer.Ordinal); + + /// Gets the keywords allowed on the schema of one argument. + public static HashSet ArgumentKeys { get; } = new(s_annotationKeys.Concat(["const", "enum", "items", "type"]), StringComparer.Ordinal); + + /// Gets the keywords allowed on the item schema of an array argument. + public static HashSet ArrayItemKeys { get; } = new(s_annotationKeys.Concat(["enum", "type"]), StringComparer.Ordinal); + + public static Dictionary ToMap(JsonElement schema) + { + var map = new Dictionary(StringComparer.Ordinal); + foreach (JsonProperty property in schema.EnumerateObject()) + { + map[property.Name] = property.Value; + } + + return map; + } + + /// + /// Resolves a local $ref, an anyOf of one schema and null, and the nullable forms that + /// System.Text.Json writes, which are a type array that includes "null" and an enum that lists + /// . + /// + /// The schema without its alternative, and whether it allowed . + public static (Dictionary Schema, bool Nullable) Resolve(Dictionary schema, Dictionary root, int depth = 0) + { + if (depth > MaxResolutionDepth) + { + throw new JevUnsupportedToolSchemaException("schema references are nested too deeply"); + } + + var resolved = new Dictionary(schema, StringComparer.Ordinal); + if (resolved.Remove("$ref", out JsonElement reference)) + { + RejectCompositionSiblings(resolved, "$ref"); + string? path = reference.ValueKind == JsonValueKind.String ? reference.GetString() : null; + const string DefinitionsPrefix = "#/$defs/"; + if (path?.StartsWith(DefinitionsPrefix, StringComparison.Ordinal) is not true) + { + throw new JevUnsupportedToolSchemaException("only local $defs references are supported"); + } + + if (!root.TryGetValue("$defs", out JsonElement definitions) || + definitions.ValueKind != JsonValueKind.Object || + !definitions.TryGetProperty(path.Substring(DefinitionsPrefix.Length), out JsonElement definition) || + definition.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException($"unresolved schema reference '{path}'"); + } + + // Annotations next to the reference, such as a description, override those of the definition. An anyOf + // inside the definition is resolved below, together with the rest of the merged schema. + resolved = Merge(ToMap(definition), resolved); + } + + bool nullable = false; + if (resolved.Remove("anyOf", out JsonElement anyOf)) + { + RejectCompositionSiblings(resolved, "anyOf"); + if (anyOf.ValueKind != JsonValueKind.Array) + { + throw new JevUnsupportedToolSchemaException("anyOf must be an array"); + } + + List arms = [.. anyOf.EnumerateArray()]; + List nullArms = [.. arms.Where(arm => arm.ValueKind == JsonValueKind.Object && IsNullSchema(arm))]; + List valueArms = [.. arms.Where(arm => arm.ValueKind == JsonValueKind.Object && !IsNullSchema(arm))]; + if (arms.Count != 2 || nullArms.Count != 1 || valueArms.Count != 1) + { + throw new JevUnsupportedToolSchemaException("only a single schema combined with null is supported"); + } + + (Dictionary nested, _) = Resolve(ToMap(valueArms[0]), root, depth + 1); + resolved = Merge(nested, resolved); + nullable = true; + } + + if (resolved.TryGetValue("type", out JsonElement type) && type.ValueKind == JsonValueKind.Array && type.EnumerateArray().Any(IsNullName)) + { + List valueTypes = [.. type.EnumerateArray().Where(item => !IsNullName(item))]; + resolved["type"] = valueTypes.Count == 1 ? valueTypes[0] : JevJsonUtilities.CreateArray(valueTypes); + nullable = true; + } + + if (resolved.TryGetValue("enum", out JsonElement values) && values.ValueKind == JsonValueKind.Array && values.EnumerateArray().Any(value => value.ValueKind == JsonValueKind.Null)) + { + resolved["enum"] = JevJsonUtilities.CreateArray(values.EnumerateArray().Where(value => value.ValueKind != JsonValueKind.Null)); + nullable = true; + } + + return (resolved, nullable); + } + + public static void RejectUnsupportedKeys(Dictionary schema, HashSet supportedKeys, string location) + { + List unsupported = [.. schema.Keys.Where(key => !supportedKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; + if (unsupported.Count > 0) + { + throw new JevUnsupportedToolSchemaException($"unsupported {location} schema constraints: {string.Join(", ", unsupported)}"); + } + } + + /// + /// Reads a type keyword, which is a type name or a non-empty array of type names. + /// + public static IReadOnlyList GetTypes(JsonElement? type, string location) + { + if (type is not { } value || value.ValueKind == JsonValueKind.Null) + { + return []; + } + + if (value.ValueKind == JsonValueKind.String) + { + return [value.GetString()!]; + } + + if (value.ValueKind == JsonValueKind.Array && value.GetArrayLength() > 0 && value.EnumerateArray().All(item => item.ValueKind == JsonValueKind.String)) + { + return [.. value.EnumerateArray().Select(item => item.GetString()!)]; + } + + throw new JevUnsupportedToolSchemaException($"{location} type must be a string or non-empty string array"); + } + + public static bool MatchesType(JsonElement value, string type) => + type switch + { + "string" => value.ValueKind == JsonValueKind.String, + + // An integer has no fraction or exponent, as a JSON parser that separates integers from floats reads it. + "integer" => value.ValueKind == JsonValueKind.Number && value.TryGetInt64(out _), + "number" => value.ValueKind == JsonValueKind.Number, + "boolean" => value.ValueKind is JsonValueKind.True or JsonValueKind.False, + "null" => value.ValueKind == JsonValueKind.Null, + "array" => value.ValueKind == JsonValueKind.Array, + "object" => value.ValueKind == JsonValueKind.Object, + _ => false, + }; + + /// + /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): + /// sorted object keys, ", " and ": " separators, and unescaped text. Jev reads these values in + /// criteria and instructions, so both connectors send the same words. + /// + public static string Describe(JsonElement value) + { + var builder = new StringBuilder(); + AppendValue(builder, value); + return builder.ToString(); + } + + /// Writes values as a JSON array, like . + public static string Describe(IEnumerable values) + { + var builder = new StringBuilder(); + AppendArray(builder, values, AppendValue); + return builder.ToString(); + } + + /// Writes the argument sets of earlier calls as a JSON array of objects, like . + public static string Describe(IEnumerable> calls) + { + var builder = new StringBuilder(); + AppendArray(builder, calls, static (target, call) => AppendObject(target, call.Select(argument => new KeyValuePair(argument.Key, argument.Value)))); + return builder.ToString(); + } + + private static void RejectCompositionSiblings(Dictionary schema, string keyword) + { + List siblings = [.. schema.Keys.Where(key => !AnnotationKeys.Contains(key)).OrderBy(key => key, StringComparer.Ordinal)]; + if (siblings.Count > 0) + { + throw new JevUnsupportedToolSchemaException($"{keyword} sibling constraints are not supported: {string.Join(", ", siblings)}"); + } + } + + private static Dictionary Merge(Dictionary target, Dictionary overrides) + { + foreach (KeyValuePair entry in overrides) + { + target[entry.Key] = entry.Value; + } + + return target; + } + + private static bool IsNullSchema(JsonElement schema) => + schema.TryGetProperty("type", out JsonElement type) && IsNullName(type); + + private static bool IsNullName(JsonElement type) => + type.ValueKind == JsonValueKind.String && type.ValueEquals("null"); + + private static void AppendValue(StringBuilder builder, JsonElement value) + { + switch (value.ValueKind) + { + case JsonValueKind.Object: + AppendObject(builder, value.EnumerateObject().Select(property => new KeyValuePair(property.Name, property.Value))); + break; + + case JsonValueKind.Array: + AppendArray(builder, value.EnumerateArray(), AppendValue); + break; + + case JsonValueKind.String: + AppendString(builder, value.GetString()!); + break; + + case JsonValueKind.Null or JsonValueKind.Undefined: + builder.Append("null"); + break; + + default: + // Numbers keep their original text, and true and false are already written as JSON. + builder.Append(value.GetRawText()); + break; + } + } + + private static void AppendObject(StringBuilder builder, IEnumerable> properties) + { + builder.Append('{'); + bool first = true; + foreach (KeyValuePair property in properties.OrderBy(property => property.Key, StringComparer.Ordinal)) + { + builder.Append(first ? string.Empty : ", "); + first = false; + AppendString(builder, property.Key); + builder.Append(": "); + AppendValue(builder, property.Value); + } + + builder.Append('}'); + } + + private static void AppendArray(StringBuilder builder, IEnumerable items, Action append) + { + builder.Append('['); + bool first = true; + foreach (T item in items) + { + builder.Append(first ? string.Empty : ", "); + first = false; + append(builder, item); + } + + builder.Append(']'); + } + + /// + /// Writes a JSON string with the escapes of Python's ensure_ascii=False: quotes, backslashes, and control + /// characters only. + /// + private static void AppendString(StringBuilder builder, string text) + { + builder.Append('"'); + foreach (char c in text) + { + switch (c) + { + case '"': builder.Append("\\\""); break; + case '\\': builder.Append("\\\\"); break; + case '\n': builder.Append("\\n"); break; + case '\r': builder.Append("\\r"); break; + case '\t': builder.Append("\\t"); break; + case '\b': builder.Append("\\b"); break; + case '\f': builder.Append("\\f"); break; + case < ' ': builder.Append("\\u").Append(((int)c).ToString("x4", CultureInfo.InvariantCulture)); break; + default: builder.Append(c); break; + } + } + + builder.Append('"'); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs new file mode 100644 index 00000000000..fe3ecfc72d9 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevUnsupportedToolSchemaException.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// A tool's JSON schema cannot be expressed as Jev questions, so the tool is left out of the request. +/// +/// +/// It is caught while a tool call plan is compiled and never leaves this package: an unsupported tool is logged and +/// excluded, or, when the tool mode requires a call and no tool remains, its reason becomes part of an +/// . +/// +internal sealed class JevUnsupportedToolSchemaException : NotSupportedException +{ + public JevUnsupportedToolSchemaException() + { + } + + public JevUnsupportedToolSchemaException(string message) + : base(message) + { + } + + public JevUnsupportedToolSchemaException(string message, Exception innerException) + : base(message, innerException) + { + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs index d1bb421f9c7..8dd97a2d0c8 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolBuilder.cs @@ -18,16 +18,16 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// Jev does not write text. It answers typed questions about a state and returns calibrated probabilities, which /// makes it a fast and inexpensive classifier for an agent whose own model writes the replies. The function created by -/// is a thin wrapper around an evaluator: its arguments are a written by -/// the agent's model, and its result is the evaluator's , as JSON. +/// is a thin wrapper around a System One call: its arguments are a written by +/// the agent's model, and its result is the call's , as JSON. /// /// -/// With an , the evaluator is the TypeSafe System One API, called through a +/// With an , the call goes to the TypeSafe System One API through a /// System.ClientModel pipeline configured by . Any other client can be used instead with -/// or one of the UseClient methods. +/// or one of the UseClient methods. /// /// -/// Every evaluator receives the of the agent that calls the tool +/// Every mapping receives the of the agent that calls the tool /// (, which FunctionInvokingChatClient sets from the services /// the agent was created with), so it can resolve its dependencies per call. In a host, register the tool with /// services.AddAIAgent(...).WithAITool(sp => new JevAIToolBuilder(credential, options).Build()). @@ -37,10 +37,10 @@ public sealed class JevAIToolBuilder { private readonly ApiKeyCredential? _credential; private readonly JevAIToolOptions _options; - private Func>? _evaluateAsync; + private Func>? _systemOneAsync; /// - /// Initializes a new instance of the class that evaluates requests with the + /// Initializes a new instance of the class that answers requests with the /// TypeSafe System One API. /// /// The TypeSafe API key, sent as a bearer token. @@ -52,12 +52,12 @@ public JevAIToolBuilder(ApiKeyCredential credential, JevAIToolOptions? options = } /// - /// Initializes a new instance of the class for a custom evaluator, which must be set - /// with or a UseClient method before . + /// Initializes a new instance of the class for a custom mapping, which must be set + /// with or a UseClient method before . /// /// /// The tool options. Only and apply - /// to a custom evaluator, which uses its own client configuration. Defaults are used when . + /// to a custom mapping, which uses its own client configuration. Defaults are used when . /// public JevAIToolBuilder(JevAIToolOptions? options = null) { @@ -65,32 +65,33 @@ public JevAIToolBuilder(JevAIToolOptions? options = null) } /// - /// Sets the evaluator that answers each request, replacing the TypeSafe API. + /// Sets the function that maps each to its , replacing the + /// TypeSafe API. /// - /// - /// The evaluator. It receives the written by the model and the agent's services, which are + /// + /// The mapping. It receives the written by the model and the agent's services, which are /// when the caller provides none, and returns one answer per question. /// /// This builder. /// /// The agent's services are often the application's root provider, because hosted agents are singletons by - /// default. To use a scoped service, create a scope inside the evaluator, or use + /// default. To use a scoped service, create a scope inside the mapping, or use /// , which does so for every call. /// - public JevAIToolBuilder UseEvaluator(Func> evaluateAsync) + public JevAIToolBuilder UseMapping(Func> mapAsync) { - this._evaluateAsync = Throw.IfNull(evaluateAsync); + this._systemOneAsync = Throw.IfNull(mapAsync); return this; } /// - /// Evaluates requests with another client, converting between the Jev contract types and the client's own types. + /// Answers requests with another client, converting between the Jev contract types and the client's own types. /// /// The request type of the client. /// The response type of the client. - /// The client operation that evaluates a request. + /// The client operation that answers a request. /// Converts the written by the model into the client's request. - /// Converts the client's response into a . + /// Converts the client's response into a . /// This builder. /// /// The mapped response is checked against the request before it reaches the model: every question needs an answer @@ -99,51 +100,51 @@ public JevAIToolBuilder UseEvaluator(Func public JevAIToolBuilder UseClient( - Func> evaluateAsync, + Func> systemOneAsync, Func inputMapper, - Func outputMapper) + Func outputMapper) { - _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(systemOneAsync); _ = Throw.IfNull(inputMapper); _ = Throw.IfNull(outputMapper); - return this.UseEvaluator(async (request, _, cancellationToken) => + return this.UseMapping(async (request, _, cancellationToken) => { TClientRequest clientRequest = inputMapper(request); - TClientResponse clientResponse = await evaluateAsync(clientRequest, cancellationToken).ConfigureAwait(false); + TClientResponse clientResponse = await systemOneAsync(clientRequest, cancellationToken).ConfigureAwait(false); return outputMapper(clientResponse); }); } /// - /// Evaluates requests with a client registered in the agent's services, converting between the Jev contract types + /// Answers requests with a client registered in the agent's services, converting between the Jev contract types /// and the client's own types. /// /// The client type, resolved from the agent's services on every call. /// The request type of the client. /// The response type of the client. - /// The client operation that evaluates a request. + /// The client operation that answers a request. /// Converts the written by the model into the client's request. - /// Converts the client's response into a . + /// Converts the client's response into a . /// This builder. /// /// The client is resolved from a new service scope on every call, and the scope is disposed when the call ends. /// Scoped clients therefore live for one call, and disposable transient clients are released after it, even when /// the agent's services are the application's root provider. The call fails when the agent has no services or /// is not registered. The mapped response is checked as described for - /// . + /// . /// public JevAIToolBuilder UseClient( - Func> evaluateAsync, + Func> systemOneAsync, Func inputMapper, - Func outputMapper) + Func outputMapper) where TClient : notnull { - _ = Throw.IfNull(evaluateAsync); + _ = Throw.IfNull(systemOneAsync); _ = Throw.IfNull(inputMapper); _ = Throw.IfNull(outputMapper); - return this.UseEvaluator(async (request, services, cancellationToken) => + return this.UseMapping(async (request, services, cancellationToken) => { if (services is null) { @@ -157,16 +158,16 @@ public JevAIToolBuilder UseClient( // transient client until shutdown, so each call gets its own scope. if (services.GetService() is not { } scopeFactory) { - return await EvaluateWithClientAsync(services).ConfigureAwait(false); + return await CallClientAsync(services).ConfigureAwait(false); } AsyncServiceScope scope = scopeFactory.CreateAsyncScope(); await using (scope.ConfigureAwait(false)) { - return await EvaluateWithClientAsync(scope.ServiceProvider).ConfigureAwait(false); + return await CallClientAsync(scope.ServiceProvider).ConfigureAwait(false); } - async Task EvaluateWithClientAsync(IServiceProvider provider) + async Task CallClientAsync(IServiceProvider provider) { if (provider.GetService(typeof(TClient)) is not TClient client) { @@ -175,7 +176,7 @@ async Task EvaluateWithClientAsync(IServiceProvider provider) } TClientRequest clientRequest = inputMapper(request); - TClientResponse clientResponse = await evaluateAsync(client, clientRequest, cancellationToken).ConfigureAwait(false); + TClientResponse clientResponse = await systemOneAsync(client, clientRequest, cancellationToken).ConfigureAwait(false); return outputMapper(clientResponse); } }); @@ -186,10 +187,10 @@ async Task EvaluateWithClientAsync(IServiceProvider provider) /// /// /// An whose JSON schema is the schema of and whose result is the - /// evaluator's serialized as JSON. + /// call's serialized as JSON. /// /// - /// The builder has neither an nor a custom evaluator, or an option that it uses is + /// The builder has neither an nor a custom mapping, or an option that it uses is /// empty or invalid. /// /// @@ -204,15 +205,15 @@ public AIFunction Build() Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Name)} and {nameof(JevAIToolOptions.Description)} must not be empty."); } - Func>? evaluateAsync = this._evaluateAsync; - if (evaluateAsync is null) + Func>? systemOneAsync = this._systemOneAsync; + if (systemOneAsync is null) { if (this._credential is null) { - Throw.InvalidOperationException($"Pass an {nameof(ApiKeyCredential)} to the constructor to use the TypeSafe API, or call {nameof(UseEvaluator)} or UseClient."); + Throw.InvalidOperationException($"Pass an {nameof(ApiKeyCredential)} to the constructor to use the TypeSafe API, or call {nameof(UseMapping)} or UseClient."); } - // The model and endpoint only matter to the TypeSafe API; a custom evaluator uses its own configuration. + // The model and endpoint only matter to the TypeSafe API; a custom mapping uses its own configuration. if (string.IsNullOrWhiteSpace(this._options.ModelId)) { Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.ModelId)} must not be empty."); @@ -223,9 +224,9 @@ public AIFunction Build() Throw.InvalidOperationException($"{nameof(JevAIToolOptions)}.{nameof(JevAIToolOptions.Endpoint)} must be an absolute URI."); } - evaluateAsync = new JevHttpEvaluator(this._credential, this._options).EvaluateAsync; + systemOneAsync = new JevHttpClient(this._credential, this._options, this._options.Endpoint, this._options.ModelId).SystemOneAsync; } - return new JevAIFunction(this._options.Name, this._options.Description, evaluateAsync); + return new JevAIFunction(this._options.Name, this._options.Description, systemOneAsync); } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs index 7f08031b762..e0239af7324 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAIToolOptions.cs @@ -19,13 +19,15 @@ namespace Microsoft.Agents.AI.TypeSafe; /// adds custom policies. The pipeline settings, /// , and apply only when the builder uses the TypeSafe API, that is, when /// it receives an ; and apply to every -/// evaluator. +/// mapping. /// /// -/// When is , the pipeline retries up to three -/// times on the statuses that TypeSafe documents as transient (429 and 529, besides the standard 408 and 5xx ones), and -/// fails at once when the API asks for a wait longer than one minute, because the wait would block the agent run -/// without any feedback. That default follows . +/// Unset pipeline settings get the defaults of the official TypeSafe SDK. When +/// is , up to two retries follow 408, 429, and +/// 5xx responses and network failures, waiting for the server's retry-after-ms or Retry-After delay up +/// to one minute, or else for an exponential backoff from half a second to five seconds; the policy follows +/// . When +/// is , each attempt times out after 10 seconds. /// /// /// Like other client options, an instance becomes read-only once a tool has been built with it. @@ -34,23 +36,17 @@ namespace Microsoft.Agents.AI.TypeSafe; public sealed class JevAIToolOptions : ClientPipelineOptions { private const string DefaultDescription = - "Evaluates content with Jev, a classification model. Jev does not write text: it answers typed questions about the " + - "given state and returns calibrated probabilities. Use it to classify, route, score, or check facts about text or " + - "JSON data, and ask all related questions in one call. A 'choice' question picks exactly one option from its " + - "criteria. A 'score' question rates the state against 2 to 10 ordered levels and can land between levels. A 'noul' " + - "question returns the probability, from 0 to 1, that its yes or no statement is true. Choice and score answers " + - "include a confidence from 0 to 1; treat a low confidence as uncertain."; + "Answers named questions about text or structured state with Jev, a classification model. Jev does not write " + + "text: it returns calibrated probabilities. Use it to classify, route, score, or check facts, and ask all related " + + "questions in one call. A 'choice' question selects one of the named alternatives in its criteria. A 'score' " + + "question assigns a score using an ordered rubric of 2 to 10 entries, and the score can fall between entries. A " + + "'noul' question returns the probability that the answer to a yes or no question is yes. Choice and score " + + "answers include a confidence from 0 to 1; treat a low confidence as uncertain."; /// public override void Freeze() { - // The default policy is assigned here, not in the constructor, because ClientLoggingOptions may be set after - // construction; ClientPipeline.Create freezes the options before it reads RetryPolicy, so the policy is in place. - this.RetryPolicy ??= new JevRetryPolicy( - JevRetryPolicy.DefaultMaxRetries, - this.ClientLoggingOptions?.EnableLogging ?? true, - this.ClientLoggingOptions?.LoggerFactory); - + JevPipelineDefaults.Apply(this); base.Freeze(); } @@ -78,10 +74,11 @@ public string ModelId this.AssertNotFrozen(); field = value; } - } = "jev-latest"; + } = JevPipelineDefaults.DefaultModelId; /// - /// Gets or sets the name of the function that the model sees. Defaults to evaluate_with_jev. + /// Gets or sets the name of the function that the model sees. Defaults to ask_system_one, after the SDK's + /// systemOne operation. /// public string Name { @@ -91,7 +88,7 @@ public string Name this.AssertNotFrozen(); field = value; } - } = "evaluate_with_jev"; + } = "ask_system_one"; /// /// Gets or sets the description of the function that the model sees. diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs deleted file mode 100644 index 77a1adf5e55..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevAnswer.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents Jev's answer to one question of a . -/// -/// -/// The JSON type property matches the kind of the question: choice (), -/// score (), or noul (). -/// -[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] -[JsonDerivedType(typeof(JevChoiceAnswer), "choice")] -[JsonDerivedType(typeof(JevScoreAnswer), "score")] -[JsonDerivedType(typeof(JevNoulAnswer), "noul")] -public abstract class JevAnswer -{ - // Only the three answer kinds that the Jev API returns can exist. - private protected JevAnswer() - { - } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs new file mode 100644 index 00000000000..aa91948f13d --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClient.cs @@ -0,0 +1,302 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Provides an for Jev, the TypeSafe System One model, which answers typed questions about +/// a conversation instead of writing text. +/// +/// +/// +/// Each request sends the conversation as the Jev state, together with the questions passed with +/// (or ). +/// The response text is the as JSON, and +/// returns it typed. This follows the TypeSafe connector for Python, so both languages behave alike. +/// +/// +/// Jev can also call function tools whose arguments come from a closed set: constants, enums, Booleans, and arrays of +/// enum values. The client adds internal questions that choose the tool and its arguments, and returns a +/// for a function-invoking client (which a ChatClientAgent adds) to run. +/// Once the turn has used calls, the client answers the +/// questions instead; the response text is then the tool results followed by one id: answer line per Choice +/// and Score answer. Tools with free-form arguments, such as text or numbers, are left out with a warning. +/// +/// +/// Features that need generated text fail with : streaming, sampling and output +/// options such as or a JSON , +/// server-side conversations and background responses, tools other than functions, and content other than text, +/// reasoning text, and function calls and results. Tool approval requests and responses are left out of the state, +/// because the call and result they lead to carry the information. +/// +/// +public sealed class JevChatClient : IChatClient +{ + private const string ProviderName = "typesafe.ai"; + + private readonly JevHttpClient _client; + private readonly IReadOnlyDictionary? _defaultQuestions; + private readonly int _maximumToolCallsPerTurn; + private readonly ILogger _logger; + private readonly ChatClientMetadata _metadata; + + /// + /// Initializes a new instance of the class. + /// + /// The TypeSafe API key, sent as a bearer token. + /// The client and pipeline options. Defaults are used when . + /// + /// is empty, or is not an + /// absolute URI. + /// + public JevChatClient(ApiKeyCredential credential, JevChatClientOptions? options = null) + { + _ = Throw.IfNull(credential); + options ??= new JevChatClientOptions(); + + if (string.IsNullOrWhiteSpace(options.ModelId)) + { + Throw.ArgumentException(nameof(options), $"{nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.ModelId)} must not be empty."); + } + + if (options.Endpoint?.IsAbsoluteUri is false) + { + Throw.ArgumentException(nameof(options), $"{nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.Endpoint)} must be an absolute URI."); + } + + // The questions are copied so that later changes to the caller's dictionary do not change this client. + this._defaultQuestions = options.DefaultQuestions?.ToDictionary(question => question.Key, question => question.Value, StringComparer.Ordinal); + this._maximumToolCallsPerTurn = options.MaximumToolCallsPerTurn; + this._logger = (options.ClientLoggingOptions?.LoggerFactory ?? NullLoggerFactory.Instance).CreateLogger(); + this._client = new JevHttpClient(credential, options, options.Endpoint, options.ModelId); + this._metadata = new ChatClientMetadata(ProviderName, this._client.Endpoint, this._client.ModelId); + } + + /// + /// + /// The request has no questions, a required tool call is impossible, or the Jev result does not answer every + /// question. + /// + /// The request uses an option, tool, or content that Jev does not support. + /// The TypeSafe API rejected the request. + public async Task GetResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(messages); + IReadOnlyList history = messages as IReadOnlyList ?? [.. messages]; + + EnsureSupported(options); + IReadOnlyDictionary questions = this.GetQuestions(options); + JevToolCallCompiler.EnsureNoReservedQuestionIds(questions.Keys); + List tools = GetFunctionTools(options); + + // Once the turn has used its tool calls, tools are no longer offered, so the next response answers the + // questions with the tool results in the state. This is what ends the function-invoking loop, like the + // max_function_calls default of one in the Python connector. + int turnStart = JevChatState.GetCurrentTurnStart(history); + ChatToolMode? toolMode = JevChatState.CountCurrentTurnCalls(history, turnStart) >= this._maximumToolCallsPerTurn + ? ChatToolMode.None + : options?.ToolMode; + JevToolCallPlan? plan = JevToolCallCompiler.Compile(tools, toolMode, JevChatState.GetCurrentTurnCalls(history, turnStart), this._logger); + + var allQuestions = new Dictionary(StringComparer.Ordinal); + foreach (KeyValuePair question in questions) + { + allQuestions[question.Key] = question.Value; + } + + foreach (KeyValuePair question in plan?.Questions ?? new Dictionary()) + { + allQuestions[question.Key] = question.Value; + } + + var request = new JevRequest { State = JevChatState.Build(history, options?.Instructions), Questions = allQuestions }; + JevContractValidator.ValidateRequest(request); + + string modelId = string.IsNullOrWhiteSpace(options?.ModelId) ? this._client.ModelId : options!.ModelId!; + JevResult result = await this._client.SystemOneAsync(request, modelId, cancellationToken).ConfigureAwait(false); + + // Validation covers the internal questions too, so the decoders below only see labels they offered. + JevContractValidator.ValidateResult(request, result); + UsageDetails? usage = result.Usage is { } tokens + ? new UsageDetails { InputTokenCount = tokens.InputTokens, OutputTokenCount = tokens.OutputTokens, TotalTokenCount = tokens.InputTokens + tokens.OutputTokens } + : null; + + if (plan?.Decode(result) is { } call) + { + var callMessage = new ChatMessage(ChatRole.Assistant, [new FunctionCallContent($"typesafe-{Guid.NewGuid():N}", call.Function.Name, call.Arguments)]); + return new ChatResponse(callMessage) + { + ModelId = result.Model, + FinishReason = ChatFinishReason.ToolCalls, + Usage = usage, + CreatedAt = DateTimeOffset.UtcNow, + }; + } + + // The caller asked only its own questions, so the internal tool answers are removed from what it sees. + JevResult answers = WithoutInternalAnswers(result, questions); + var message = new ChatMessage(ChatRole.Assistant, BuildText(history, turnStart, answers)) { RawRepresentation = answers }; + return new ChatResponse(message) + { + ModelId = result.Model, + FinishReason = ChatFinishReason.Stop, + Usage = usage, + CreatedAt = DateTimeOffset.UtcNow, + RawRepresentation = answers, + }; + } + + /// + /// Always: Jev answers all questions at once and does not stream. + public IAsyncEnumerable GetStreamingResponseAsync(IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) => + throw new NotSupportedException($"Jev answers all questions at once and does not stream. Call {nameof(GetResponseAsync)}, or RunAsync on an agent."); + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = Throw.IfNull(serviceType); + + return serviceKey is not null ? null + : serviceType == typeof(ChatClientMetadata) ? this._metadata + : serviceType.IsInstanceOfType(this) ? this + : null; + } + + /// + public void Dispose() + { + // The pipeline owns no disposable resources; its default transport shares one HttpClient per process. + } + + private static void EnsureSupported(ChatOptions? options) + { + if (options is null) + { + return; + } + + var unsupported = new List(); + AddIf(options.Temperature is not null, nameof(ChatOptions.Temperature)); + AddIf(options.TopP is not null, nameof(ChatOptions.TopP)); + AddIf(options.TopK is not null, nameof(ChatOptions.TopK)); + AddIf(options.MaxOutputTokens is not null, nameof(ChatOptions.MaxOutputTokens)); + AddIf(options.FrequencyPenalty is not null, nameof(ChatOptions.FrequencyPenalty)); + AddIf(options.PresencePenalty is not null, nameof(ChatOptions.PresencePenalty)); + AddIf(options.Seed is not null, nameof(ChatOptions.Seed)); + AddIf(options.Reasoning is not null, nameof(ChatOptions.Reasoning)); + AddIf(options.StopSequences is { Count: > 0 }, nameof(ChatOptions.StopSequences)); + AddIf(options.ResponseFormat is not (null or ChatResponseFormatText), nameof(ChatOptions.ResponseFormat)); + AddIf(options.ConversationId is not null, nameof(ChatOptions.ConversationId)); + AddIf(options.AllowBackgroundResponses is true, nameof(ChatOptions.AllowBackgroundResponses)); +#pragma warning disable MEAI001 // ContinuationToken is experimental; it is only checked to reject it. + AddIf(options.ContinuationToken is not null, nameof(ChatOptions.ContinuationToken)); +#pragma warning restore MEAI001 + + if (unsupported.Count > 0) + { + throw new NotSupportedException( + $"Jev does not support these chat options: {string.Join(", ", unsupported)}. Jev does not generate text; it answers the questions passed with {nameof(JevChatOptionsExtensions.WithJevQuestions)}."); + } + + if (options.AllowMultipleToolCalls is true) + { + throw new NotSupportedException($"Jev requests one tool call per response. Leave {nameof(ChatOptions)}.{nameof(ChatOptions.AllowMultipleToolCalls)} unset or false."); + } + + void AddIf(bool condition, string name) + { + if (condition) + { + unsupported.Add(name); + } + } + } + + private IReadOnlyDictionary GetQuestions(ChatOptions? options) + { + IReadOnlyDictionary? questions = + (options?.RawRepresentationFactory?.Invoke(this) as JevChatRequestOptions)?.Questions ?? this._defaultQuestions; + if (questions is null) + { + Throw.InvalidOperationException( + $"Jev needs questions to answer. Pass them with {nameof(ChatOptions)}.{nameof(JevChatOptionsExtensions.WithJevQuestions)}, or set {nameof(JevChatClientOptions)}.{nameof(JevChatClientOptions.DefaultQuestions)}."); + } + + if (questions.Count == 0) + { + Throw.ArgumentException(nameof(options), "Jev needs at least one question."); + } + + return questions; + } + + private static List GetFunctionTools(ChatOptions? options) + { + if (options?.Tools is not { Count: > 0 } tools) + { + return []; + } + + List unsupported = [.. tools.Where(tool => tool is not AIFunctionDeclaration).Select(tool => tool.Name)]; + if (unsupported.Count > 0) + { + throw new NotSupportedException($"Jev can only call function tools; these tools are not supported: {string.Join(", ", unsupported)}."); + } + + return [.. tools.Cast()]; + } + + private static JevResult WithoutInternalAnswers(JevResult result, IReadOnlyDictionary questions) + { + var answers = new Dictionary(questions.Count, StringComparer.Ordinal); + foreach (string id in questions.Keys) + { + answers[id] = result.Answers[id]; + } + + return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; + } + + /// + /// Builds the response text: the result as JSON, or, after tool calls, the tool results followed by the Choice and + /// Score decisions, which is what a caller that reads only text needs to see. + /// + private static string BuildText(IReadOnlyList history, int turnStart, JevResult answers) + { + List lines = JevChatState.GetCurrentTurnResultTexts(history, turnStart); + if (lines.Count == 0) + { + return JsonSerializer.Serialize(answers, JevJsonContext.Default.JevResult); + } + + // Noul answers are probabilities rather than decisions, so, as in the Python connector, they are left to the + // typed result. + foreach (KeyValuePair answer in answers.Answers) + { + switch (answer.Value) + { + case JevChoiceResponse choice: + lines.Add($"{answer.Key}: {choice.Choice}"); + break; + + case JevScoreResponse score: + lines.Add($"{answer.Key}: {score.Score.ToString("G6", CultureInfo.InvariantCulture)}"); + break; + } + } + + return string.Join("\n", lines); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs new file mode 100644 index 00000000000..a8de9a73812 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Configures a and the client pipeline that sends its requests to the TypeSafe System +/// One API. +/// +/// +/// +/// This type follows the pattern of the OpenAI and Azure AI client libraries: +/// selects the HTTP stack, +/// controls retries, and adds custom +/// policies. Unset pipeline settings get the defaults of the official TypeSafe SDK, as described for +/// . also supplies the logger +/// that reports tools left out of a request. +/// +/// +/// Like other client options, an instance becomes read-only once a client has been created with it. +/// +/// +public sealed class JevChatClientOptions : ClientPipelineOptions +{ + /// + public override void Freeze() + { + JevPipelineDefaults.Apply(this); + base.Freeze(); + } + + /// + /// Gets or sets the System One endpoint. When , https://api.typesafe.ai/v1/systemone is used. + /// + public Uri? Endpoint + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } + + /// + /// Gets or sets the Jev model that answers the questions, unless + /// selects another for a request. Defaults to jev-latest. + /// + public string ModelId + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } = JevPipelineDefaults.DefaultModelId; + + /// + /// Gets or sets the questions to answer when a request does not pass its own with + /// . + /// + /// + /// Default questions let code that knows nothing about Jev, such as a framework component with a fixed task, + /// use the client as any other . The client copies the dictionary when it + /// is created. + /// + public IReadOnlyDictionary? DefaultQuestions + { + get; + set + { + this.AssertNotFrozen(); + field = value; + } + } + + /// + /// Gets or sets how many tool calls Jev may request in one turn, that is, while answering one user request. + /// Defaults to 1. + /// + /// + /// The turn starts after the last user message, except that text sent together with a tool approval response + /// continues the turn of the call it approves. Once the turn has this many function calls, the client stops + /// offering tools and answers the questions, with the tool results in the state. The route question tells Jev + /// which calls the turn already made, but a closed-set choice can still repeat one, so the limit is what + /// guarantees that the tool loop ends. Raise it when one user request needs several distinct calls, such as the + /// weather in two cities; set it to 0 to never call tools. + /// + /// The value is negative. + public int MaximumToolCallsPerTurn + { + get; + set + { + this.AssertNotFrozen(); + field = Throw.IfLessThan(value, 0); + } + } = 1; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs new file mode 100644 index 00000000000..119269f117c --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatOptionsExtensions.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Provides extension methods that pass Jev questions through . +/// +public static class JevChatOptionsExtensions +{ + /// + /// Sets the questions that a answers for a request. + /// + /// The chat options to configure. + /// The questions to answer, keyed by an ID that the caller chooses. + /// , for chaining. + /// + /// + /// This sets to return a + /// to a . For any other client it calls the factory that was set before, so the same + /// options can still carry provider options for another client. Calling this method again replaces the + /// questions. + /// + /// + /// With an agent, pass the options in a ChatClientAgentRunOptions for one run, or in the agent's + /// ChatClientAgentOptions.ChatOptions for every run; the agent prefers the run's questions. + /// + /// + public static ChatOptions WithJevQuestions(this ChatOptions options, IReadOnlyDictionary questions) + { + _ = Throw.IfNull(options); + _ = Throw.IfNull(questions); + + Func? previous = options.RawRepresentationFactory; + options.RawRepresentationFactory = client => + client is JevChatClient ? new JevChatRequestOptions { Questions = questions } : previous?.Invoke(client); + return options; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs new file mode 100644 index 00000000000..3701e3da708 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatRequestOptions.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the Jev-specific options of one request. +/// +/// +/// A reads this type from , which is +/// how Microsoft.Extensions.AI passes options that only one provider understands. Other chat clients ignore it. +/// sets the factory for you. +/// +public sealed class JevChatRequestOptions +{ + /// + /// Gets or sets the questions to answer about the conversation, keyed by an ID that the caller chooses. + /// + /// + /// Each answer of the uses the ID of its question. IDs that start with + /// __af_tool__ are reserved for the questions that route tool calls. + /// + public required IReadOnlyDictionary Questions { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs deleted file mode 100644 index 9af3f283071..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceAnswer.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents the answer to a . -/// -[Description("The answer to a Choice question.")] -public sealed class JevChoiceAnswer : JevAnswer -{ - /// - /// Gets or sets the option with the highest probability. - /// - [JsonPropertyName("choice")] - [Description("The option with the highest probability.")] - public required string Choice { get; set; } - - /// - /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the options. - /// - [JsonPropertyName("confidence")] - [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] - public required double Confidence { get; set; } - - /// - /// Gets or sets the probability of each option. The probabilities sum to 1. - /// - [JsonPropertyName("probabilities")] - [Description("The probability of each option. The probabilities sum to 1.")] - public required IReadOnlyDictionary Probabilities { get; set; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs index 7377a778036..a07918fa44d 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceQuestion.cs @@ -7,22 +7,22 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents a Choice question, which picks exactly one option from a set that the caller defines. +/// Represents a Choice question, which selects one of the named alternatives in . /// /// -/// The answer is a . +/// The answer is a . This type mirrors ChoiceQuestion of the official TypeSafe SDK. /// -[Description("A Choice question: picks exactly one option from criteria.")] +[Description("A Choice question: selects one of the named alternatives in criteria.")] public sealed class JevChoiceQuestion : JevQuestion { /// - /// Gets or sets the options, keyed by the option name that the answer returns. + /// Gets or sets the alternatives, keyed by the label that the answer returns. /// /// - /// Each value describes when the option applies, or is when the name is enough. - /// Jev accepts from 1 to 255 options. + /// Each value describes the alternative as text or JSON, or is when the label is + /// enough. Jev accepts from 1 to 255 alternatives. /// [JsonPropertyName("criteria")] - [Description("The options to choose from, keyed by the option name that the answer returns. Each value describes when the option applies. From 1 to 255 options.")] - public required IReadOnlyDictionary Criteria { get; set; } + [Description("The alternatives, keyed by the label that the answer returns. Each value describes the alternative as text or JSON, or is null when the label is enough. From 1 to 255 alternatives.")] + public required IReadOnlyDictionary Criteria { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs new file mode 100644 index 00000000000..103fb2b12ef --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChoiceResponse.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a : the selected label and its probabilities. +/// +/// +/// This type mirrors ChoiceResponse of the official TypeSafe SDK. +/// +[Description("The answer to a Choice question: the selected label and its probabilities.")] +public sealed class JevChoiceResponse : JevResponse +{ + /// + /// Gets or sets the selected label. + /// + [JsonPropertyName("choice")] + [Description("The selected label.")] + public required string Choice { get; set; } + + /// + /// Gets or sets the reported confidence in the selected label, from 0 to 1. + /// + [JsonPropertyName("confidence")] + [Description("Reported confidence in the selected label, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the probabilities, keyed by label. They sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("Probabilities keyed by label. They sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs new file mode 100644 index 00000000000..ae0999a26ad --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevEntry.cs @@ -0,0 +1,138 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents a Jev entry: text, a JSON object or array, or . +/// +/// +/// This type mirrors EntryType of the official TypeSafe SDK. The state of a , the +/// instructions of a , and every criterion description are entries, so each can be plain +/// text or structured data that questions refer to by field name. A converts implicitly, a +/// converts explicitly because a JSON number or Boolean is not an entry, and +/// is the entry. +/// +[JsonConverter(typeof(JevEntryJsonConverter))] +public readonly struct JevEntry : IEquatable +{ + private readonly JsonElement _value; + + /// + /// Initializes a new instance of the struct from JSON. + /// + /// A JSON string, object, array, or null. + /// is a JSON number or Boolean. + public JevEntry(JsonElement value) + { + if (value.ValueKind is not (JsonValueKind.String or JsonValueKind.Object or JsonValueKind.Array or JsonValueKind.Null or JsonValueKind.Undefined)) + { + Throw.ArgumentException(nameof(value), "A Jev entry must be text, a JSON object or array, or null."); + } + + // Cloning detaches the value from the document it came from, which may be disposed after this call. + this._value = value.ValueKind is JsonValueKind.Null or JsonValueKind.Undefined ? default : value.Clone(); + } + + /// + /// Gets the entry, which leaves a criterion undescribed. + /// + public static JevEntry Null => default; + + /// + /// Gets the kind of the entry: , , + /// , or . + /// + public JsonValueKind Kind => this._value.ValueKind == JsonValueKind.Undefined ? JsonValueKind.Null : this._value.ValueKind; + + /// + /// Gets a value indicating whether the entry is . + /// + public bool IsNull => this.Kind == JsonValueKind.Null; + + /// + /// Gets the text of an entry whose is , or otherwise. + /// + public string? Text => this.Kind == JsonValueKind.String ? this._value.GetString() : null; + + /// + /// Gets the entry as JSON, or for the entry. + /// + public JsonElement? Json => this.IsNull ? null : this._value; + + /// + /// Creates an entry from text. + /// + /// The text, or for the entry. + /// The entry. + public static JevEntry FromString(string? text) => + text is null ? default : new JevEntry(JsonSerializer.SerializeToElement(text, JevJsonContext.Default.String)); + + /// + /// Creates an entry from JSON. + /// + /// A JSON string, object, array, or null. + /// The entry. + public static JevEntry FromJsonElement(JsonElement value) => new(value); + + /// + /// Converts text to an entry. + /// + /// The text, or for the entry. + public static implicit operator JevEntry(string? text) => FromString(text); + + /// + /// Converts JSON to an entry. + /// + /// A JSON string, object, array, or null. + /// is a JSON number or Boolean. + /// + /// The conversion is explicit because it fails for JSON numbers and Booleans, which are not entries. + /// + public static explicit operator JevEntry(JsonElement value) => FromJsonElement(value); + + /// Determines whether two entries have the same JSON value. + /// The first entry. + /// The second entry. + /// when the entries are equal. + public static bool operator ==(JevEntry left, JevEntry right) => left.Equals(right); + + /// Determines whether two entries have different JSON values. + /// The first entry. + /// The second entry. + /// when the entries differ. + public static bool operator !=(JevEntry left, JevEntry right) => !left.Equals(right); + + /// + public bool Equals(JevEntry other) => + this.Kind == other.Kind && (this.IsNull || JsonElement.DeepEquals(this._value, other._value)); + + /// + public override bool Equals(object? obj) => obj is JevEntry other && this.Equals(other); + + /// + /// + /// Objects are equal regardless of property order, so only the kind (and the text of a string) is hashed. + /// + public override int GetHashCode() => + this.Kind == JsonValueKind.String + ? HashCode.Combine(this.Kind, this._value.GetString()) + : this.Kind.GetHashCode(); + + /// + /// Returns the text of a string entry, the JSON of an object or array entry, or an empty string for the + /// entry. + /// + /// The entry as a string. + public override string ToString() => + this.Kind switch + { + JsonValueKind.String => this._value.GetString()!, + JsonValueKind.Null => string.Empty, + _ => this._value.GetRawText(), + }; +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs deleted file mode 100644 index b737c69c977..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulAnswer.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.ComponentModel; -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents the answer to a . -/// -[Description("The answer to a Noul question.")] -public sealed class JevNoulAnswer : JevAnswer -{ - /// - /// Gets or sets the probability, from 0 to 1, that the answer is yes. - /// - [JsonPropertyName("noul")] - [Description("The probability, from 0 to 1, that the answer is yes.")] - public required double Noul { get; set; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs index 4540e047ec0..609b9c0b90a 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulCriteria.cs @@ -6,23 +6,23 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Describes what a yes and a no mean for a . +/// Describes the yes and no outcomes of a . /// public sealed class JevNoulCriteria { /// - /// Gets or sets what a yes, a probability near 1, means. + /// Gets or sets the description of the yes outcome, as text or JSON. /// [JsonPropertyName("true")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("What a yes, a probability near 1, means.")] - public string? True { get; set; } + [Description("Description of the yes outcome, as text or JSON.")] + public JevEntry? True { get; set; } /// - /// Gets or sets what a no, a probability near 0, means. + /// Gets or sets the description of the no outcome, as text or JSON. /// [JsonPropertyName("false")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("What a no, a probability near 0, means.")] - public string? False { get; set; } + [Description("Description of the no outcome, as text or JSON.")] + public JevEntry? False { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs index bee933254dd..cf54a32eefe 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulQuestion.cs @@ -6,19 +6,20 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents a Noul question, which returns the probability that a yes or no statement is true. +/// Represents a Noul question, a yes or no question whose answer is the probability of yes. /// /// -/// The answer is a . +/// The answer is a . A Noul needs , +/// , or both. This type mirrors NoulQuestion of the official TypeSafe SDK. /// -[Description("A Noul question: returns the probability, from 0 to 1, that the yes or no statement in instructions is true.")] +[Description("A Noul question: a yes or no question whose answer is the probability of yes. Needs instructions, criteria, or both.")] public sealed class JevNoulQuestion : JevQuestion { /// - /// Gets or sets optional descriptions of what a yes and a no mean. + /// Gets or sets optional descriptions of the yes and no outcomes. /// [JsonPropertyName("criteria")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("Optional descriptions of what a yes and a no mean.")] + [Description("Optional descriptions of the yes and no outcomes.")] public JevNoulCriteria? Criteria { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs new file mode 100644 index 00000000000..f67e5cab17b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevNoulResponse.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a : the probability of yes. +/// +/// +/// This type mirrors NoulResponse of the official TypeSafe SDK. +/// +[Description("The answer to a Noul question: the probability of yes.")] +public sealed class JevNoulResponse : JevResponse +{ + /// + /// Gets or sets the probability of a yes answer, from 0 to 1. + /// + [JsonPropertyName("noul")] + [Description("Probability of a yes answer, from 0 to 1.")] + public required double Noul { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs index af3fe909da1..bbaa8c956c8 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevQuestion.cs @@ -10,7 +10,8 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// /// The JSON type property selects the question kind: choice (), -/// score (), or noul (). +/// score (), or noul (). It mirrors +/// Question of the official TypeSafe SDK. /// [JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] [JsonDerivedType(typeof(JevChoiceQuestion), "choice")] @@ -24,9 +25,15 @@ private protected JevQuestion() } /// - /// Gets or sets what Jev should decide, rate, or check. + /// Gets or sets what Jev should decide, rate, or check, as text or JSON, or when the + /// criteria say enough. /// + /// + /// Structured instructions hold the question in one field and data it refers to in others, for example + /// { "potential_duplicate": { ... }, "question": "Is the resume for the same person as `potential_duplicate`?" }. + /// [JsonPropertyName("instructions")] - [Description("What Jev should decide, rate, or check. Describe the idea rather than exact words, because Jev matches on meaning.")] - public required string Instructions { get; set; } + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [Description("What Jev should decide, rate, or check: text, or a JSON object that holds the question in one field and the data it refers to in others. Optional when the criteria say enough. Describe the idea rather than exact words, because Jev matches on meaning.")] + public JevEntry? Instructions { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs index 2899a89ccf8..b4529531d26 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevRequest.cs @@ -2,38 +2,38 @@ using System.Collections.Generic; using System.ComponentModel; -using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents the input of a Jev evaluation: the state to evaluate and the typed questions to answer about it. +/// Represents the input of a Jev call: the state and the typed questions to answer about it. /// /// -/// This type mirrors the body of a TypeSafe System One API request without the model, which the tool configuration -/// selects. Its JSON form is the argument shape of the that -/// creates. +/// This type mirrors SystemOneRequest of the official TypeSafe SDK without the model, which the tool +/// configuration selects. Its JSON form is the argument shape of the that +/// creates. /// public sealed class JevRequest { /// - /// Gets or sets the content to evaluate. + /// Gets or sets the content that the questions are about. /// /// - /// A JSON string for plain text, or a JSON object or array for structured data, such as a conversation or - /// related records. + /// Text, or a JSON object or array for structured data, such as a conversation or related records. It mirrors + /// state of SystemOneRequest in the official TypeSafe SDK, except that the API rejects + /// . /// [JsonPropertyName("state")] - [Description("The content to evaluate. Use a string for plain text, or a JSON object or array for structured data. Questions can refer to object fields by name in backticks, for example `order`.")] - public required JsonElement State { get; set; } + [Description("The content to answer questions about: text, or a JSON object or array for structured data. Questions can refer to object fields by name in backticks, for example `order`.")] + public required JevEntry State { get; set; } /// /// Gets or sets the questions to answer, keyed by an ID that the caller chooses. /// /// - /// Each answer in uses the ID of its question. + /// Each answer in uses the ID of its question. /// [JsonPropertyName("questions")] [Description("The questions to answer about the state, keyed by an ID you choose. Each answer comes back under the same ID. Ask all related questions in one call.")] diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs index 944240124d8..f04265209d5 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResponse.cs @@ -1,41 +1,25 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Collections.Generic; -using System.ComponentModel; using System.Text.Json.Serialization; -using Microsoft.Extensions.AI; namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents the result of a Jev evaluation: one typed answer per question of the . +/// Represents Jev's answer to one question of a . /// /// -/// This type mirrors the body of a TypeSafe System One API response. Its JSON form is the result of the -/// that creates. +/// The JSON type property matches the kind of the question: choice (), +/// score (), or noul (). The derived types +/// mirror ChoiceResponse, ScoreResponse, and NoulResponse of the official TypeSafe SDK. /// -public sealed class JevResponse +[JsonPolymorphic(TypeDiscriminatorPropertyName = "type")] +[JsonDerivedType(typeof(JevChoiceResponse), "choice")] +[JsonDerivedType(typeof(JevScoreResponse), "score")] +[JsonDerivedType(typeof(JevNoulResponse), "noul")] +public abstract class JevResponse { - /// - /// Gets or sets the model that performed the evaluation, when the evaluator reports it. - /// - [JsonPropertyName("model")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("The model that performed the evaluation.")] - public string? Model { get; set; } - - /// - /// Gets or sets the answers, keyed by the ID of the question they answer. - /// - [JsonPropertyName("answers")] - [Description("One answer per question, keyed by the question ID.")] - public required IReadOnlyDictionary Answers { get; set; } - - /// - /// Gets or sets the token usage of the evaluation, when the evaluator reports it. - /// - [JsonPropertyName("usage")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - [Description("The token usage of the evaluation.")] - public JevUsage? Usage { get; set; } + // Only the three response kinds that the Jev API returns can exist. + private protected JevResponse() + { + } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs new file mode 100644 index 00000000000..a847eb2f774 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResult.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the result of a Jev call: one typed answer per question of the , with the model +/// and token usage. +/// +/// +/// This type mirrors SystemOneResult of the official TypeSafe SDK. Its JSON form is the result of the +/// that creates. +/// +public sealed class JevResult +{ + /// + /// Gets or sets the model that answered the request. + /// + [JsonPropertyName("model")] + [Description("The model that answered the request.")] + public required string Model { get; set; } + + /// + /// Gets or sets the answers, keyed by the ID of the question they answer. + /// + [JsonPropertyName("answers")] + [Description("One answer per question, keyed by the question ID.")] + public required IReadOnlyDictionary Answers { get; set; } + + /// + /// Gets or sets the token usage of the request. + /// + [JsonPropertyName("usage")] + [Description("The token usage of the request.")] + public required JevUsage Usage { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs new file mode 100644 index 00000000000..a233cf7e45d --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevResultExtensions.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.Extensions.AI; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Provides extension methods that read the of a response. +/// +public static class JevResultExtensions +{ + /// + /// Gets the typed answers of a response. + /// + /// The chat response. + /// + /// The answers to the request's questions, with their probabilities and token usage, or + /// when the response did not come from a or ended with a tool call. + /// + public static JevResult? GetJevResult(this ChatResponse response) + { + _ = Throw.IfNull(response); + return response.RawRepresentation as JevResult ?? FromMessages(response.Messages); + } + + /// + /// Gets the typed answers of an agent response produced with a . + /// + /// The agent response. + /// + /// The answers to the run's questions, with their probabilities and token usage, or when + /// the response has none. + /// + public static JevResult? GetJevResult(this AgentResponse response) + { + _ = Throw.IfNull(response); + + // A chat client agent keeps the chat response as the raw representation; agents that rebuild the response + // still keep the messages, which carry the result too. + return response.RawRepresentation switch + { + JevResult result => result, + ChatResponse chatResponse => chatResponse.GetJevResult(), + _ => FromMessages(response.Messages), + }; + } + + private static JevResult? FromMessages(IList messages) + { + for (int index = messages.Count - 1; index >= 0; index--) + { + if (messages[index].RawRepresentation is JevResult result) + { + return result; + } + } + + return null; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs deleted file mode 100644 index a4f4be565b0..00000000000 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreAnswer.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.ComponentModel; -using System.Text.Json.Serialization; - -namespace Microsoft.Agents.AI.TypeSafe; - -/// -/// Represents the answer to a . -/// -[Description("The answer to a Score question.")] -public sealed class JevScoreAnswer : JevAnswer -{ - /// - /// Gets or sets the score, weighted by the probability of each level. It can fall between levels. - /// - [JsonPropertyName("score")] - [Description("The score, weighted by the probability of each level. It can fall between levels.")] - public required double Score { get; set; } - - /// - /// Gets or sets how certain the answer is, from 0 to 1, derived from how the probability is spread across the levels. - /// - [JsonPropertyName("confidence")] - [Description("How certain the answer is, from 0 to 1. Treat low confidence as uncertain.")] - public required double Confidence { get; set; } - - /// - /// Gets or sets the description of each level, keyed by the level number. - /// - [JsonPropertyName("legend")] - [Description("The description of each level, keyed by the level number.")] - public required IReadOnlyDictionary Legend { get; set; } - - /// - /// Gets or sets the probability of each level, keyed by the level number. The probabilities sum to 1. - /// - [JsonPropertyName("probabilities")] - [Description("The probability of each level, keyed by the level number. The probabilities sum to 1.")] - public required IReadOnlyDictionary Probabilities { get; set; } -} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs index fa8a7954b1a..fde0cc3f19e 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreQuestion.cs @@ -7,21 +7,23 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents a Score question, which rates the state against ordered levels that the caller defines. +/// Represents a Score question, which assigns a score using the ordered rubric in . /// /// -/// The answer is a , whose score is weighted by probability and can fall between levels. +/// The answer is a , whose score is weighted by probability and can fall between +/// levels. This type mirrors ScoreQuestion of the official TypeSafe SDK. /// -[Description("A Score question: rates the state against ordered levels in criteria.")] +[Description("A Score question: assigns a score using the ordered rubric in criteria.")] public sealed class JevScoreQuestion : JevQuestion { /// - /// Gets or sets the level descriptions, ordered from level 0 (the lowest) to the highest level. + /// Gets or sets the rubric: one description per score, indexed from zero (the lowest score). /// /// - /// A Score needs from 2 to 10 levels. + /// Each description is text or JSON. The official SDK types also allow to leave a + /// score undescribed, but the API rejects it. A Score needs from 2 to 10 scores. /// [JsonPropertyName("criteria")] - [Description("The level descriptions, ordered from level 0 (the lowest) to the highest level. From 2 to 10 levels.")] - public required IReadOnlyList Criteria { get; set; } + [Description("The rubric: one description per score, as text or JSON, indexed from zero (the lowest score). From 2 to 10 scores.")] + public required IReadOnlyList Criteria { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs new file mode 100644 index 00000000000..30cec062631 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevScoreResponse.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.ComponentModel; +using System.Text.Json.Serialization; + +namespace Microsoft.Agents.AI.TypeSafe; + +/// +/// Represents the answer to a : an expected score with its rubric and probabilities. +/// +/// +/// This type mirrors ScoreResponse of the official TypeSafe SDK. +/// +[Description("The answer to a Score question: an expected score with its rubric and probabilities.")] +public sealed class JevScoreResponse : JevResponse +{ + /// + /// Gets or sets the expected score, which may fall between integer rubric levels. + /// + [JsonPropertyName("score")] + [Description("Expected score, which may fall between integer rubric levels.")] + public required double Score { get; set; } + + /// + /// Gets or sets the reported confidence in the score, from 0 to 1. + /// + [JsonPropertyName("confidence")] + [Description("Reported confidence in the score, from 0 to 1. Treat low confidence as uncertain.")] + public required double Confidence { get; set; } + + /// + /// Gets or sets the rubric descriptions, keyed by score. They echo the rubric of the question. + /// + [JsonPropertyName("legend")] + [Description("Rubric descriptions keyed by score.")] + public required IReadOnlyDictionary Legend { get; set; } + + /// + /// Gets or sets the probabilities, keyed by score. They sum to 1. + /// + [JsonPropertyName("probabilities")] + [Description("Probabilities keyed by score. They sum to 1.")] + public required IReadOnlyDictionary Probabilities { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs index 1c004164070..34cbff9229f 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevUsage.cs @@ -6,7 +6,7 @@ namespace Microsoft.Agents.AI.TypeSafe; /// -/// Represents the token usage of a Jev evaluation. +/// Represents the token usage of a Jev call. This type mirrors Usage of the official TypeSafe SDK. /// public sealed class JevUsage { diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj index 203d77b4310..90159d6d784 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Microsoft.Agents.AI.TypeSafe.csproj @@ -15,15 +15,20 @@ Microsoft Agent Framework - TypeSafe (Jev) integration - Provides an AIFunction that lets agents classify, route, score, and check content with Jev, the TypeSafe System One model. + Provides an IChatClient and an AIFunction that let agents classify, route, score, and check content with Jev, the TypeSafe System One model. + + + + + diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs index 9135880f01b..386c834b9a9 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIFunctionTests.cs @@ -14,14 +14,14 @@ namespace Microsoft.Agents.AI.TypeSafe.UnitTests; public sealed class JevAIFunctionTests { [Fact] - public async Task InvokeAsync_ModelArguments_ReachTheEvaluatorAsTypedRequestAsync() + public async Task InvokeAsync_ModelArguments_ReachTheMappingAsTypedRequestAsync() { // Arrange JevRequest? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => { received = request; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); // Act @@ -29,27 +29,27 @@ public async Task InvokeAsync_ModelArguments_ReachTheEvaluatorAsTypedRequestAsyn // Assert Assert.NotNull(received); - Assert.Equal(JevTestData.TicketText, received.State.GetString()); + Assert.Equal(JevTestData.TicketText, received.State.Text); JevChoiceQuestion department = Assert.IsType(received.Questions["department"]); - Assert.Equal("Which team should handle this", department.Instructions); + Assert.Equal("Which team should handle this", department.Instructions?.Text); Assert.Equal(["billing", "technical", "sales"], department.Criteria.Keys); - Assert.Null(department.Criteria["sales"]); + Assert.True(department.Criteria["sales"].IsNull); JevScoreQuestion frustration = Assert.IsType(received.Questions["frustration"]); Assert.Equal(3, frustration.Criteria.Count); - Assert.Equal("Calm, just stating facts", frustration.Criteria[0]); + Assert.Equal("Calm, just stating facts", frustration.Criteria[0].Text); JevNoulQuestion urgent = Assert.IsType(received.Questions["is_urgent"]); - Assert.Equal("Explicitly time-sensitive", urgent.Criteria!.True); - Assert.Equal("No urgency expressed", urgent.Criteria.False); + Assert.Equal("Explicitly time-sensitive", urgent.Criteria!.True?.Text); + Assert.Equal("No urgency expressed", urgent.Criteria.False?.Text); } [Fact] - public async Task InvokeAsync_ReturnsTheResponseAsJsonWithWireNamesAsync() + public async Task InvokeAsync_ReturnsTheResultAsJsonWithWireNamesAsync() { // Arrange - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Act object? result = await tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); @@ -73,10 +73,10 @@ public async Task InvokeAsync_TypeWrittenLastAndObjectState_AreAcceptedAsync() { // Arrange JevRequest? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => { received = request; - return Task.FromResult(new JevResponse { Answers = new Dictionary { ["paid"] = new JevNoulAnswer { Noul = 0.1 } } }); + return Task.FromResult(Result("paid", new JevNoulResponse { Noul = 0.1 })); }).Build(); // Act @@ -88,20 +88,65 @@ await tool.InvokeAsync(JevTestData.Arguments(""" """)); // Assert - Assert.Equal(JsonValueKind.Object, received!.State.ValueKind); - Assert.Equal("A-104", received.State.GetProperty("order").GetProperty("id").GetString()); + Assert.Equal(JsonValueKind.Object, received!.State.Kind); + Assert.Equal("A-104", received.State.Json!.Value.GetProperty("order").GetProperty("id").GetString()); Assert.IsType(received.Questions["paid"]); } + [Fact] + public async Task InvokeAsync_StructuredEntries_AreAcceptedLikeTheSdkAsync() + { + // Arrange + JevRequest? received = null; + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => + { + received = request; + return Task.FromResult(new JevResult + { + Model = "m", + Answers = new Dictionary + { + ["same_person"] = new JevNoulResponse { Noul = 0.9 }, + ["team"] = new JevChoiceResponse { Choice = "billing", Confidence = 1, Probabilities = new Dictionary { ["billing"] = 1 } }, + }, + Usage = new JevUsage { InputTokens = 1, OutputTokens = 1 }, + }); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(""" + { + "state": ["Hi", "The payment failed."], + "questions": { + "same_person": { + "type": "noul", + "instructions": { "potential_duplicate": { "name": "John Smith" }, "question": "Is the resume for the same person as `potential_duplicate`?" } + }, + "team": { "type": "choice", "criteria": { "billing": { "covers": ["payments", "refunds"] } } } + } + } + """)); + + // Assert + Assert.Equal(JsonValueKind.Array, received!.State.Kind); + JevEntry instructions = Assert.IsType(received.Questions["same_person"]).Instructions!.Value; + Assert.Equal(JsonValueKind.Object, instructions.Kind); + Assert.Equal("John Smith", instructions.Json!.Value.GetProperty("potential_duplicate").GetProperty("name").GetString()); + + JevChoiceQuestion team = Assert.IsType(received.Questions["team"]); + Assert.Null(team.Instructions); + Assert.Equal(JsonValueKind.Object, team.Criteria["billing"].Kind); + } + [Fact] public async Task InvokeAsync_DotNetObjectArguments_AreAcceptedAsync() { // Arrange JevRequest? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((request, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((request, _, _) => { received = request; - return Task.FromResult(new JevResponse { Answers = new Dictionary { ["q"] = new JevNoulAnswer { Noul = 0.5 } } }); + return Task.FromResult(Result("q", new JevNoulResponse { Noul = 0.5 })); }).Build(); // Act @@ -112,31 +157,33 @@ await tool.InvokeAsync(new AIFunctionArguments }); // Assert - Assert.Equal("The payment failed.", received!.State.GetString()); - Assert.Equal("Did the payment fail?", Assert.IsType(received.Questions["q"]).Instructions); + Assert.Equal("The payment failed.", received!.State.Text); + Assert.Equal("Did the payment fail?", Assert.IsType(received.Questions["q"]).Instructions?.Text); } [Theory] [InlineData("""{"questions":{"q":{"type":"noul","instructions":"x"}}}""", "state")] - [InlineData("""{"state":42,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "The state must be a string")] - [InlineData("""{"state":"x","questions":{}}""", "at least one question")] + [InlineData("""{"state":42,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "must be text, a JSON object or array, or null")] + [InlineData("""{"state":null,"questions":{"q":{"type":"noul","instructions":"x"}}}""", "The state must be text or a JSON object or array")] + [InlineData("""{"state":"x","questions":{}}""", "At least one question is required")] [InlineData("""{"state":"x","questions":{"q":{"type":"rank","instructions":"x"}}}""", "not a valid Jev request")] [InlineData("""{"state":"x","questions":{"q":{"instructions":"x"}}}""", "not a valid Jev request")] [InlineData("""{"state":"x","questions":{"q":null}}""", "Question 'q' is null")] - [InlineData("""{"state":"x","questions":{"q":{"type":"noul","instructions":" "}}}""", "Question 'q' must have instructions")] - [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{}}}}""", "has 0 options; it needs from 1 to 255")] - [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{"":"empty"}}}}""", "empty option name")] - [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["only"]}}}""", "has 1 level; it needs from 2 to 10")] - [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["0","1","2","3","4","5","6","7","8","9","10"]}}}""", "has 11 levels")] - [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["low",""]}}}""", "empty level description")] - public async Task InvokeAsync_InvalidArguments_FailBeforeEvaluatingAsync(string arguments, string expectedMessage) + [InlineData("""{"state":"x","questions":{"q":{"type":"noul","instructions":" "}}}""", "Noul question 'q' needs instructions or criteria")] + [InlineData("""{"state":"x","questions":{"q":{"type":"noul","criteria":{"true":null}}}}""", "Noul question 'q' needs instructions or criteria")] + [InlineData("""{"state":"x","questions":{"q":{"type":"choice","instructions":"x","criteria":{}}}}""", "has 0 alternatives; it needs from 1 to 255")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["only"]}}}""", "has 1 criterion; it needs from 2 to 10")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":["0","1","2","3","4","5","6","7","8","9","10"]}}}""", "has 11 criteria")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":[1,2]}}}""", "must be text, a JSON object or array, or null")] + [InlineData("""{"state":"x","questions":{"q":{"type":"score","instructions":"x","criteria":[null,"High"]}}}""", "has a null criterion at index 0")] + public async Task InvokeAsync_InvalidArguments_FailBeforeCallingAsync(string arguments, string expectedMessage) { // Arrange int calls = 0; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => { calls++; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); // Act @@ -148,42 +195,79 @@ public async Task InvokeAsync_InvalidArguments_FailBeforeEvaluatingAsync(string } [Fact] - public async Task InvokeAsync_TooManyChoiceOptions_FailBeforeEvaluatingAsync() + public async Task InvokeAsync_QuestionsWithoutInstructionsButWithCriteria_AreAcceptedAsync() { // Arrange - string options = string.Join(",", Enumerable.Range(0, 256).Select(static i => $"\"o{i}\":null")); - string arguments = "{\"state\":\"x\",\"questions\":{\"q\":{\"type\":\"choice\",\"instructions\":\"x\",\"criteria\":{" + options + "}}}}"; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + int calls = 0; + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => + { + calls++; + return Task.FromResult(new JevResult + { + Model = "m", + Answers = new Dictionary + { + ["urgent"] = new JevNoulResponse { Noul = 0.2 }, + ["level"] = new JevScoreResponse + { + Score = 0, + Confidence = 1, + Legend = new Dictionary { ["0"] = "Low", ["1"] = "High" }, + Probabilities = new Dictionary { ["0"] = 1, ["1"] = 0 }, + }, + }, + Usage = new JevUsage { InputTokens = 1, OutputTokens = 1 }, + }); + }).Build(); + + // Act + await tool.InvokeAsync(JevTestData.Arguments(""" + {"state":"x","questions":{ + "urgent":{"type":"noul","criteria":{"true":"Explicitly urgent"}}, + "level":{"type":"score","criteria":["Low",{"level":"high","signals":["outage"]}]}}} + """)); + + // Assert + Assert.Equal(1, calls); + } + + [Fact] + public async Task InvokeAsync_TooManyChoiceAlternatives_FailBeforeCallingAsync() + { + // Arrange + string alternatives = string.Join(",", Enumerable.Range(0, 256).Select(static i => $"\"o{i}\":null")); + string arguments = "{\"state\":\"x\",\"questions\":{\"q\":{\"type\":\"choice\",\"instructions\":\"x\",\"criteria\":{" + alternatives + "}}}}"; + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Act ArgumentException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(arguments)).AsTask()); // Assert - Assert.Contains("has 256 options", exception.Message, StringComparison.Ordinal); + Assert.Contains("has 256 alternatives", exception.Message, StringComparison.Ordinal); } - public static TheoryData, string> InvalidResponses => new() + public static TheoryData, string> InvalidResults => new() { { "no answer", static r => Without(r, "is_urgent"), "no answer for question 'is_urgent'" }, - { "wrong kind", static r => With(r, "department", new JevNoulAnswer { Noul = 0.5 }), "is a Noul answer, but the question is a Choice question" }, - { "unknown option", static r => With(r, "department", Choice("refunds", 0.9)), "chose 'refunds', which is not one of its options" }, + { "wrong kind", static r => With(r, "department", new JevNoulResponse { Noul = 0.5 }), "is a Noul answer, but the question is a Choice question" }, + { "unknown label", static r => With(r, "department", Choice("refunds", 0.9)), "chose 'refunds', which is not one of its labels" }, { "confidence out of range", static r => With(r, "department", Choice("technical", 1.2)), "has confidence 1.2, which is outside the range 0 to 1" }, { "score out of range", static r => With(r, "frustration", Score(3)), "has score 3, which is outside the range 0 to 2" }, - { "noul as percent", static r => With(r, "is_urgent", new JevNoulAnswer { Noul = 97 }), "has noul 97, which is outside the range 0 to 1" }, - { "no answers", static r => new JevResponse { Answers = null! }, "has no answers" }, - { "probability for an option not offered", static r => With(r, "department", ChoiceWith("technical", new() { ["TECHNICAL"] = 1 })), "has probabilities for 'TECHNICAL', which is not one of its options" }, + { "noul as percent", static r => With(r, "is_urgent", new JevNoulResponse { Noul = 97 }), "has noul 97, which is outside the range 0 to 1" }, + { "no answers", static r => new JevResult { Model = r.Model, Answers = null!, Usage = r.Usage }, "has no answers" }, + { "probability for a label not offered", static r => With(r, "department", ChoiceWith("technical", new() { ["TECHNICAL"] = 1 })), "has probabilities for 'TECHNICAL', which is not one of its labels" }, { "probability above one", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = 42 })), "has probability 'technical' 42, which is outside the range 0 to 1" }, { "probability not a number", static r => With(r, "department", ChoiceWith("technical", new() { ["technical"] = double.NaN })), "has probability 'technical' NaN" }, - { "score probability for a missing level", static r => With(r, "frustration", ScoreWith(new() { ["3"] = 1 }, [])), "has probabilities for '3', which is not a level number from 0 to 2" }, - { "legend for a missing level", static r => With(r, "frustration", ScoreWith([], new() { ["high"] = "Very angry" })), "has legend for 'high', which is not a level number from 0 to 2" }, + { "score probability for a missing score", static r => With(r, "frustration", ScoreWith(new() { ["3"] = 1 }, [])), "has probabilities for '3', which is not a score from 0 to 2" }, + { "legend for a missing score", static r => With(r, "frustration", ScoreWith([], new() { ["high"] = "Very angry" })), "has legend for 'high', which is not a score from 0 to 2" }, }; [Theory] - [MemberData(nameof(InvalidResponses))] - public async Task InvokeAsync_InvalidEvaluatorResponse_FailsAsync(string scenario, Func corrupt, string expectedMessage) + [MemberData(nameof(InvalidResults))] + public async Task InvokeAsync_InvalidMappingResult_FailsAsync(string scenario, Func corrupt, string expectedMessage) { // Arrange - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(corrupt(JevTestData.CreateResponse()))).Build(); + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(corrupt(JevTestData.CreateResult()))).Build(); // Act InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); @@ -193,28 +277,28 @@ public async Task InvokeAsync_InvalidEvaluatorResponse_FailsAsync(string scenari } [Fact] - public async Task InvokeAsync_EvaluatorReturnsNull_FailsAsync() + public async Task InvokeAsync_MappingReturnsNull_FailsAsync() { // Arrange - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(null!)).Build(); + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(null!)).Build(); // Act InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); // Assert - Assert.Contains("returned no response", exception.Message, StringComparison.Ordinal); + Assert.Contains("returned no result", exception.Message, StringComparison.Ordinal); } [Fact] - public async Task InvokeAsync_PassesTheCancellationTokenToTheEvaluatorAsync() + public async Task InvokeAsync_PassesTheCancellationTokenToTheMappingAsync() { // Arrange using var cancellation = new CancellationTokenSource(); CancellationToken received = default; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, _, cancellationToken) => + AIFunction tool = new JevAIToolBuilder().UseMapping((_, _, cancellationToken) => { received = cancellationToken; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); // Act @@ -234,15 +318,9 @@ public async Task UseClient_MapsBetweenTheContractAndTheClientTypesAsync() inputMapper: static request => { (string id, JevQuestion question) = request.Questions.Single(); - return new LabelRequest(id, request.State.GetString()!, ((JevChoiceQuestion)question).Criteria.Keys.ToList()); + return new LabelRequest(id, request.State.Text!, ((JevChoiceQuestion)question).Criteria.Keys.ToList()); }, - outputMapper: static response => new JevResponse - { - Answers = new Dictionary - { - [response.QuestionId] = Choice(response.Label, response.Score), - }, - }).Build(); + outputMapper: static response => Result(response.QuestionId, Choice(response.Label, response.Score))).Build(); // Act object? result = await tool.InvokeAsync(JevTestData.Arguments(""" @@ -256,14 +334,14 @@ public async Task UseClient_MapsBetweenTheContractAndTheClientTypesAsync() } [Fact] - public async Task UseClient_MapperReturnsAnOptionThatWasNotOffered_FailsAsync() + public async Task UseClient_MapperReturnsALabelThatWasNotOffered_FailsAsync() { // Arrange var client = new LabelClient(label: "TECHNICAL"); AIFunction tool = new JevAIToolBuilder().UseClient( client.ClassifyAsync, - static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), - static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + static request => new LabelRequest("team", request.State.Text!, ["billing", "technical"]), + static response => Result("team", Choice(response.Label, response.Score))).Build(); // Act InvalidOperationException exception = await Assert.ThrowsAsync(() => tool.InvokeAsync(JevTestData.Arguments(""" @@ -275,15 +353,15 @@ public async Task UseClient_MapperReturnsAnOptionThatWasNotOffered_FailsAsync() } [Fact] - public async Task UseEvaluator_ReceivesTheServicesOfTheCallAsync() + public async Task UseMapping_ReceivesTheServicesOfTheCallAsync() { // Arrange using ServiceProvider services = new ServiceCollection().BuildServiceProvider(); IServiceProvider? received = null; - AIFunction tool = new JevAIToolBuilder().UseEvaluator((_, callServices, _) => + AIFunction tool = new JevAIToolBuilder().UseMapping((_, callServices, _) => { received = callServices; - return Task.FromResult(JevTestData.CreateResponse()); + return Task.FromResult(JevTestData.CreateResult()); }).Build(); AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); @@ -311,8 +389,8 @@ public async Task UseClient_RegisteredClient_IsResolvedFromTheServicesOnEveryCal AIFunction tool = new JevAIToolBuilder().UseClient( static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), - static request => new LabelRequest("team", request.State.GetString()!, ["billing", "technical"]), - static response => new JevResponse { Answers = new Dictionary { ["team"] = Choice(response.Label, response.Score) } }).Build(); + static request => new LabelRequest("team", request.State.Text!, ["billing", "technical"]), + static response => Result("team", Choice(response.Label, response.Score))).Build(); AIFunctionArguments arguments = JevTestData.Arguments(""" {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} @@ -338,7 +416,7 @@ public async Task UseClient_ClientNotAvailable_FailsWithWhatIsMissingAsync(bool AIFunction tool = new JevAIToolBuilder().UseClient( static (client, request, cancellationToken) => client.ClassifyAsync(request, cancellationToken), static request => new LabelRequest("q", "x", []), - static response => JevTestData.CreateResponse()).Build(); + static response => JevTestData.CreateResult()).Build(); AIFunctionArguments arguments = JevTestData.Arguments(JevTestData.RequestJson); arguments.Services = withServices ? services : null; @@ -366,8 +444,8 @@ public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsy AIFunction tool = new JevAIToolBuilder().UseClient( static (client, text, _) => Task.FromResult(client.IsDisposed ? "disposed" : "technical"), - static request => request.State.GetString()!, - static label => new JevResponse { Answers = new Dictionary { ["team"] = Choice(label, 0.9) } }).Build(); + static request => request.State.Text!, + static label => Result("team", Choice(label, 0.9))).Build(); AIFunctionArguments arguments = JevTestData.Arguments(""" {"state":"x","questions":{"team":{"type":"choice","instructions":"Which team?","criteria":{"billing":null,"technical":null}}}} @@ -383,29 +461,36 @@ public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsy Assert.All(created, static client => Assert.True(client.IsDisposed)); } - private static JevChoiceAnswer Choice(string choice, double confidence) => new() + private static JevResult Result(string id, JevResponse answer) => new() + { + Model = "custom-model", + Answers = new Dictionary { [id] = answer }, + Usage = new JevUsage { InputTokens = 1, OutputTokens = 1 }, + }; + + private static JevChoiceResponse Choice(string choice, double confidence) => new() { Choice = choice, Confidence = confidence, Probabilities = new Dictionary { [choice] = confidence }, }; - private static JevScoreAnswer Score(double score) => new() + private static JevScoreResponse Score(double score) => new() { Score = score, Confidence = 1, - Legend = new Dictionary(), + Legend = new Dictionary(), Probabilities = new Dictionary(), }; - private static JevChoiceAnswer ChoiceWith(string choice, Dictionary probabilities) => new() + private static JevChoiceResponse ChoiceWith(string choice, Dictionary probabilities) => new() { Choice = choice, Confidence = 0.5, Probabilities = probabilities, }; - private static JevScoreAnswer ScoreWith(Dictionary probabilities, Dictionary legend) => new() + private static JevScoreResponse ScoreWith(Dictionary probabilities, Dictionary legend) => new() { Score = 1, Confidence = 1, @@ -413,17 +498,17 @@ public async Task UseClient_ScopedClientFromTheRootProvider_GetsAScopePerCallAsy Probabilities = probabilities, }; - private static JevResponse With(JevResponse response, string id, JevAnswer answer) + private static JevResult With(JevResult result, string id, JevResponse answer) { - var answers = new Dictionary(response.Answers) { [id] = answer }; - return new JevResponse { Answers = answers }; + var answers = new Dictionary(result.Answers) { [id] = answer }; + return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; } - private static JevResponse Without(JevResponse response, string id) + private static JevResult Without(JevResult result, string id) { - var answers = new Dictionary(response.Answers); + var answers = new Dictionary(result.Answers); answers.Remove(id); - return new JevResponse { Answers = answers }; + return new JevResult { Model = result.Model, Answers = answers, Usage = result.Usage }; } /// The request type of a hypothetical third-party classification client. diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs index 1fd41c9ec41..9c7be795d9e 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAIToolBuilderTests.cs @@ -19,20 +19,32 @@ public void Build_Defaults_DescribeTheJevContract() AIFunction tool = new JevAIToolBuilder(new ApiKeyCredential("test-key")).Build(); // Assert - Assert.Equal("evaluate_with_jev", tool.Name); + Assert.Equal("ask_system_one", tool.Name); Assert.Contains("'choice'", tool.Description, StringComparison.Ordinal); Assert.Contains("'score'", tool.Description, StringComparison.Ordinal); Assert.Contains("'noul'", tool.Description, StringComparison.Ordinal); JsonElement schema = tool.JsonSchema; Assert.Equal(["state", "questions"], schema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + Assert.Equal(["string", "object", "array"], schema.GetProperty("properties").GetProperty("state").GetProperty("type").EnumerateArray().Select(static t => t.GetString())); JsonElement question = schema.GetProperty("properties").GetProperty("questions").GetProperty("additionalProperties"); Assert.Equal( ["choice", "score", "noul"], question.GetProperty("anyOf").EnumerateArray().Select(static b => b.GetProperty("properties").GetProperty("type").GetProperty("const").GetString())); + JsonElement[] branches = [.. question.GetProperty("anyOf").EnumerateArray()]; + JsonElement choiceCriteria = branches[0].GetProperty("properties").GetProperty("criteria"); + Assert.Equal(["string", "object", "array", "null"], choiceCriteria.GetProperty("additionalProperties").GetProperty("type").EnumerateArray().Select(static t => t.GetString())); + Assert.Equal(1, choiceCriteria.GetProperty("minProperties").GetInt32()); + Assert.Equal(255, choiceCriteria.GetProperty("maxProperties").GetInt32()); + JsonElement scoreCriteria = branches[1].GetProperty("properties").GetProperty("criteria"); + Assert.Equal(["string", "object", "array"], scoreCriteria.GetProperty("items").GetProperty("type").EnumerateArray().Select(static t => t.GetString())); + Assert.Equal(2, scoreCriteria.GetProperty("minItems").GetInt32()); + Assert.Equal(10, scoreCriteria.GetProperty("maxItems").GetInt32()); + Assert.Equal(4, branches[2].GetProperty("properties").GetProperty("instructions").GetProperty("type").GetArrayLength()); + JsonElement returnSchema = tool.ReturnJsonSchema!.Value; - Assert.Equal(["answers"], returnSchema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); + Assert.Equal(["model", "answers", "usage"], returnSchema.GetProperty("required").EnumerateArray().Select(static r => r.GetString())); Assert.True(returnSchema.GetProperty("properties").GetProperty("usage").GetProperty("properties").TryGetProperty("input_tokens", out _)); } @@ -45,16 +57,20 @@ public void Options_Defaults_MatchTheTypeSafeApi() // Assert Assert.Null(options.Endpoint); Assert.Equal("jev-latest", options.ModelId); - Assert.Equal("evaluate_with_jev", options.Name); + Assert.Equal("ask_system_one", options.Name); Assert.Null(options.RetryPolicy); } [Fact] - public void Build_WithTheTypeSafeApi_AssignsTheDefaultRetryPolicyOnlyWhenNoneIsSet() + public void Build_WithTheTypeSafeApi_AssignsTheSdkDefaultsOnlyWhenUnset() { // Arrange var defaults = new JevAIToolOptions(); - var custom = new JevAIToolOptions { RetryPolicy = new System.ClientModel.Primitives.ClientRetryPolicy(maxRetries: 1) }; + var custom = new JevAIToolOptions + { + RetryPolicy = new System.ClientModel.Primitives.ClientRetryPolicy(maxRetries: 1), + NetworkTimeout = TimeSpan.FromSeconds(30), + }; // Act _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), defaults).Build(); @@ -62,20 +78,22 @@ public void Build_WithTheTypeSafeApi_AssignsTheDefaultRetryPolicyOnlyWhenNoneIsS // Assert Assert.IsType(defaults.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(10), defaults.NetworkTimeout); Assert.IsType(custom.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(30), custom.NetworkTimeout); } [Fact] - public void Build_WithACustomEvaluator_IgnoresTheApiOnlyOptions() + public void Build_WithACustomMapping_IgnoresTheApiOnlyOptions() { // Arrange var options = new JevAIToolOptions { ModelId = " ", Endpoint = new Uri("/relative", UriKind.Relative) }; // Act - AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction tool = new JevAIToolBuilder(options).UseMapping(static (_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Assert - Assert.Equal("evaluate_with_jev", tool.Name); + Assert.Equal("ask_system_one", tool.Name); } [Fact] @@ -85,7 +103,7 @@ public void Build_WithNameAndDescription_UsesTheOptions() var options = new JevAIToolOptions { Name = "triage_ticket", Description = "Classifies support tickets." }; // Act - AIFunction tool = new JevAIToolBuilder(options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction tool = new JevAIToolBuilder(options).UseMapping(static (_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Assert Assert.Equal("triage_ticket", tool.Name); @@ -107,13 +125,13 @@ public void Build_WithTheTypeSafeApi_MakesTheOptionsReadOnly() } [Fact] - public void Build_WithACustomEvaluator_LeavesTheOptionsWritable() + public void Build_WithACustomMapping_LeavesTheOptionsWritable() { // Arrange var options = new JevAIToolOptions(); // Act - _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).UseEvaluator(static (_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + _ = new JevAIToolBuilder(new ApiKeyCredential("test-key"), options).UseMapping(static (_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); // Assert options.ModelId = "jev-1.13"; @@ -121,7 +139,7 @@ public void Build_WithACustomEvaluator_LeavesTheOptionsWritable() } [Fact] - public void Build_WithoutCredentialOrEvaluator_Throws() + public void Build_WithoutCredentialOrMapping_Throws() { // Act InvalidOperationException exception = Assert.Throws(() => new JevAIToolBuilder().Build()); @@ -156,17 +174,17 @@ public void Methods_NullArguments_Throw() { // Arrange var builder = new JevAIToolBuilder(); - Task evaluateAsync(string request, CancellationToken _) => Task.FromResult(request); + Task systemOneAsync(string request, CancellationToken _) => Task.FromResult(request); string input(JevRequest _) => "request"; - JevResponse output(string _) => JevTestData.CreateResponse(); - Task evaluateWithClientAsync(object _1, string request, CancellationToken _2) => Task.FromResult(request); + JevResult output(string _) => JevTestData.CreateResult(); + Task systemOneWithClientAsync(object _1, string request, CancellationToken _2) => Task.FromResult(request); // Act & Assert Assert.Throws(() => new JevAIToolBuilder((ApiKeyCredential)null!)); - Assert.Throws(() => builder.UseEvaluator(null!)); - Assert.Throws(() => builder.UseClient(null!, input, (Func)output)); - Assert.Throws(() => builder.UseClient((Func>)evaluateAsync, null!, output)); - Assert.Throws(() => builder.UseClient(evaluateAsync, input, null!)); - Assert.Throws(() => builder.UseClient((Func>)evaluateWithClientAsync, null!, output)); + Assert.Throws(() => builder.UseMapping(null!)); + Assert.Throws(() => builder.UseClient(null!, input, (Func)output)); + Assert.Throws(() => builder.UseClient((Func>)systemOneAsync, null!, output)); + Assert.Throws(() => builder.UseClient(systemOneAsync, input, null!)); + Assert.Throws(() => builder.UseClient((Func>)systemOneWithClientAsync, null!, output)); } } diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs index 4a3582d140f..fee2448f03a 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevAgentIntegrationTests.cs @@ -51,7 +51,7 @@ public async Task RunAsync_AgentCallsTheTool_ModelReceivesTheAnswersAsync() public async Task GetResponseAsync_InvalidArgumentsWithDetailedErrors_TellTheModelWhatToFixAsync() { // Arrange - AIFunction jev = new JevAIToolBuilder().UseEvaluator((_, _, _) => Task.FromResult(JevTestData.CreateResponse())).Build(); + AIFunction jev = new JevAIToolBuilder().UseMapping((_, _, _) => Task.FromResult(JevTestData.CreateResult())).Build(); const string InvalidArguments = """{"state":"x","questions":{"q":{"type":"score","instructions":"How bad?","criteria":["only"]}}}"""; using var model = new ScriptedChatClient( @@ -64,7 +64,7 @@ public async Task GetResponseAsync_InvalidArgumentsWithDetailedErrors_TellTheMod // Assert FunctionResultContent result = model.Calls[1].SelectMany(static m => m.Contents).OfType().Single(); - Assert.Contains("has 1 level; it needs from 2 to 10", result.Result?.ToString(), StringComparison.Ordinal); + Assert.Contains("has 1 criterion; it needs from 2 to 10", result.Result?.ToString(), StringComparison.Ordinal); } [Fact] @@ -77,10 +77,12 @@ public async Task RunAsync_AgentWithServices_ToolResolvesTheClientFromThemAsync( AIFunction jev = new JevAIToolBuilder() .UseClient( static (client, text, cancellationToken) => client.ScoreAsync(text, cancellationToken), - inputMapper: static request => request.State.GetString()!, - outputMapper: static probability => new JevResponse + inputMapper: static request => request.State.Text!, + outputMapper: static probability => new JevResult { - Answers = new Dictionary { ["is_urgent"] = new JevNoulAnswer { Noul = probability } }, + Model = "urgency-classifier", + Answers = new Dictionary { ["is_urgent"] = new JevNoulResponse { Noul = probability } }, + Usage = new JevUsage { InputTokens = 0, OutputTokens = 0 }, }) .Build(); diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs new file mode 100644 index 00000000000..8b328f0a2de --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientTests.cs @@ -0,0 +1,386 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevChatClientTests +{ + private const string Ticket = JevTestData.TicketText; + + [Fact] + public async Task GetResponseAsync_SendsTheConversationAndQuestions_ReturnsTheTypedResultAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + ChatMessage[] messages = [new(ChatRole.System, "You triage support tickets."), new(ChatRole.User, Ticket)]; + + // Act + ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Instructions = "Be strict." }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the request carries the conversation as the state, in the Python connector's shape. + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(new Uri("https://api.typesafe.ai/v1/systemone"), request.Uri); + Assert.Equal("test-key", request.Authorization!.Parameter); + JsonNode body = JsonNode.Parse(request.Body)!; + Assert.Equal("jev-latest", (string?)body["model"]); + JsonNode expectedState = JsonNode.Parse($$""" + { + "messages": [ + { "role": "system", "contents": [{ "type": "text", "text": "You triage support tickets." }] }, + { "role": "user", "contents": [{ "type": "text", "text": {{JsonSerializer.Serialize(Ticket)}} }] } + ], + "instructions": "Be strict." + } + """)!; + Assert.True(JsonNode.DeepEquals(expectedState, body["state"]), body["state"]!.ToJsonString()); + Assert.Equal(["department", "is_urgent"], body["questions"]!.AsObject().Select(question => question.Key)); + + // Assert: the text is the result as JSON, and the typed result, usage, and model are on the response. + JevResult result = Assert.IsType(response.GetJevResult()); + Assert.Equal("technical", Assert.IsType(result.Answers["department"]).Choice); + Assert.Equal(0.9, Assert.IsType(result.Answers["is_urgent"]).Noul); + Assert.True(JsonNode.DeepEquals(JsonNode.Parse(response.Text), JsonNode.Parse(JsonSerializer.Serialize(result, JevJsonUtilities.Options))), response.Text); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + Assert.Equal("jev-test", response.ModelId); + Assert.Equal(10, response.Usage!.InputTokenCount); + Assert.Equal(2, response.Usage.OutputTokenCount); + Assert.Equal(12, response.Usage.TotalTokenCount); + Assert.Same(result, Assert.Single(response.Messages).RawRepresentation); + } + + [Fact] + public async Task GetResponseAsync_ModelIdAndEndpoint_AreUsedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { ModelId = "jev-1.13", Endpoint = new Uri("https://proxy.contoso.com/jev") }); + + // Act + await client.GetResponseAsync(Ticket, new ChatOptions { ModelId = "jev-1.14" }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: a request's model wins over the client's. + JevTestHttpHandler.CapturedRequest request = Assert.Single(handler.Requests); + Assert.Equal(new Uri("https://proxy.contoso.com/jev"), request.Uri); + Assert.Equal("jev-1.14", (string?)JsonNode.Parse(request.Body)!["model"]); + + ChatClientMetadata metadata = Assert.IsType(client.GetService(typeof(ChatClientMetadata))); + Assert.Equal("typesafe.ai", metadata.ProviderName); + Assert.Equal(new Uri("https://proxy.contoso.com/jev"), metadata.ProviderUri); + Assert.Equal("jev-1.13", metadata.DefaultModelId); + Assert.Same(client, client.GetService(typeof(JevChatClient))); + Assert.Null(client.GetService(typeof(JevChatClient), "key")); + } + + [Fact] + public async Task GetResponseAsync_DefaultQuestions_AnswerRequestsWithoutQuestionsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = JevChatTestData.Questions() }); + + // Act + ChatResponse response = await client.GetResponseAsync(Ticket); + + // Assert + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.NotNull(response.GetJevResult()); + } + + [Fact] + public async Task GetResponseAsync_RequestQuestions_ReplaceTheDefaultsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("spam", 0.1))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = JevChatTestData.Questions() }); + var questions = new Dictionary { ["spam"] = new JevNoulQuestion { Instructions = "Is it spam?" } }; + + // Act + await client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions)); + + // Assert + Assert.Equal(["spam"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + } + + [Fact] + public async Task GetResponseAsync_NoQuestions_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket)); + Assert.Contains(nameof(JevChatOptionsExtensions.WithJevQuestions), exception.Message, StringComparison.Ordinal); + await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(new Dictionary()))); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_ReservedQuestionId_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var questions = new Dictionary { ["__af_tool__.mine"] = new JevNoulQuestion { Instructions = "Yes?" } }; + + // Act & Assert + ArgumentException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions))); + Assert.Contains("__af_tool__.mine", exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_InvalidQuestion_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var questions = new Dictionary { ["level"] = new JevScoreQuestion { Instructions = "How bad?", Criteria = ["only one"] } }; + + // Act & Assert + await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(questions))); + Assert.Empty(handler.Requests); + } + + public static TheoryData UnsupportedOptions() => new() + { + { nameof(ChatOptions.Temperature), new ChatOptions { Temperature = 0.2f } }, + { nameof(ChatOptions.TopP), new ChatOptions { TopP = 0.5f } }, + { nameof(ChatOptions.TopK), new ChatOptions { TopK = 3 } }, + { nameof(ChatOptions.MaxOutputTokens), new ChatOptions { MaxOutputTokens = 100 } }, + { nameof(ChatOptions.FrequencyPenalty), new ChatOptions { FrequencyPenalty = 0.1f } }, + { nameof(ChatOptions.PresencePenalty), new ChatOptions { PresencePenalty = 0.1f } }, + { nameof(ChatOptions.Seed), new ChatOptions { Seed = 7 } }, + { nameof(ChatOptions.Reasoning), new ChatOptions { Reasoning = new ReasoningOptions() } }, + { nameof(ChatOptions.StopSequences), new ChatOptions { StopSequences = ["stop"] } }, + { nameof(ChatOptions.ResponseFormat), new ChatOptions { ResponseFormat = ChatResponseFormat.Json } }, + { nameof(ChatOptions.ConversationId), new ChatOptions { ConversationId = "conv_1" } }, + { nameof(ChatOptions.AllowBackgroundResponses), new ChatOptions { AllowBackgroundResponses = true } }, + { nameof(ChatOptions.AllowMultipleToolCalls), new ChatOptions { AllowMultipleToolCalls = true } }, + { "HostedWebSearchTool", new ChatOptions { Tools = [new HostedWebSearchTool()] } }, + }; + + [Theory] + [MemberData(nameof(UnsupportedOptions))] + public async Task GetResponseAsync_UnsupportedOption_ThrowsBeforeSendingAsync(string option, ChatOptions options) + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + NotSupportedException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, options.WithJevQuestions(JevChatTestData.Questions()))); + Assert.Contains(option == "HostedWebSearchTool" ? "web_search" : option, exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_TextResponseFormatAndSingleToolCalls_AreAllowedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var options = new ChatOptions { ResponseFormat = ChatResponseFormat.Text, AllowMultipleToolCalls = false, AllowBackgroundResponses = false, StopSequences = [] }; + + // Act + ChatResponse response = await client.GetResponseAsync(Ticket, options.WithJevQuestions(JevChatTestData.Questions())); + + // Assert + Assert.NotNull(response.GetJevResult()); + } + + [Fact] + public async Task GetResponseAsync_NonTextContent_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + var message = new ChatMessage(ChatRole.User, [new TextContent("See the screenshot."), new DataContent(new byte[] { 1, 2, 3 }, "image/png")]); + + // Act & Assert + NotSupportedException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync([message], new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Contains(nameof(DataContent), exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_EmptyConversation_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + await Assert.ThrowsAsync(() => client.GetResponseAsync([new ChatMessage(ChatRole.User, "")], new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_ReasoningAndFunctionContent_AreSerializedAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + ChatMessage[] messages = + [ + new(ChatRole.User, Ticket), + new(ChatRole.Assistant, [new TextReasoningContent("Check the order first.") { ProtectedData = "opaque" }, new FunctionCallContent("call_1", "lookup_order", new Dictionary { ["order"] = "A-1" })]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", JsonSerializer.SerializeToElement("""{"result":"Order A-1 shipped."}"""))]), + new(ChatRole.Tool, [new FunctionResultContent("call_2", new TextContent("Refund issued."))]), + ]; + + // Act + await client.GetResponseAsync(messages, new ChatOptions().WithJevQuestions(JevChatTestData.Questions())); + + // Assert: protected reasoning stays out of the state, and rich results become text items. + JsonNode state = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["state"]!; + JsonNode expected = JsonNode.Parse(""" + [ + { "type": "text_reasoning", "text": "Check the order first." }, + { "type": "function_call", "call_id": "call_1", "name": "lookup_order", "arguments": { "order": "A-1" } } + ] + """)!; + Assert.True(JsonNode.DeepEquals(expected, state["messages"]![1]!["contents"]), state.ToJsonString()); + Assert.Equal("tool", (string?)state["messages"]![2]!["role"]); + Assert.Equal("""{"result":"Order A-1 shipped."}""", (string?)state["messages"]![2]!["contents"]![0]!["result"]); + JsonNode rich = JsonNode.Parse("""{ "type": "function_result", "call_id": "call_2", "result": "Refund issued.", "items": [{ "type": "text", "text": "Refund issued." }] }""")!; + Assert.True(JsonNode.DeepEquals(rich, state["messages"]![3]!["contents"]![0]), state.ToJsonString()); + } + + [Fact] + public async Task GetResponseAsync_RichFunctionResult_ThrowsBeforeSendingAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + ChatMessage[] messages = + [ + new(ChatRole.User, Ticket), + new(ChatRole.Tool, [new FunctionResultContent("call_1", new DataContent(new byte[] { 1 }, "image/png"))]), + ]; + + // Act & Assert + await Assert.ThrowsAsync(() => client.GetResponseAsync(messages, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_ResultMissingAnAnswer_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("is_urgent", 0.9))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Contains("department", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task GetResponseAsync_ApiError_ThrowsClientResultExceptionAsync() + { + // Arrange + using var handler = new JevTestHttpHandler().Reply(System.Net.HttpStatusCode.UnprocessableEntity, """{"detail":"questions.department.criteria: Field required"}"""); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + ClientResultException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + Assert.Equal(422, exception.Status); + Assert.Contains("Field required", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task GetStreamingResponseAsync_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = CreateClient(httpClient); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (ChatResponseUpdate update in client.GetStreamingResponseAsync(Ticket, new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))) + { + } + }); + Assert.Empty(handler.Requests); + } + + [Fact] + public void Constructor_InvalidOptions_Throw() + { + Assert.Throws(() => new JevChatClient(new ApiKeyCredential("k"), new JevChatClientOptions { ModelId = " " })); + Assert.Throws(() => new JevChatClient(new ApiKeyCredential("k"), new JevChatClientOptions { Endpoint = new Uri("/relative", UriKind.Relative) })); + Assert.Throws(() => new JevChatClientOptions { MaximumToolCallsPerTurn = -1 }); + } + + [Fact] + public void Constructor_FreezesTheOptions() + { + // Arrange + var options = new JevChatClientOptions(); + + // Act + using var client = new JevChatClient(new ApiKeyCredential("k"), options); + + // Assert: like other client options, the options are read-only once used. + Assert.Throws(() => options.ModelId = "other"); + Assert.Throws(() => options.DefaultQuestions = JevChatTestData.Questions()); + Assert.IsType(options.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(10), options.NetworkTimeout); + } + + [Fact] + public void WithJevQuestions_OtherClients_KeepThePreviousFactory() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient jev = CreateClient(httpClient); + using var other = new ScriptedChatClient(); + object otherOptions = new(); + var options = new ChatOptions { RawRepresentationFactory = _ => otherOptions }; + + // Act + options.WithJevQuestions(JevChatTestData.Questions()); + + // Assert + Assert.Same(otherOptions, options.RawRepresentationFactory!(other)); + Assert.Equal(["department", "is_urgent"], Assert.IsType(options.RawRepresentationFactory(jev)).Questions.Keys); + } + + internal static JevChatClient CreateClient(HttpClient httpClient, JevChatClientOptions? options = null) + { + options ??= new JevChatClientOptions(); + options.Transport = new HttpClientPipelineTransport(httpClient); + options.RetryPolicy = new JevRetryPolicy { RecordedWaitsForTests = [] }; + return new JevChatClient(new ApiKeyCredential("test-key"), options); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs new file mode 100644 index 00000000000..7ff09a33c35 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs @@ -0,0 +1,463 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevChatClientToolTests +{ + private const string Route = JevChatTestData.Route; + + public enum City + { + Seattle, + Paris, + } + + public enum Unit + { + Celsius, + Fahrenheit, + } + + public enum Feature + { + Wind, + Humidity, + Uv, + } + + [Fact] + public async Task GetResponseAsync_JevSelectsATool_ReturnsTheFunctionCallAsync() + { + // Arrange: city and detailed are required; unit is optional and nullable. + AIFunction weather = AIFunctionFactory.Create( + ([Description("The city")] City city, [Description("Include the hourly forecast")] bool detailed, Unit? unit = null) => "sunny", + "get_weather", + "Gets the weather for a supported city."); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.value", 0.9), + JevChatTestData.Noul("__af_tool__.t0.a2.present", 0.2), + JevChatTestData.Choice("__af_tool__.t0.a2.value", "v1", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the call has the chosen enum value and Boolean, and leaves out the optional argument. + FunctionCallContent call = Assert.IsType(Assert.Single(Assert.Single(response.Messages).Contents)); + Assert.Equal("get_weather", call.Name); + Assert.StartsWith("typesafe-", call.CallId, StringComparison.Ordinal); + Assert.Equal(["city", "detailed"], call.Arguments!.Keys.Order()); + Assert.Equal("Paris", Assert.IsType(call.Arguments["city"]).GetString()); + Assert.True(Assert.IsType(call.Arguments["detailed"]).GetBoolean()); + Assert.Equal(ChatFinishReason.ToolCalls, response.FinishReason); + Assert.Null(response.GetJevResult()); + + // Assert: the request adds the internal routing questions after the caller's questions. + JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); + Assert.Equal( + ["department", "is_urgent", Route, "__af_tool__.t0.a0.value", "__af_tool__.t0.a1.value", "__af_tool__.t0.a2.present", "__af_tool__.t0.a2.value"], + questions.Select(question => question.Key)); + Assert.Equal("Tool 'get_weather': Gets the weather for a supported city.", (string?)questions[Route]!["criteria"]!["t0"]); + Assert.NotNull(questions[Route]!["criteria"]!["none"]); + Assert.Equal("\"Paris\"", (string?)questions["__af_tool__.t0.a0.value"]!["criteria"]!["v1"]); + Assert.Equal("noul", (string?)questions["__af_tool__.t0.a1.value"]!["type"]); + Assert.Contains("did the user explicitly specify the unit argument", (string?)questions["__af_tool__.t0.a2.present"]!["instructions"], StringComparison.Ordinal); + } + + [Fact] + public async Task GetResponseAsync_OptionalArgumentPresent_IsPassedAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city, Unit? unit = null, Feature[]? features = null) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.8), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a2.present", 0.7), + JevChatTestData.Noul("__af_tool__.t0.a2.m0", 0.9), + JevChatTestData.Noul("__af_tool__.t0.a2.m1", 0.1), + JevChatTestData.Noul("__af_tool__.t0.a2.m2", 0.6))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: a nullable enum is a Choice without null, and an enum array is one Noul per member. + FunctionCallContent call = response.Messages.SelectMany(message => message.Contents).OfType().Single(); + Assert.Equal("Seattle", Assert.IsType(call.Arguments!["city"]).GetString()); + Assert.Equal("Fahrenheit", Assert.IsType(call.Arguments["unit"]).GetString()); + Assert.Equal(["Wind", "Uv"], Assert.IsType(call.Arguments["features"]).EnumerateArray().Select(item => item.GetString())); + + JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); + Assert.Equal(["v0", "v1"], questions["__af_tool__.t0.a1.value"]!["criteria"]!.AsObject().Select(criterion => criterion.Key)); + } + + [Fact] + public async Task GetResponseAsync_JevChoosesNoTool_ReturnsOnlyTheCallersAnswersAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("none", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert + JevResult result = Assert.IsType(response.GetJevResult()); + Assert.Equal(["department", "is_urgent"], result.Answers.Keys); + Assert.DoesNotContain("__af_tool__", response.Text, StringComparison.Ordinal); + Assert.Equal(ChatFinishReason.Stop, response.FinishReason); + } + + [Fact] + public async Task GetResponseAsync_AfterTheToolCallLimit_AnswersWithTheToolResultsAsync() + { + // Arrange: the turn already has one call and its result, which is the default limit. + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.Score("severity", 1.5, 3))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + Dictionary questions = JevChatTestData.Questions(); + questions["severity"] = new JevScoreQuestion { Instructions = "How severe?", Criteria = ["Low", "Medium", "High"] }; + ChatMessage[] messages = + [ + new(ChatRole.User, "Earlier question"), + new(ChatRole.Assistant, [new FunctionCallContent("call_0", "get_weather", new Dictionary { ["city"] = "Paris" })]), + new(ChatRole.Tool, [new FunctionResultContent("call_0", "Paris is cloudy.")]), + new(ChatRole.Assistant, "Paris is cloudy.\ndepartment: technical"), + new(ChatRole.User, "Weather in Seattle?"), + new(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = "Seattle" })]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", JsonSerializer.SerializeToElement("Seattle is sunny."))]), + ]; + + // Act + ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Tools = [weather] }.WithJevQuestions(questions)); + + // Assert: no tool questions, and the text has this turn's tool result and the Choice and Score decisions. + Assert.Equal(["department", "is_urgent", "severity"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Seattle is sunny.\ndepartment: technical\nseverity: 1.5", response.Text); + Assert.NotNull(response.GetJevResult()); + } + + [Fact] + public async Task GetResponseAsync_HigherLimit_OffersToolsAgainWithHintsAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather", "Gets the weather."); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient, new JevChatClientOptions { MaximumToolCallsPerTurn = 2 }); + ChatMessage[] messages = + [ + new(ChatRole.User, "Weather in Seattle and Paris?"), + new(ChatRole.Assistant, [new FunctionCallContent("call_1", "get_weather", new Dictionary { ["city"] = JsonSerializer.SerializeToElement("Seattle") })]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", "Seattle is sunny.")]), + ]; + + // Act + ChatResponse response = await client.GetResponseAsync(messages, new ChatOptions { Tools = [weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the earlier call becomes a hint in the route and argument questions. + JsonObject questions = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject(); + Assert.EndsWith("""Do not repeat argument sets already called this turn: [{"city": "Seattle"}].""", (string?)questions[Route]!["criteria"]!["t0"], StringComparison.Ordinal); + Assert.Contains("""Values already used for this argument in earlier calls this turn: ["Seattle"].""", (string?)questions["__af_tool__.t0.a0.value"]!["instructions"], StringComparison.Ordinal); + Assert.Equal("Paris", Assert.IsType(response.Messages.SelectMany(message => message.Contents).OfType().Single().Arguments!["city"]).GetString()); + } + + [Fact] + public async Task GetResponseAsync_UnsupportedTool_IsLeftOutWithAWarningAsync() + { + // Arrange: free-form text cannot be chosen from a closed set. + AIFunction search = AIFunctionFactory.Create((string query) => "found", "search"); + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("none", "t1", "none"), + JevChatTestData.Choice("__af_tool__.t1.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + var loggerFactory = new ListLoggerFactory(); + var options = new JevChatClientOptions(); + options.ClientLoggingOptions = new() { LoggerFactory = loggerFactory, EnableLogging = false }; + using JevChatClient client = JevChatClientTests.CreateClient(httpClient, options); + + // Act + await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [search, weather] }.WithJevQuestions(JevChatTestData.Questions())); + + // Assert: the tools keep their position, so the supported tool is still t1. + JsonObject criteria = JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]!["criteria"]!.AsObject(); + Assert.Equal(["t1", "none"], criteria.Select(criterion => criterion.Key)); + (LogLevel level, string message) = Assert.Single(loggerFactory.Entries, entry => entry.Message.Contains("'search'", StringComparison.Ordinal)); + Assert.Equal(LogLevel.Warning, level); + Assert.Contains("required argument 'query'", message, StringComparison.Ordinal); + } + + [Fact] + public async Task GetResponseAsync_RequiredModeWithoutSupportedTools_ThrowsAsync() + { + // Arrange + AIFunction search = AIFunctionFactory.Create((string query) => "found", "search"); + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + ChatOptions options = new ChatOptions { Tools = [search], ToolMode = ChatToolMode.RequireAny }.WithJevQuestions(JevChatTestData.Questions()); + + // Act & Assert + InvalidOperationException exception = await Assert.ThrowsAsync(() => client.GetResponseAsync(JevTestData.TicketText, options)); + Assert.Contains("search: required argument 'query'", exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } + + [Fact] + public async Task GetResponseAsync_RequireSpecificTool_SkipsTheRouteQuestionAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + AIFunction forecast = AIFunctionFactory.Create((City city) => "rain", "get_forecast"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + ChatOptions options = new ChatOptions { Tools = [weather, forecast], ToolMode = ChatToolMode.RequireSpecific("get_forecast") }.WithJevQuestions(JevChatTestData.Questions()); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, options); + + // Assert + Assert.Equal("get_forecast", response.Messages.SelectMany(message => message.Contents).OfType().Single().Name); + Assert.Null(JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]); + } + + [Fact] + public async Task GetResponseAsync_RequiredModeWithSeveralTools_OmitsTheNoneRouteAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + AIFunction forecast = AIFunctionFactory.Create((City city) => "rain", "get_forecast"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t1", "t0", "t1"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"), + JevChatTestData.Choice("__af_tool__.t1.a0.value", "v1", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + ChatOptions options = new ChatOptions { Tools = [weather, forecast], ToolMode = ChatToolMode.RequireAny }.WithJevQuestions(JevChatTestData.Questions()); + + // Act + ChatResponse response = await client.GetResponseAsync(JevTestData.TicketText, options); + + // Assert + Assert.Equal(["t0", "t1"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]![Route]!["criteria"]!.AsObject().Select(criterion => criterion.Key)); + Assert.Equal("get_forecast", response.Messages.SelectMany(message => message.Contents).OfType().Single().Name); + } + + [Fact] + public async Task GetResponseAsync_RequiredToolMissingOrNoneMode_BehaveAsExpectedAsync() + { + // Arrange + AIFunction weather = AIFunctionFactory.Create((City city) => "sunny", "get_weather"); + using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + + // Act & Assert: a missing required tool fails, and the none mode sends only the caller's questions. + await Assert.ThrowsAsync(() => client.GetResponseAsync( + JevTestData.TicketText, + new ChatOptions { Tools = [weather], ToolMode = ChatToolMode.RequireSpecific("other") }.WithJevQuestions(JevChatTestData.Questions()))); + await client.GetResponseAsync(JevTestData.TicketText, new ChatOptions { Tools = [weather], ToolMode = ChatToolMode.None }.WithJevQuestions(JevChatTestData.Questions())); + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(Assert.Single(handler.Requests).Body)!["questions"]!.AsObject().Select(question => question.Key)); + } + + [Fact] + public async Task RunAsync_AgentExecutesTheToolOnce_ThenAnswersAsync() + { + // Arrange + var executions = new List<(City City, bool Detailed)>(); + AIFunction weather = AIFunctionFactory.Create( + (City city, bool detailed) => + { + executions.Add((city, detailed)); + return $"{city} is sunny{(detailed ? " with an hourly forecast" : string.Empty)}."; + }, + "get_weather", + "Gets the weather for a supported city."); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.value", 0.9))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(instructions: "Answer weather requests.", tools: [weather]); + + // Act + AgentResponse response = await agent.RunAsync( + "Give me a detailed weather report for Paris.", + options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(JevChatTestData.Questions()))); + + // Assert: the tool ran once, and the second request answered the caller's questions with its result. + Assert.Equal([(City.Paris, true)], executions); + Assert.Equal(2, handler.Requests.Count); + JsonNode second = JsonNode.Parse(handler.Requests[1].Body)!; + Assert.Equal(["department", "is_urgent"], second["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Answer weather requests.", (string?)second["state"]!["instructions"]); + JsonNode toolResult = second["state"]!["messages"]!.AsArray().SelectMany(message => message!["contents"]!.AsArray()).Single(content => (string?)content!["type"] == "function_result")!; + Assert.Equal("Paris is sunny with an hourly forecast.", (string?)toolResult["result"]); + + Assert.Equal("Paris is sunny with an hourly forecast.\ndepartment: technical", response.Text); + JevResult result = Assert.IsType(response.GetJevResult()); + Assert.Equal(["department", "is_urgent"], result.Answers.Keys); + Assert.Equal(24, response.Usage!.TotalTokenCount); + } + + [Fact] + public async Task RunAsync_AgentLevelQuestions_AreUsedUnlessTheRunPassesItsOwnAsync() + { + // Arrange + var runQuestions = new Dictionary { ["spam"] = new JevNoulQuestion { Instructions = "Is it spam?" } }; + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("spam", 0.05))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions().WithJevQuestions(JevChatTestData.Questions()) }); + + // Act + AgentResponse agentLevel = await agent.RunAsync(JevTestData.TicketText); + AgentResponse runLevel = await agent.RunAsync(JevTestData.TicketText, options: new ChatClientAgentRunOptions(new ChatOptions().WithJevQuestions(runQuestions))); + + // Assert + Assert.Equal(["department", "is_urgent"], agentLevel.GetJevResult()!.Answers.Keys); + Assert.Equal(["spam"], runLevel.GetJevResult()!.Answers.Keys); + } + + [Fact] + public async Task RunAsync_ApprovalRequiredTool_RunsOnlyAfterApprovalAsync() + { + // Arrange + int executions = 0; + AIFunction refund = new ApprovalRequiredAIFunction(AIFunctionFactory.Create( + (City city) => + { + executions++; + return "Refund issued."; + }, + "issue_refund")); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers("billing"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers("billing"), + JevChatTestData.RouteTo("none", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refund] }.WithJevQuestions(JevChatTestData.Questions()) }); + AgentSession session = await agent.CreateSessionAsync(); + + // Act + AgentResponse pending = await agent.RunAsync("Refund my Seattle order.", session); + ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); + int executionsBeforeApproval = executions; + AgentResponse approved = await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true)]), session); + AgentResponse later = await agent.RunAsync("Thanks. Anything else?", session); + + // Assert + Assert.Equal(0, executionsBeforeApproval); + Assert.Equal(1, executions); + Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); + + // Assert: the next turn still works. The handled approval stays in the history, but Jev does not read it. + Assert.Equal(3, handler.Requests.Count); + Assert.DoesNotContain("approval", handler.Requests[2].Body, StringComparison.OrdinalIgnoreCase); + Assert.NotNull(later.GetJevResult()); + } + + [Fact] + public async Task RunAsync_ApprovalSentWithText_ContinuesTheTurnOfTheApprovedCallAsync() + { + // Arrange + int executions = 0; + AIFunction refund = new ApprovalRequiredAIFunction(AIFunctionFactory.Create( + (City city) => + { + executions++; + return "Refund issued."; + }, + "issue_refund")); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result( + JevChatTestData.UserAnswers(), + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v0", "v0", "v1"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers("billing"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refund] }.WithJevQuestions(JevChatTestData.Questions()) }); + AgentSession session = await agent.CreateSessionAsync(); + + // Act: FunctionInvokingChatClient moves the text after the approved call and its result. + AgentResponse pending = await agent.RunAsync("Refund my Seattle order.", session); + ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); + AgentResponse approved = await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")]), session); + + // Assert: the approved call counts toward the limit, so the tool is not offered again. + Assert.Equal(1, executions); + Assert.Equal(2, handler.Requests.Count); + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); + } + + [Fact] + public async Task RunStreamingAsync_ThrowsAsync() + { + // Arrange + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions().WithJevQuestions(JevChatTestData.Questions()) }); + + // Act & Assert + await Assert.ThrowsAsync(async () => + { + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync(JevTestData.TicketText)) + { + } + }); + Assert.Empty(handler.Requests); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs new file mode 100644 index 00000000000..a20f99a0ff1 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatTestData.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Globalization; +using System.Linq; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// Builds TypeSafe API responses and questions for the chat client tests. +/// +internal static class JevChatTestData +{ + public const string Route = "__af_tool__.route"; + + /// Two questions about a support ticket: a Choice and a Noul. + public static Dictionary Questions() => new() + { + ["department"] = new JevChoiceQuestion + { + Instructions = "Which team should handle the request?", + Criteria = new Dictionary { ["billing"] = "Payments or refunds", ["technical"] = "Bugs or integrations" }, + }, + ["is_urgent"] = new JevNoulQuestion { Instructions = "Does the request need urgent attention?" }, + }; + + /// The answers to . + public static string[] UserAnswers(string department = "technical") => + [Choice("department", department, "billing", "technical"), Noul("is_urgent", 0.9)]; + + public static string Result(params string[] answers) => + "{\"model\":\"jev-test\",\"answers\":{" + string.Join(",", answers) + "},\"usage\":{\"input_tokens\":10,\"output_tokens\":2}}"; + + public static string Result(IEnumerable answers, params string[] more) => Result([.. answers, .. more]); + + public static string Choice(string id, string choice, params string[] labels) + { + string probabilities = string.Join(",", labels.Select(label => $"\"{label}\":{(label == choice ? "1.0" : "0.0")}")); + return $"\"{id}\":{{\"type\":\"choice\",\"choice\":\"{choice}\",\"confidence\":0.9,\"probabilities\":{{{probabilities}}}}}"; + } + + public static string Noul(string id, double noul) => + $"\"{id}\":{{\"type\":\"noul\",\"noul\":{noul.ToString(CultureInfo.InvariantCulture)}}}"; + + public static string Score(string id, double score, int levels) + { + string legend = string.Join(",", Enumerable.Range(0, levels).Select(level => $"\"{level}\":\"Level {level}\"")); + string probabilities = string.Join(",", Enumerable.Range(0, levels).Select(level => $"\"{level}\":{(level == (int)score ? "1.0" : "0.0")}")); + return $"\"{id}\":{{\"type\":\"score\",\"score\":{score.ToString(CultureInfo.InvariantCulture)},\"confidence\":0.8,\"legend\":{{{legend}}},\"probabilities\":{{{probabilities}}}}}"; + } + + /// A route answer that selects one of the route labels. + public static string RouteTo(string label, params string[] labels) => Choice(Route, label, labels); +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs new file mode 100644 index 00000000000..532d670a315 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevEntryTests.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevEntryTests +{ + [Fact] + public void Conversions_FromTextAndJson_KeepTheValue() + { + // Arrange + using JsonDocument document = JsonDocument.Parse("""{ "question": "Is `order` paid?" }"""); + + // Act + JevEntry text = "Is this urgent?"; + JevEntry nullText = (string?)null; + var json = (JevEntry)document.RootElement; + + // Assert + Assert.Equal(JsonValueKind.String, text.Kind); + Assert.Equal("Is this urgent?", text.Text); + Assert.Equal("Is this urgent?", text.ToString()); + Assert.True(nullText.IsNull); + Assert.Equal(JevEntry.Null, nullText); + Assert.Equal(JsonValueKind.Object, json.Kind); + Assert.Null(json.Text); + Assert.Equal("""{ "question": "Is `order` paid?" }""", json.ToString()); + } + + [Fact] + public void Default_IsTheNullEntry() + { + // Act + JevEntry entry = default; + + // Assert + Assert.True(entry.IsNull); + Assert.Equal(JsonValueKind.Null, entry.Kind); + Assert.Null(entry.Json); + Assert.Equal(string.Empty, entry.ToString()); + } + + [Fact] + public void Json_OutlivesTheDocumentItCameFrom() + { + // Arrange + JevEntry entry; + using (JsonDocument document = JsonDocument.Parse("""["a", "b"]""")) + { + entry = JevEntry.FromJsonElement(document.RootElement); + } + + // Act & Assert + Assert.Equal(2, entry.Json!.Value.GetArrayLength()); + } + + [Theory] + [InlineData("42")] + [InlineData("true")] + [InlineData("false")] + public void FromJsonElement_NumberOrBoolean_Throws(string json) + { + // Arrange + using JsonDocument document = JsonDocument.Parse(json); + JsonElement element = document.RootElement; + + // Act & Assert + Assert.Throws(() => (JevEntry)element); + Assert.Throws(() => JevEntry.FromJsonElement(element)); + } + + [Fact] + public void Equality_ComparesJsonValuesRegardlessOfPropertyOrder() + { + // Arrange + using JsonDocument first = JsonDocument.Parse("""{ "a": 1, "b": [true] }"""); + using JsonDocument second = JsonDocument.Parse("""{ "b": [true], "a": 1 }"""); + using JsonDocument other = JsonDocument.Parse("""{ "a": 2 }"""); + var left = (JevEntry)first.RootElement; + var right = (JevEntry)second.RootElement; + + // Act & Assert + Assert.True(left == right); + Assert.Equal(left.GetHashCode(), right.GetHashCode()); + Assert.True(left != (JevEntry)other.RootElement); + Assert.Equal((JevEntry)"same", (JevEntry)"same"); + Assert.NotEqual((JevEntry)"same", JevEntry.Null); + } + + [Theory] + [InlineData("\"text\"", JsonValueKind.String)] + [InlineData("{\"k\":\"v\"}", JsonValueKind.Object)] + [InlineData("[\"x\"]", JsonValueKind.Array)] + [InlineData("null", JsonValueKind.Null)] + public void Serialization_RoundTripsTheJsonValue(string json, JsonValueKind kind) + { + // Act + var entry = (JevEntry)JsonSerializer.Deserialize(json, JevJsonUtilities.Options.GetTypeInfo(typeof(JevEntry)))!; + string written = JsonSerializer.Serialize(entry, JevJsonUtilities.Options.GetTypeInfo(typeof(JevEntry))); + + // Assert + Assert.Equal(kind, entry.Kind); + Assert.Equal(json, written); + } + + [Theory] + [InlineData("42")] + [InlineData("true")] + public void Deserialization_NumberOrBoolean_Throws(string json) + { + // Act & Assert + Assert.Throws(() => JsonSerializer.Deserialize(json, JevJsonUtilities.Options.GetTypeInfo(typeof(JevEntry)))); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpClientTests.cs similarity index 76% rename from dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs rename to dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpClientTests.cs index 7ec9e0ab56c..a1e0d699931 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpEvaluatorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevHttpClientTests.cs @@ -15,7 +15,7 @@ namespace Microsoft.Agents.AI.TypeSafe.UnitTests; -public sealed class JevHttpEvaluatorTests +public sealed class JevHttpClientTests { private const string ApiKey = "test-key-9f2c"; @@ -79,7 +79,7 @@ public async Task InvokeAsync_CustomPolicy_RunsInThePipelineAsync() } [Fact] - public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesAsync() + public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesWithBackoffAsync() { // Arrange using var handler = new JevTestHttpHandler() @@ -92,12 +92,33 @@ public async Task InvokeAsync_RateLimitedOrOverloaded_RetriesAsync() // Act object? result = await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); - // Assert + // Assert: the SDK backoff is 500 ms doubled per attempt, minus up to 25% jitter. Assert.Equal(3, handler.Requests.Count); Assert.Equal(2, waits.Count); + Assert.InRange(waits[0], TimeSpan.FromMilliseconds(375), TimeSpan.FromMilliseconds(500)); + Assert.InRange(waits[1], TimeSpan.FromMilliseconds(750), TimeSpan.FromMilliseconds(1000)); Assert.Equal("technical", Assert.IsType(result).GetProperty("answers").GetProperty("department").GetProperty("choice").GetString()); } + [Theory] + [InlineData(503)] + [InlineData(520)] + [InlineData(408)] + public async Task InvokeAsync_TransientStatus_IsRetriedLikeTheSdkAsync(int status) + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply((HttpStatusCode)status, "{}") + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + + // Act + await CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(2, handler.Requests.Count); + } + [Fact] public async Task InvokeAsync_StillRateLimitedAfterRetries_FailsWithTheStatusAsync() { @@ -109,9 +130,9 @@ public async Task InvokeAsync_StillRateLimitedAfterRetries_FailsWithTheStatusAsy ClientResultException exception = await Assert.ThrowsAsync(() => CreateBuilder(httpClient).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); - // Assert + // Assert: the first attempt plus the SDK's two retries. Assert.Equal(429, exception.Status); - Assert.Equal(4, handler.Requests.Count); + Assert.Equal(3, handler.Requests.Count); } [Fact] @@ -132,6 +153,23 @@ public async Task InvokeAsync_RetryAfterSeconds_IsWaitedAsync() Assert.Equal(TimeSpan.FromSeconds(5), Assert.Single(waits)); } + [Fact] + public async Task InvokeAsync_RetryAfterMilliseconds_IsPreferredAsync() + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, "{}", new RetryConditionHeaderValue(TimeSpan.FromSeconds(30)), retryAfterMs: "1500") + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert + Assert.Equal(TimeSpan.FromMilliseconds(1500), Assert.Single(waits)); + } + [Fact] public async Task InvokeAsync_RetryAfterDate_IsWaitedAsync() { @@ -150,27 +188,54 @@ public async Task InvokeAsync_RetryAfterDate_IsWaitedAsync() Assert.InRange(Assert.Single(waits), TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(21)); } + [Theory] + [InlineData("Retry-After", "1e300")] + [InlineData("Retry-After", "99999999999999")] + [InlineData("Retry-After", "NaN")] + [InlineData("retry-after-ms", "1e30")] + [InlineData("retry-after-ms", "NaN")] + public async Task InvokeAsync_MalformedOrHugeServerDelay_FallsBackToBackoffAsync(string header, string value) + { + // Arrange + using var handler = new JevTestHttpHandler() + .Reply( + HttpStatusCode.TooManyRequests, + "{}", + retryAfterMs: header == "retry-after-ms" ? value : null, + retryAfterRaw: header == "Retry-After" ? value : null) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); + using var httpClient = new HttpClient(handler); + List waits = []; + + // Act + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); + + // Assert: like the SDK, an unusable server delay is ignored instead of failing the call. + Assert.Equal(2, handler.Requests.Count); + Assert.InRange(Assert.Single(waits), TimeSpan.FromMilliseconds(375), TimeSpan.FromMilliseconds(500)); + } + [Theory] [InlineData(false)] [InlineData(true)] - public async Task InvokeAsync_RetryAfterBeyondTheLimit_FailsAtOnceAsync(bool asDate) + public async Task InvokeAsync_RetryAfterBeyondTheLimit_FallsBackToBackoffAsync(bool asDate) { // Arrange RetryConditionHeaderValue retryAfter = asDate ? new RetryConditionHeaderValue(DateTimeOffset.UtcNow.AddHours(1)) : new RetryConditionHeaderValue(TimeSpan.FromHours(1)); - using var handler = new JevTestHttpHandler().Reply(HttpStatusCode.TooManyRequests, """{"detail":"Slow down"}""", retryAfter); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.TooManyRequests, """{"detail":"Slow down"}""", retryAfter) + .Reply(HttpStatusCode.OK, JevTestData.ResponseJson); using var httpClient = new HttpClient(handler); List waits = []; // Act - ClientResultException exception = await Assert.ThrowsAsync(() => - CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)).AsTask()); + await CreateBuilder(httpClient, waits: waits).Build().InvokeAsync(JevTestData.Arguments(JevTestData.RequestJson)); - // Assert - Assert.Equal(429, exception.Status); - Assert.Single(handler.Requests); - Assert.Empty(waits); + // Assert: waiting an hour would block the agent run, so the SDK's backoff is used instead. + Assert.Equal(2, handler.Requests.Count); + Assert.InRange(Assert.Single(waits), TimeSpan.FromMilliseconds(375), TimeSpan.FromMilliseconds(500)); } [Fact] diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs new file mode 100644 index 00000000000..88d99d27b80 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevLoopJudgeTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; + +#pragma warning disable MAAI001 // LoopAgent and its evaluators are experimental. + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// Tests Jev as the judge of a , as the loop judge sample does. +/// +public sealed class JevLoopJudgeTests +{ + private static readonly Dictionary s_criteria = new() + { + ["blue_sky"] = new JevNoulQuestion { Instructions = "Does the assistant's latest response explain why the sky is blue?" }, + ["red_sunsets"] = new JevNoulQuestion { Instructions = "Does the assistant's latest response explain why sunsets are red?" }, + }; + + [Fact] + public async Task RunAsync_DelegateEvaluatorWithJev_LoopsUntilEveryCriterionIsMetAsync() + { + // Arrange: the first answer misses the sunsets, the second covers both. + using var answerer = new ScriptedChatClient( + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light.")), + _ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light, and sunsets are red because light crosses more air."))); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("blue_sky", 0.98), JevChatTestData.Noul("red_sunsets", 0.03))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.Noul("blue_sky", 0.97), JevChatTestData.Noul("red_sunsets", 0.95))); + using var httpClient = new HttpClient(handler); + using JevChatClient judge = JevChatClientTests.CreateClient(httpClient); + + var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => + { + List conversation = [.. context.InitialMessages, new ChatMessage(ChatRole.Assistant, context.LastResponse.Text)]; + ChatResponse verdict = await judge.GetResponseAsync(conversation, new ChatOptions().WithJevQuestions(s_criteria), cancellationToken); + List missing = [.. verdict.GetJevResult()!.Answers.Where(answer => ((JevNoulResponse)answer.Value).Noul <= 0.5).Select(answer => answer.Key)]; + return missing.Count == 0 ? LoopEvaluation.Stop() : LoopEvaluation.Continue($"Missing: {string.Join(", ", missing)}."); + }); + var loop = new LoopAgent(answerer.AsAIAgent(), evaluator, new LoopAgentOptions { MaxIterations = 3, NonStreamingReturnsLastResponseOnly = true }); + + // Act + AgentResponse response = await loop.RunAsync("Explain why the sky is blue and why sunsets are red."); + + // Assert: two runs, and the second run received the missing criterion as feedback. + Assert.Equal(2, answerer.Calls.Count); + Assert.Contains(answerer.Calls[1], message => message.Role == ChatRole.User && message.Text == "Missing: red_sunsets."); + Assert.Contains("sunsets are red", response.Text, StringComparison.Ordinal); + + // Assert: Jev read the original request and the latest answer, and only the criteria questions. + Assert.Equal(2, handler.Requests.Count); + JsonNode first = JsonNode.Parse(handler.Requests[0].Body)!; + Assert.Equal(["blue_sky", "red_sunsets"], first["questions"]!.AsObject().Select(question => question.Key)); + JsonArray messages = first["state"]!["messages"]!.AsArray(); + Assert.Equal(["user", "assistant"], messages.Select(message => (string?)message!["role"])); + Assert.Equal("Air scatters blue light.", (string?)messages[1]!["contents"]![0]!["text"]); + } + + [Fact] + public async Task EvaluateAsync_AIJudgeLoopEvaluatorWithJev_IsNotSupportedAsync() + { + // Arrange: AIJudgeLoopEvaluator asks for a JudgeVerdict as JSON structured output, which Jev cannot write. + using var handler = new JevTestHttpHandler(); + using var httpClient = new HttpClient(handler); + using JevChatClient judge = JevChatClientTests.CreateClient(httpClient, new JevChatClientOptions { DefaultQuestions = s_criteria }); + using var answerer = new ScriptedChatClient(_ => new ChatResponse(new ChatMessage(ChatRole.Assistant, "Air scatters blue light."))); + var loop = new LoopAgent(answerer.AsAIAgent(), new AIJudgeLoopEvaluator(judge)); + + // Act & Assert: the request fails before anything is sent, which is why the sample uses a delegate evaluator. + NotSupportedException exception = await Assert.ThrowsAsync(() => loop.RunAsync("Why is the sky blue?")); + Assert.Contains(nameof(ChatOptions.ResponseFormat), exception.Message, StringComparison.Ordinal); + Assert.Empty(handler.Requests); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs index 3afde9ae642..e0c045c97e2 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestData.cs @@ -79,26 +79,26 @@ public static AIFunctionArguments Arguments(string json) return arguments; } - /// A correct response to , built from the contract types. - public static JevResponse CreateResponse() => new() + /// A correct result for , built from the contract types. + public static JevResult CreateResult() => new() { Model = "custom-model", - Answers = new Dictionary + Answers = new Dictionary { - ["department"] = new JevChoiceAnswer + ["department"] = new JevChoiceResponse { Choice = "technical", Confidence = 0.78, Probabilities = new Dictionary { ["technical"] = 0.85, ["billing"] = 0.15, ["sales"] = 0 }, }, - ["frustration"] = new JevScoreAnswer + ["frustration"] = new JevScoreResponse { Score = 1, Confidence = 1, - Legend = new Dictionary { ["0"] = "Calm", ["1"] = "Frustrated", ["2"] = "Very angry" }, + Legend = new Dictionary { ["0"] = "Calm", ["1"] = "Frustrated", ["2"] = "Very angry" }, Probabilities = new Dictionary { ["0"] = 0, ["1"] = 1, ["2"] = 0 }, }, - ["is_urgent"] = new JevNoulAnswer { Noul = 0.97 }, + ["is_urgent"] = new JevNoulResponse { Noul = 0.97 }, }, Usage = new JevUsage { InputTokens = 10, OutputTokens = 5 }, }; diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs index bc53c152cd1..178285578f0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevTestHttpHandler.cs @@ -16,14 +16,14 @@ namespace Microsoft.Agents.AI.TypeSafe.UnitTests; /// internal sealed class JevTestHttpHandler : HttpMessageHandler { - private readonly Queue<(HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter)> _responses = new(); - private (HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter) _last = (HttpStatusCode.OK, JevTestData.ResponseJson, null); + private readonly Queue<(HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter, string? RetryAfterMs, string? RetryAfterRaw)> _responses = new(); + private (HttpStatusCode Status, string Body, RetryConditionHeaderValue? RetryAfter, string? RetryAfterMs, string? RetryAfterRaw) _last = (HttpStatusCode.OK, JevTestData.ResponseJson, null, null, null); public List Requests { get; } = []; - public JevTestHttpHandler Reply(HttpStatusCode status, string body, RetryConditionHeaderValue? retryAfter = null) + public JevTestHttpHandler Reply(HttpStatusCode status, string body, RetryConditionHeaderValue? retryAfter = null, string? retryAfterMs = null, string? retryAfterRaw = null) { - this._responses.Enqueue((status, body, retryAfter)); + this._responses.Enqueue((status, body, retryAfter, retryAfterMs, retryAfterRaw)); return this; } @@ -48,6 +48,17 @@ protected override async Task SendAsync(HttpRequestMessage Content = new StringContent(this._last.Body, Encoding.UTF8, "application/json"), }; response.Headers.RetryAfter = this._last.RetryAfter; + if (this._last.RetryAfterMs is { } retryAfterMs) + { + response.Headers.Add("retry-after-ms", retryAfterMs); + } + + // A raw value lets tests send Retry-After headers that HttpClient would refuse to parse. + if (this._last.RetryAfterRaw is { } raw) + { + response.Headers.TryAddWithoutValidation("Retry-After", raw); + } + return response; } diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs new file mode 100644 index 00000000000..e78d148c492 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +public sealed class JevToolCallCompilerTests +{ + private static readonly Dictionary>> s_noCalls = []; + + [Fact] + public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() + { + // Arrange: a $defs reference, and an optional anyOf of an enum and null, as Pydantic writes them. + AIFunctionDeclaration tool = Declaration("set_mode", """ + { + "type": "object", + "properties": { + "mode": { "$ref": "#/$defs/Mode", "description": "The mode" }, + "level": { "anyOf": [{ "enum": ["low", "high"] }, { "type": "null" }], "default": null }, + "kind": { "const": "fixed" } + }, + "required": ["mode", "kind"], + "$defs": { "Mode": { "type": "string", "enum": ["fast", "safe"] } } + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v0", "v0", "v1"))); + + // Assert + Assert.Contains("The mode", plan.Questions["__af_tool__.t0.a0.value"].Instructions!.Value.Text, StringComparison.Ordinal); + Assert.Equal(["route", "t0.a0.value", "t0.a1.present", "t0.a1.value"], plan.Questions.Keys.Select(key => key["__af_tool__.".Length..])); + Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["mode"]!).GetString()); + Assert.Equal("low", ((JsonElement)call.Value.Arguments["level"]!).GetString()); + Assert.Equal("fixed", ((JsonElement)call.Value.Arguments["kind"]!).GetString()); + } + + [Theory] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean" } }, "minProperties": 1 }""", "unsupported root-level schema constraints: minProperties")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "enum": ["x", "y"], "minLength": 1 } }, "required": ["a"] }""", "required argument 'a': unsupported argument schema constraints: minLength")] + [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A", "enum": ["x"] } }, "$defs": { "A": { "type": "boolean" } } }""", "optional argument 'a': $ref sibling constraints are not supported: enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "anyOf": [{ "type": "boolean" }, { "type": "string", "enum": ["x"] }] } } }""", "only a single schema combined with null is supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": ["boolean", "null"] } }, "required": ["a"] }""", "required nullable arguments are not supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "string", "enum": ["x"], "maxLength": 3 } } } }""", "unsupported array item schema constraints: maxLength")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "integer", "enum": [1, 2.5] } } } }""", "array enum member 2.5 does not match declared item type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": ["string", "null"], "enum": ["x", null] } } } }""", "nullable array members are not supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer" } } }""", "only const, enum, boolean, and arrays of enum values are supported")] + [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] + [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] + public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) + { + // Arrange + var logger = new ListLoggerFactory(); + + // Act + JevToolCallPlan? plan = Compile([Declaration("tool", schema)], logger: logger.CreateLogger("test")); + + // Assert + Assert.Null(plan); + Assert.Contains(reason, Assert.Single(logger.Entries).Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_SingleValueEnumAndNoArguments_NeedNoArgumentQuestions() + { + // Arrange + AIFunctionDeclaration refresh = Declaration("refresh", """{ "type": "object", "properties": {} }"""); + AIFunctionDeclaration single = Declaration("single", """{ "type": "object", "properties": { "scope": { "enum": ["all"] } }, "required": ["scope"] }"""); + + // Act + JevToolCallPlan plan = Compile([refresh, single])!; + JevToolCall? call = plan.Decode(Result(JevChatTestData.RouteTo("t1", "t0", "t1", "none"))); + + // Assert + Assert.Equal([JevChatTestData.Route], plan.Questions.Keys); + Assert.Equal("single", call!.Value.Function.Name); + Assert.Equal("all", ((JsonElement)call.Value.Arguments["scope"]!).GetString()); + } + + [Fact] + public void Compile_TooManyTools_Throws() + { + // Arrange + List tools = [.. Enumerable.Range(0, JevToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; + + // Act & Assert + Assert.Throws(() => Compile(tools)); + } + + [Fact] + public void Compile_RequiredToolFilter_AppliesBeforeTheToolLimit() + { + // Arrange + List tools = [.. Enumerable.Range(0, JevToolCallCompiler.MaxRoutableTools + 1).Select(index => Declaration($"tool{index}", """{ "type": "object", "properties": {} }"""))]; + + // Act + JevToolCallPlan plan = Compile(tools, ChatToolMode.RequireSpecific("tool3"))!; + + // Assert + Assert.Equal("tool3", plan.RequiredTool!.Function.Name); + Assert.Null(plan.RouteQuestionId); + } + + [Fact] + public void Compile_TooManyInternalQuestions_Throws() + { + // Arrange: each tool needs 40 member questions, so the fourth exceeds 128. + string members = string.Join(",", Enumerable.Range(0, 40).Select(index => $"\"m{index}\"")); + List tools = [.. Enumerable.Range(0, 4).Select(index => Declaration($"tool{index}", $$"""{ "type": "object", "properties": { "tags": { "type": "array", "items": { "enum": [{{members}}] } } }, "required": ["tags"] }"""))]; + + // Act & Assert + InvalidOperationException exception = Assert.Throws(() => Compile(tools)); + Assert.Contains("128", exception.Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() + { + // Arrange: the first tool fails only after its array reserved 64 questions; they must be released. + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + AIFunctionDeclaration failing = Declaration("failing", $$"""{ "type": "object", "properties": { "tags": { "type": "array", "items": { "enum": [{{members}}] } }, "text": { "type": "string" } }, "required": ["tags"] }"""); + AIFunctionDeclaration large = Declaration("large", $$"""{ "type": "object", "properties": { "a": { "type": "array", "items": { "enum": [{{members}}] } }, "b": { "type": "array", "items": { "enum": [{{string.Join(",", Enumerable.Range(0, 60).Select(index => $"\"n{index}\""))}}] } } }, "required": ["a", "b"] }"""); + + // Act + JevToolCallPlan plan = Compile([failing, large])!; + + // Assert + Assert.Equal(125, plan.Questions.Count); + } + + [Fact] + public void Compile_TooManyEnumValuesOrProperties_ExcludeTheTool() + { + // Arrange + string values = string.Join(",", Enumerable.Range(0, JevToolCallCompiler.MaxEnumValues + 1).Select(index => $"\"v{index}\"")); + string properties = string.Join(",", Enumerable.Range(0, JevToolCallCompiler.MaxToolProperties + 1).Select(index => $"\"p{index}\": {{ \"type\": \"boolean\" }}")); + + // Act + JevToolCallPlan? plan = Compile( + [ + Declaration("enum", $$"""{ "type": "object", "properties": { "a": { "enum": [{{values}}] } } }"""), + Declaration("wide", $$"""{ "type": "object", "properties": { {{properties}} } }"""), + ]); + + // Assert + Assert.Null(plan); + } + + [Fact] + public void Decode_UnknownRouteOrMissingAnswer_Throws() + { + // Arrange + JevToolCallPlan plan = Compile([Declaration("flag", """{ "type": "object", "properties": { "on": { "type": "boolean" } }, "required": ["on"] }""")])!; + + // Act & Assert + Assert.Throws(() => plan.Decode(Result(JevChatTestData.RouteTo("t9", "t9")))); + Assert.Throws(() => plan.Decode(Result(JevChatTestData.RouteTo("t0", "t0")))); + } + + [Fact] + public void Compile_DuplicateDescriptions_KeepToolNamesInTheRoute() + { + // Arrange + AIFunctionDeclaration first = Declaration("first", """{ "type": "object", "properties": {} }""", "Does it."); + AIFunctionDeclaration second = Declaration("second", """{ "type": "object", "properties": {} }""", "Does it."); + + // Act + JevChoiceQuestion route = Assert.IsType(Compile([first, second])!.Questions[JevChatTestData.Route]); + + // Assert + Assert.Equal("Tool 'first': Does it.", route.Criteria["t0"].Text); + Assert.Equal("Tool 'second': Does it.", route.Criteria["t1"].Text); + } + + [Fact] + public void Describe_MatchesPythonJsonDumps() + { + // Arrange: json.dumps(value, ensure_ascii=False, sort_keys=True) in the Python connector. + JsonElement value = JsonDocument.Parse("""{"city":"São Paulo","b":[1,2.5,true,null],"q":"say \"hi\"\n"}""").RootElement; + + // Act + string described = JevToolSchema.Describe(value); + + // Assert + Assert.Equal("""{"b": [1, 2.5, true, null], "city": "São Paulo", "q": "say \"hi\"\n"}""", described); + } + + private static AIFunctionDeclaration Declaration(string name, string schema, string? description = null) => + AIFunctionFactory.CreateDeclaration(name, description, JsonDocument.Parse(schema).RootElement.Clone()); + + private static JevToolCallPlan? Compile(IReadOnlyList tools, ChatToolMode? mode = null, ILogger? logger = null) => + JevToolCallCompiler.Compile(tools, mode, s_noCalls, logger ?? new ListLoggerFactory().CreateLogger("test")); + + private static JevResult Result(params string[] answers) => + JsonSerializer.Deserialize(JevChatTestData.Result(answers), JevJsonContext.Default.JevResult)!; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs new file mode 100644 index 00000000000..df54fb10202 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/ListLoggerFactory.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; + +namespace Microsoft.Agents.AI.TypeSafe.UnitTests; + +/// +/// A logger factory that records every message, so tests can check what a client logs. +/// +internal sealed class ListLoggerFactory : ILoggerFactory +{ + public List<(LogLevel Level, string Message)> Entries { get; } = []; + + public ILogger CreateLogger(string categoryName) => new ListLogger(this.Entries); + + public void AddProvider(ILoggerProvider provider) + { + } + + public void Dispose() + { + } + + private sealed class ListLogger(List<(LogLevel Level, string Message)> entries) : ILogger + { + public IDisposable? BeginScope(TState state) + where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => true; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) => + entries.Add((logLevel, formatter(state, exception))); + } +} From 3e7756ee2b53259b17065e716ce78e851cb677d1 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:46:36 +0100 Subject: [PATCH 3/6] .NET: Address TypeSafe review feedback on schema checks and the loop judge sample - Follow chains of local $ref aliases when compiling tool schemas, bounded by the existing resolution depth, so an alias of a definition is routed instead of being excluded and a reference cycle excludes the tool. - Check const and enum values against their declared type, as array members already were, so a tool whose values could only fail when called is excluded before Jev routes a call to it. - Name the criterion threshold in the loop judge sample and document how to choose it, and how to word criteria for a judge. --- .../Program.cs | 10 ++- .../Agent_TypeSafe_Step01_LoopJudge/README.md | 20 +++++- .../Core/JevToolCallCompiler.cs | 16 ++--- .../Core/JevToolSchema.cs | 46 ++++++++++++- .../JevToolCallCompilerTests.cs | 66 +++++++++++++++++++ 5 files changed, 142 insertions(+), 16 deletions(-) diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs index 4bfe9cb9c7c..5310dbff117 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/Program.cs @@ -18,7 +18,8 @@ var typeSafeApiKey = Environment.GetEnvironmentVariable("TYPESAFE_API_KEY") ?? throw new InvalidOperationException("TYPESAFE_API_KEY is not set."); // 1. The criteria. Each one becomes a Noul question, whose answer is the probability that the answer meets it, so the -// judge reports exactly which criteria are missing instead of a single verdict. +// judge reports exactly which criteria are missing instead of a single verdict. The questions describe what a good +// answer contains; wording that steers the judge, such as "only say yes if", tends to push every answer one way. Dictionary criteria = new() { ["blue_sky"] = "explains why the daytime sky appears blue", @@ -29,6 +30,11 @@ criterion => criterion.Key, criterion => (JevQuestion)new JevNoulQuestion { Instructions = $"Does the assistant's latest response {criterion.Value}?" }); +// A criterion is met when its probability is above this threshold. 0.5 is not calibrated for any task: a higher value +// accepts fewer answers that still miss a criterion, at the cost of more loop runs. Choose it on labeled examples of +// your own answers. +const double CriterionThreshold = 0.5; + // 2. The agent that writes the answers. Its instructions hold back part of the first answer, so the judge has a gap // to find and the loop runs more than once. // WARNING: DefaultAzureCredential is convenient for development but requires careful consideration in production. @@ -89,7 +95,7 @@ async Task> JudgeAsync(IEnumerable request, string ans foreach ((string id, string description) in criteria) { double probability = ((JevNoulResponse)result.Answers[id]).Noul; - bool met = probability > 0.5; + bool met = probability > CriterionThreshold; Console.WriteLine($" {id}: {(met ? "met" : "missing")} (probability {probability:F2})"); if (!met) { diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md index be92f66704a..12c563f986b 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_TypeSafe_Step01_LoopJudge/README.md @@ -39,12 +39,30 @@ var evaluator = new DelegateLoopEvaluator(async (context, cancellationToken) => ``` `JudgeAsync` sends Jev the original request and the answer as a conversation, with -`new ChatOptions().WithJevQuestions(questions)`, and returns the criteria whose Noul is 0.5 or lower. +`new ChatOptions().WithJevQuestions(questions)`, and returns the criteria whose Noul is at or below +`CriterionThreshold`. Jev reads the original request and the latest answer as a conversation, and each question refers to "the assistant's latest response". The same pattern works with a `JevScoreQuestion` for a graded rubric, or a `JevChoiceQuestion` to classify what kind of revision is needed. +## Writing criteria for a judge + +- **Choose the threshold on your own data.** The sample treats a criterion as met when its probability is above + `CriterionThreshold`, which is 0.5. That value is not calibrated for any task. The threshold trades two errors + against each other: a higher value accepts fewer answers that still miss a criterion, and costs more loop runs, while + a lower value stops sooner and lets more incomplete answers through. Pick it from labeled examples of the answers your + agent writes, based on which error costs you more. +- **Describe the criterion, do not steer the judge.** Ask what a good answer contains, as the sample does ("Does the + assistant's latest response explain why sunsets appear red?"). Directional wording such as "only say yes if" or + "lean towards yes" tends to push every answer the same way instead of making the judge more accurate. When a + criterion is hard to judge, a neutral description of what is being judged helps more than stronger wording. +- **Ask about the response, not the topic.** When the conversation contains harmful or sensitive content, a question + such as "Is this safe?" can be read as a question about that content rather than about the answer. A correct + refusal could then look like a missing criterion, and the loop would keep revising an answer that was already right. + Ask how the response handles the request, for example "Does the assistant's latest response decline to give + dangerous instructions?", and check such criteria on labeled examples, including refusals, before relying on them. + ## Prerequisites - .NET 10 SDK or later diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs index 4661cd6e096..e731c1692c6 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs @@ -17,7 +17,9 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// /// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same -/// question IDs, limits, and wording, so both connectors ask Jev the same questions. +/// question IDs, limits, and wording, so both connectors ask Jev the same questions. Two schema checks are stricter +/// than in Python: chains of $ref aliases are followed instead of excluding the tool, and const and +/// enum values must match their declared type, so a tool that could only fail when called is excluded. /// /// /// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum @@ -312,6 +314,7 @@ void AddPresenceQuestion() if (schema.TryGetValue("const", out JsonElement constValue)) { + JevToolSchema.EnsureValuesMatchType(schema, [constValue], "argument", "const value", "type"); budget.Reserve(required ? 0 : 1); AddPresenceQuestion(); return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; @@ -325,6 +328,7 @@ void AddPresenceQuestion() throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "enum defines {0} values; the supported maximum is {1}", values.Count, MaxEnumValues)); } + JevToolSchema.EnsureValuesMatchType(schema, values, "argument", "enum value", "type"); if (values.Count == 1) { budget.Reserve(required ? 0 : 1); @@ -390,15 +394,7 @@ void AddPresenceQuestion() throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "array enum defines {0} values; the supported maximum is {1}", members.Count, MaxEnumValues)); } - IReadOnlyList itemTypes = JevToolSchema.GetTypes(items.TryGetValue("type", out JsonElement itemType) ? itemType : null, "array item"); - foreach (JsonElement member in members) - { - if (itemTypes.Count > 0 && !itemTypes.Any(candidate => JevToolSchema.MatchesType(member, candidate))) - { - string declared = itemTypes.Count == 1 ? $"\"{itemTypes[0]}\"" : JevToolSchema.Describe(itemTypes.Select(candidate => JevJsonUtilities.ToElement(candidate))); - throw new JevUnsupportedToolSchemaException($"array enum member {JevToolSchema.Describe(member)} does not match declared item type {declared}"); - } - } + JevToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); budget.Reserve(members.Count + (required ? 0 : 1)); AddPresenceQuestion(); diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs index 07bcbb61309..52209e687a8 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -57,8 +57,16 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di } var resolved = new Dictionary(schema, StringComparer.Ordinal); - if (resolved.Remove("$ref", out JsonElement reference)) + + // A definition can itself be a reference, such as an alias of another definition, so references are followed + // until a schema without one. A cycle would never end, so the chain is bounded like nested resolution. + for (int hops = 0; resolved.Remove("$ref", out JsonElement reference); hops++) { + if (depth + hops >= MaxResolutionDepth) + { + throw new JevUnsupportedToolSchemaException("schema references are nested too deeply"); + } + RejectCompositionSiblings(resolved, "$ref"); string? path = reference.ValueKind == JsonValueKind.String ? reference.GetString() : null; const string DefinitionsPrefix = "#/$defs/"; @@ -75,8 +83,9 @@ public static (Dictionary Schema, bool Nullable) Resolve(Di throw new JevUnsupportedToolSchemaException($"unresolved schema reference '{path}'"); } - // Annotations next to the reference, such as a description, override those of the definition. An anyOf - // inside the definition is resolved below, together with the rest of the merged schema. + // Annotations next to the reference, such as a description, override those of the definition, so the + // annotations closest to the argument win along a chain. An anyOf inside the definition is resolved below, + // together with the rest of the merged schema. resolved = Merge(ToMap(definition), resolved); } @@ -165,6 +174,37 @@ public static bool MatchesType(JsonElement value, string type) => _ => false, }; + /// + /// Checks that every const or enum value of a schema matches its declared type. + /// + /// + /// A value of another type can never be a valid argument, for example "fast" for an integer. Jev would + /// still choose it, and the call would fail only when the function binds its arguments, so such a tool is left out + /// of routing instead. A schema without a type is not checked. + /// + /// The resolved schema whose type applies. + /// The values to check. + /// Where the schema is, for the message about an invalid type keyword. + /// How a value is named in the message, such as enum value. + /// How the type is named in the message, such as type. + public static void EnsureValuesMatchType(Dictionary schema, IEnumerable values, string location, string valueName, string typeName) + { + IReadOnlyList types = GetTypes(schema.TryGetValue("type", out JsonElement type) ? type : null, location); + if (types.Count == 0) + { + return; + } + + foreach (JsonElement value in values) + { + if (!types.Any(candidate => MatchesType(value, candidate))) + { + string declared = types.Count == 1 ? $"\"{types[0]}\"" : Describe(types.Select(candidate => JevJsonUtilities.ToElement(candidate))); + throw new JevUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared {typeName} {declared}"); + } + } + } + /// /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): /// sorted object keys, ", " and ": " separators, and unescaped text. Jev reads these values in diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs index e78d148c492..3c2abeeaddb 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -46,6 +46,66 @@ public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() Assert.Equal("fixed", ((JsonElement)call.Value.Arguments["kind"]!).GetString()); } + [Fact] + public void Compile_ChainedReferences_ResolveAndDecode() + { + // Arrange: Speed aliases Mode, and the nullable Level aliases a nullable enum, so each reference leads to another. + AIFunctionDeclaration tool = Declaration("set_speed", """ + { + "type": "object", + "properties": { + "speed": { "$ref": "#/$defs/Speed", "description": "How fast to go" }, + "level": { "$ref": "#/$defs/Level" } + }, + "required": ["speed"], + "$defs": { + "Speed": { "$ref": "#/$defs/Mode" }, + "Mode": { "type": "string", "enum": ["fast", "safe"] }, + "Level": { "$ref": "#/$defs/NullableLevel" }, + "NullableLevel": { "anyOf": [{ "type": "string", "enum": ["low", "high"] }, { "type": "null" }] } + } + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a1.present", 0.9), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); + + // Assert: the description next to the first reference wins, and the nullable alias still gets a presence question. + Assert.Contains("How fast to go", plan.Questions["__af_tool__.t0.a0.value"].Instructions!.Value.Text, StringComparison.Ordinal); + Assert.Equal("safe", ((JsonElement)call!.Value.Arguments["speed"]!).GetString()); + Assert.Equal("high", ((JsonElement)call.Value.Arguments["level"]!).GetString()); + } + + [Fact] + public void Compile_EnumAndConstValuesMatchingTheirType_Compile() + { + // Arrange: numbers, mixed types, and a nullable enum whose null was removed before the check. + AIFunctionDeclaration tool = Declaration("tune", """ + { + "type": "object", + "properties": { + "ratio": { "type": "number", "enum": [1, 2.5] }, + "mode": { "type": ["string", "integer"], "enum": ["auto", 3] }, + "unit": { "type": ["string", "null"], "enum": ["c", "f", null] }, + "version": { "type": "integer", "const": 2 } + }, + "required": ["ratio", "mode", "version"] + } + """); + + // Act + JevToolCallPlan? plan = Compile([tool]); + + // Assert + Assert.NotNull(plan); + Assert.Single(plan.Tools); + } + [Theory] [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean" } }, "minProperties": 1 }""", "unsupported root-level schema constraints: minProperties")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "enum": ["x", "y"], "minLength": 1 } }, "required": ["a"] }""", "required argument 'a': unsupported argument schema constraints: minLength")] @@ -56,6 +116,12 @@ public void Compile_ReferencesAndAnyOfNull_ResolveAndDecode() [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": "integer", "enum": [1, 2.5] } } } }""", "array enum member 2.5 does not match declared item type \"integer\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "items": { "type": ["string", "null"], "enum": ["x", null] } } } }""", "nullable array members are not supported")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer" } } }""", "only const, enum, boolean, and arrays of enum values are supported")] + [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A" } }, "$defs": { "A": { "$ref": "#/$defs/B" }, "B": { "$ref": "#/$defs/A" } } }""", "schema references are nested too deeply")] + [InlineData("""{ "type": "object", "properties": { "a": { "$ref": "#/$defs/A" } }, "$defs": { "A": { "$ref": "#/$defs/Missing" } } }""", "unresolved schema reference '#/$defs/Missing'")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast", "safe"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": ["integer", "boolean"], "enum": [1, "x"] } } }""", "enum value \"x\" does not match declared type [\"integer\", \"boolean\"]")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "const": 5 } }, "required": ["a"] }""", "required argument 'a': const value 5 does not match declared type \"string\"")] [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) From 3ac445d587fc845b9956c2e44d9f19eabb5f155e Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 2 Oct 2026 15:58:55 +0100 Subject: [PATCH 4/6] .NET: Link the TypeSafe quickstart instead of the sign-in console in the Jev sample The console page returns 403 to unauthenticated link checkers, which failed markdown-link-check. The quickstart explains how to get an API key and is publicly reachable. --- .../Agents/Agent_Step25_ClassificationWithJev/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md index cd289614e72..e4753e04d0e 100644 --- a/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md +++ b/dotnet/samples/02-agents/Agents/Agent_Step25_ClassificationWithJev/README.md @@ -110,7 +110,7 @@ the container or configuration. - .NET 10 SDK or later - A Microsoft Foundry project with a model deployment - Azure CLI installed and authenticated (for Azure credential authentication) -- A TypeSafe API key from the [TypeSafe console](https://console.typesafe.ai/keys) +- A TypeSafe API key (see the [TypeSafe quickstart](https://docs.typesafe.ai/introduction/quickstart)) Set the following environment variables: From 9894d5bea739e3724a6fb1b1ae217ba00e309298 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 2 Oct 2026 16:43:37 +0100 Subject: [PATCH 5/6] .NET: Accept integers of any size in TypeSafe tool schema checks The integer type check used TryGetInt64, so const, enum, and array values beyond the long range, such as unsigned 64-bit values, excluded otherwise valid tools. An integer is now any JSON number written without a fraction or exponent, like a Python int, so 1.0 and 1e2 are still rejected. --- .../Core/JevToolSchema.cs | 5 +-- .../JevToolCallCompilerTests.cs | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs index 52209e687a8..dc0d97e5c40 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -164,8 +164,9 @@ public static bool MatchesType(JsonElement value, string type) => { "string" => value.ValueKind == JsonValueKind.String, - // An integer has no fraction or exponent, as a JSON parser that separates integers from floats reads it. - "integer" => value.ValueKind == JsonValueKind.Number && value.TryGetInt64(out _), + // An integer has no fraction or exponent, as a JSON parser that separates integers from floats reads it, + // and may be of any size, like a Python int; a long would reject valid unsigned 64-bit values. + "integer" => value.ValueKind == JsonValueKind.Number && value.GetRawText().AsSpan().IndexOfAny('.', 'e', 'E') < 0, "number" => value.ValueKind == JsonValueKind.Number, "boolean" => value.ValueKind is JsonValueKind.True or JsonValueKind.False, "null" => value.ValueKind == JsonValueKind.Null, diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs index 3c2abeeaddb..e32f3fd8538 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -81,6 +81,36 @@ public void Compile_ChainedReferences_ResolveAndDecode() Assert.Equal("high", ((JsonElement)call.Value.Arguments["level"]!).GetString()); } + [Fact] + public void Compile_IntegersOutsideTheInt64Range_Compile() + { + // Arrange: an unsigned 64-bit contract, whose largest values do not fit in a long. + AIFunctionDeclaration tool = Declaration("set_mask", """ + { + "type": "object", + "properties": { + "mask": { "type": "integer", "enum": [0, 18446744073709551615] }, + "all": { "type": "integer", "const": 18446744073709551615 }, + "bits": { "type": "array", "items": { "type": "integer", "enum": [1, 9223372036854775808] } } + }, + "required": ["mask", "all", "bits"] + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a0.value", "v1", "v0", "v1"), + JevChatTestData.Noul("__af_tool__.t0.a2.m0", 0.1), + JevChatTestData.Noul("__af_tool__.t0.a2.m1", 0.9))); + + // Assert: the values reach the function unchanged. + Assert.Equal(ulong.MaxValue, ((JsonElement)call!.Value.Arguments["mask"]!).GetUInt64()); + Assert.Equal(ulong.MaxValue, ((JsonElement)call.Value.Arguments["all"]!).GetUInt64()); + Assert.Equal([9223372036854775808UL], ((JsonElement)call.Value.Arguments["bits"]!).EnumerateArray().Select(item => item.GetUInt64())); + } + [Fact] public void Compile_EnumAndConstValuesMatchingTheirType_Compile() { @@ -122,6 +152,8 @@ public void Compile_EnumAndConstValuesMatchingTheirType_Compile() [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": ["fast"] } } }""", "enum value \"fast\" does not match declared type \"integer\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": ["integer", "boolean"], "enum": [1, "x"] } } }""", "enum value \"x\" does not match declared type [\"integer\", \"boolean\"]")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "const": 5 } }, "required": ["a"] }""", "required argument 'a': const value 5 does not match declared type \"string\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": [1, 2.0] } } }""", "enum value 2.0 does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "const": 1e2 } }, "required": ["a"] }""", "const value 1e2 does not match declared type \"integer\"")] [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) From 78d2948474e2ec594cf6ec107c6e4b7c92a0ffcc Mon Sep 17 00:00:00 2001 From: Roger Barreto Date: Tue, 20 Oct 2026 15:44:15 -0700 Subject: [PATCH 6/6] .NET: Fix TypeSafe turn boundaries and tool schema compilation --- .../typesafe/Agent_With_TypeSafe/README.md | 7 + .../Core/JevChatState.cs | 36 ++++- .../Core/JevQuestionBudget.cs | 14 +- .../Core/JevToolCallCompiler.cs | 75 +++++----- .../Core/JevToolSchema.cs | 75 ++++++++++ .../JevChatClientOptions.cs | 4 + .../JevChatClientToolTests.cs | 137 ++++++++++++++++++ .../JevToolCallCompilerTests.cs | 106 ++++++++++++++ 8 files changed, 409 insertions(+), 45 deletions(-) diff --git a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md index 77ac532b541..0d5dd3a98e4 100644 --- a/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md +++ b/dotnet/samples/02-agents/AgentProviders/typesafe/Agent_With_TypeSafe/README.md @@ -51,6 +51,8 @@ The client follows the TypeSafe connector for Python (`agent-framework-typesafe` mean nothing. In this sample, saying that `detailed` is true "when the user asks for a detailed report" is what makes Jev choose it reliably. Tools with free-form arguments, such as text or numbers, are left out of the request with a warning in the log. + Constant and enum values must satisfy their other schema constraints too, including array item constraints; + tools with conflicting constraints are excluded rather than called with invalid arguments. 5. **Tool approval.** A tool wrapped in `ApprovalRequiredAIFunction` runs only after the user approves the call that Jev chose. The first run returns a `ToolApprovalRequestContent`; the sample asks for approval in the console and @@ -67,6 +69,11 @@ Jev reads, because the call and result they lead to carry the information. Because streaming is not supported, run agents with `RunAsync`. A function-invoking client runs approved tools before it calls the chat client, so a streaming run that resumes a tool approval would run the tool and then fail. +Agents tag stored messages as chat history, which lets Jev distinguish a new user request from text moved after a +resumed approval's tool result. When managing history yourself with `FunctionInvokingChatClient`, tag prior messages +with `message.WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory)` before appending new input. +Without that attribution, a user message starts a new turn. + ## Configuring the client `JevChatClientOptions` follows the `ClientPipelineOptions` pattern of the OpenAI and Azure AI client libraries, with the diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs index bbbc5a84582..27c121da184 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevChatState.cs @@ -97,10 +97,9 @@ public static JevEntry Build(IReadOnlyList messages, string? instru /// request. /// /// - /// User messages that directly follow a function result continue the turn instead of starting one. - /// FunctionInvokingChatClient moves text that was sent together with an approval response, such as "Yes, go - /// ahead", after the approved call and its result. Starting the turn there would hide the approved call from the - /// tool call limit and offer the tool again with no hint that it already ran. + /// FunctionInvokingChatClient moves text sent with an approval response after the resumed call and its result. + /// The agent tags the reconstructed call as chat history, while its new result is external. That pairing + /// distinguishes an approval continuation from a new user request after a run that ended with a tool result. /// public static int GetCurrentTurnStart(IReadOnlyList messages) { @@ -120,7 +119,7 @@ public static int GetCurrentTurnStart(IReadOnlyList messages) runStart--; } - if (runStart > 0 && messages[runStart - 1].Contents.Any(content => content is FunctionResultContent)) + if (runStart > 0 && IsResumedToolResult(messages, runStart - 1)) { index = runStart - 1; continue; @@ -132,6 +131,33 @@ public static int GetCurrentTurnStart(IReadOnlyList messages) return 0; } + private static bool IsResumedToolResult(IReadOnlyList messages, int resultIndex) + { + ChatMessage resultMessage = messages[resultIndex]; + if (resultMessage.GetAgentRequestMessageSourceType() != AgentRequestMessageSourceType.External) + { + return false; + } + + var callIds = new HashSet(resultMessage.Contents.OfType().Select(result => result.CallId), StringComparer.Ordinal); + for (int index = resultIndex - 1; callIds.Count > 0 && index >= 0; index--) + { + ChatMessage message = messages[index]; + if (message.Role == ChatRole.User) + { + break; + } + + if (message.GetAgentRequestMessageSourceType() == AgentRequestMessageSourceType.ChatHistory && + message.Contents.OfType().Any(call => callIds.Contains(call.CallId))) + { + return true; + } + } + + return false; + } + /// Gets the argument sets of the calls in the current turn, by tool name, oldest first. public static Dictionary>> GetCurrentTurnCalls(IReadOnlyList messages, int turnStart) { diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs index f6fe3f278cf..96e995e4911 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevQuestionBudget.cs @@ -12,6 +12,7 @@ namespace Microsoft.Agents.AI.TypeSafe; /// /// Each tool is compiled against a copy of the budget, which is committed only when the tool compiles. A tool that /// is excluded for an unsupported schema therefore returns the questions it reserved to the tools after it. +/// Exceeding the budget stops question materialization, but is reported only after the whole tool schema is checked. /// internal sealed class JevQuestionBudget { @@ -24,10 +25,15 @@ public JevQuestionBudget(int count = 0) public int Count { get; set; } - public void Reserve(int count) + public bool TryReserve(int count) { - int next = this.Count + count; - if (next > MaxInternalQuestions) + this.Count += count; + return this.Count <= MaxInternalQuestions; + } + + public void Validate() + { + if (this.Count > MaxInternalQuestions) { // Too many questions is a property of the whole tool set, not of one tool, so it stops the request instead // of excluding the tool that crossed the limit. @@ -36,7 +42,5 @@ public void Reserve(int count) "The tools need more than {0} internal Jev questions. Offer fewer tools, or tools with fewer arguments.", MaxInternalQuestions)); } - - this.Count = next; } } diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs index e731c1692c6..1f420dbf877 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolCallCompiler.cs @@ -19,7 +19,7 @@ namespace Microsoft.Agents.AI.TypeSafe; /// This is a port of the tool routing in the Python TypeSafe connector (_tool_calls.py), with the same /// question IDs, limits, and wording, so both connectors ask Jev the same questions. Two schema checks are stricter /// than in Python: chains of $ref aliases are followed instead of excluding the tool, and const and -/// enum values must match their declared type, so a tool that could only fail when called is excluded. +/// enum values must satisfy their declared constraints, so a tool that could fail when called is excluded. /// /// /// A route Choice selects the tool, or none. Each argument must come from a closed set: a constant, an enum @@ -163,7 +163,8 @@ public static void EnsureNoReservedQuestionIds(IEnumerable questionIds) criteria[RouteNone] = "Do not call a tool because all requested tool actions are already satisfied, or because no tool is needed."; } - budget.Reserve(1); + _ = budget.TryReserve(1); + budget.Validate(); questions[RouteQuestionId] = new JevChoiceQuestion { Instructions = "Which available tool, if any, should handle the latest user request?", @@ -255,7 +256,10 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI List previousValues = [.. previousCalls.Where(call => call.ContainsKey(property.Name)).Select(call => call[property.Name])]; try { - arguments.Add(CompileArgument(tool, schema, property.Name, JevToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions)); + if (CompileArgument(tool, schema, property.Name, JevToolSchema.ToMap(property.Value), isRequired, toolIndex, argumentIndex, previousValues, budget, questions) is { } argument) + { + arguments.Add(argument); + } } catch (JevUnsupportedToolSchemaException ex) { @@ -263,6 +267,8 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI } } + // Unsupported tools are excluded even if their earlier arguments exceeded the question budget. + budget.Validate(); return new JevCompiledTool { Function = tool, @@ -272,7 +278,7 @@ private static JevCompiledTool CompileTool(AIFunctionDeclaration tool, int toolI }; } - private static JevArgumentPlan CompileArgument( + private static JevArgumentPlan? CompileArgument( AIFunctionDeclaration tool, Dictionary root, string name, @@ -314,28 +320,36 @@ void AddPresenceQuestion() if (schema.TryGetValue("const", out JsonElement constValue)) { - JevToolSchema.EnsureValuesMatchType(schema, [constValue], "argument", "const value", "type"); - budget.Reserve(required ? 0 : 1); + JevToolSchema.EnsureValuesMatchSchema(schema, [constValue], root, "const value"); + if (!budget.TryReserve(required ? 0 : 1)) + { + return null; + } + AddPresenceQuestion(); return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = constValue }; } - if (schema.TryGetValue("enum", out JsonElement enumElement) && enumElement.ValueKind == JsonValueKind.Array && enumElement.GetArrayLength() > 0) + if (schema.TryGetValue("enum", out JsonElement enumElement)) { - List values = [.. enumElement.EnumerateArray()]; - if (values.Count > MaxEnumValues) - { - throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "enum defines {0} values; the supported maximum is {1}", values.Count, MaxEnumValues)); - } - - JevToolSchema.EnsureValuesMatchType(schema, values, "argument", "enum value", "type"); + List values = JevToolSchema.ReadEnumValues(enumElement, "enum"); + JevToolSchema.EnsureValuesMatchSchema(schema, values, root, "enum value"); if (values.Count == 1) { - budget.Reserve(required ? 0 : 1); + if (!budget.TryReserve(required ? 0 : 1)) + { + return null; + } + AddPresenceQuestion(); return new JevArgumentPlan { Name = name, Kind = JevArgumentKind.Const, PresenceQuestionId = presenceQuestionId, ConstValue = values[0] }; } + if (!budget.TryReserve(required ? 1 : 2)) + { + return null; + } + var labeled = new Dictionary(StringComparer.Ordinal); var criteria = new Dictionary(StringComparer.Ordinal); for (int index = 0; index < values.Count; index++) @@ -346,7 +360,6 @@ void AddPresenceQuestion() } string questionId = prefix + ".value"; - budget.Reserve(required ? 1 : 2); AddPresenceQuestion(); questions[questionId] = new JevChoiceQuestion { @@ -360,7 +373,11 @@ void AddPresenceQuestion() if (schemaType == "boolean") { string questionId = prefix + ".value"; - budget.Reserve(required ? 1 : 2); + if (!budget.TryReserve(required ? 1 : 2)) + { + return null; + } + AddPresenceQuestion(); questions[questionId] = new JevNoulQuestion { @@ -371,32 +388,20 @@ void AddPresenceQuestion() if (schemaType == "array") { - if (!schema.TryGetValue("items", out JsonElement itemsElement) || itemsElement.ValueKind != JsonValueKind.Object) - { - throw new JevUnsupportedToolSchemaException("array items must define an enum"); - } - - (Dictionary items, bool itemsNullable) = JevToolSchema.Resolve(JevToolSchema.ToMap(itemsElement), root); - JevToolSchema.RejectUnsupportedKeys(items, JevToolSchema.ArrayItemKeys, "array item"); - if (itemsNullable) - { - throw new JevUnsupportedToolSchemaException("nullable array members are not supported"); - } - + Dictionary items = JevToolSchema.ResolveArrayItems(schema, root); if (!items.TryGetValue("enum", out JsonElement membersElement) || membersElement.ValueKind != JsonValueKind.Array || membersElement.GetArrayLength() == 0) { throw new JevUnsupportedToolSchemaException("array items must define a non-empty enum"); } - List members = [.. membersElement.EnumerateArray()]; - if (members.Count > MaxEnumValues) + List members = JevToolSchema.ReadEnumValues(membersElement, "array enum"); + JevToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); + + if (!budget.TryReserve(members.Count + (required ? 0 : 1))) { - throw new JevUnsupportedToolSchemaException(string.Format(CultureInfo.InvariantCulture, "array enum defines {0} values; the supported maximum is {1}", members.Count, MaxEnumValues)); + return null; } - JevToolSchema.EnsureValuesMatchType(items, members, "array item", "array enum member", "item type"); - - budget.Reserve(members.Count + (required ? 0 : 1)); AddPresenceQuestion(); var memberQuestions = new List>(members.Count); for (int index = 0; index < members.Count; index++) diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs index dc0d97e5c40..59454182438 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/Core/JevToolSchema.cs @@ -206,6 +206,81 @@ public static void EnsureValuesMatchType(Dictionary schema, } } + public static List ReadEnumValues(JsonElement values, string name) + { + if (values.ValueKind != JsonValueKind.Array || values.GetArrayLength() == 0) + { + throw new JevUnsupportedToolSchemaException($"{name} must be a non-empty array"); + } + + if (values.GetArrayLength() > JevToolCallCompiler.MaxEnumValues) + { + throw new JevUnsupportedToolSchemaException(string.Format( + CultureInfo.InvariantCulture, + "{0} defines {1} values; the supported maximum is {2}", + name, + values.GetArrayLength(), + JevToolCallCompiler.MaxEnumValues)); + } + + return [.. values.EnumerateArray()]; + } + + public static Dictionary ResolveArrayItems(Dictionary schema, Dictionary root) + { + if (!schema.TryGetValue("items", out JsonElement element) || element.ValueKind != JsonValueKind.Object) + { + throw new JevUnsupportedToolSchemaException("array items must define an object schema"); + } + + (Dictionary items, bool nullable) = Resolve(ToMap(element), root); + RejectUnsupportedKeys(items, ArrayItemKeys, "array item"); + if (nullable) + { + throw new JevUnsupportedToolSchemaException("nullable array members are not supported"); + } + + return items; + } + + /// Checks all supported constraints before offering a constant or enum value to Jev. + public static void EnsureValuesMatchSchema(Dictionary schema, IReadOnlyList values, Dictionary root, string valueName) + { + EnsureValuesMatchType(schema, values, "argument", valueName, "type"); + EnsureValuesMatchEnum(schema, values, valueName); + if (!schema.ContainsKey("items")) + { + return; + } + + Dictionary items = ResolveArrayItems(schema, root); + if (items.TryGetValue("enum", out JsonElement itemEnum)) + { + EnsureValuesMatchType(items, ReadEnumValues(itemEnum, "array enum"), "array item", "array enum member", "item type"); + } + + foreach (JsonElement value in values.Where(value => value.ValueKind == JsonValueKind.Array)) + { + EnsureValuesMatchType(items, value.EnumerateArray(), "array item", "array member", "item type"); + EnsureValuesMatchEnum(items, value.EnumerateArray(), "array member"); + } + } + + private static void EnsureValuesMatchEnum(Dictionary schema, IEnumerable values, string valueName) + { + if (schema.TryGetValue("enum", out JsonElement element)) + { + List alternatives = ReadEnumValues(element, "enum"); + foreach (JsonElement value in values) + { + if (!alternatives.Any(alternative => JsonElement.DeepEquals(value, alternative))) + { + throw new JevUnsupportedToolSchemaException($"{valueName} {Describe(value)} does not match declared enum {Describe(alternatives)}"); + } + } + } + } + /// /// Writes a value the way the Python connector does with json.dumps(value, ensure_ascii=False, sort_keys=True): /// sorted object keys, ", " and ": " separators, and unescaped text. Jev reads these values in diff --git a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs index a8de9a73812..61d365557b5 100644 --- a/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.TypeSafe/JevChatClientOptions.cs @@ -90,6 +90,10 @@ public IReadOnlyDictionary? DefaultQuestions /// which calls the turn already made, but a closed-set choice can still repeat one, so the limit is what /// guarantees that the tool loop ends. Raise it when one user request needs several distinct calls, such as the /// weather in two cities; set it to 0 to never call tools. + /// When managing history without an agent, tag prior messages with + /// and + /// so a newly produced approval result can be distinguished + /// from an older tool result. Without that attribution, a user message starts a new turn. /// /// The value is negative. public int MaximumToolCallsPerTurn diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs index 7ff09a33c35..7774a0dac01 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevChatClientToolTests.cs @@ -442,6 +442,143 @@ public async Task RunAsync_ApprovalSentWithText_ContinuesTheTurnOfTheApprovedCal Assert.Equal("Refund issued.\ndepartment: billing", approved.Text); } + [Fact] + public async Task GetResponseAsync_ApprovalWithManuallyAttributedHistory_ContinuesTheTurnAsync() + { + // Arrange + int executions = 0; + AIFunction refresh = new ApprovalRequiredAIFunction(AIFunctionFactory.Create(() => + { + executions++; + return "Refreshed."; + }, "refresh")); + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers())); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + using var invoking = new FunctionInvokingChatClient(client); + ChatOptions options = new ChatOptions { Tools = [refresh] }.WithJevQuestions(JevChatTestData.Questions()); + List messages = [new(ChatRole.User, "Refresh the data.")]; + + // Act + ChatResponse pending = await invoking.GetResponseAsync(messages, options); + ToolApprovalRequestContent approval = pending.Messages.SelectMany(message => message.Contents).OfType().Single(); + messages = [.. messages.Concat(pending.Messages).Select(message => message.WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory))]; + messages.Add(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")])); + ChatResponse response = await invoking.GetResponseAsync(messages, options); + + // Assert + Assert.Equal(1, executions); + Assert.Equal(2, handler.Requests.Count); + Assert.Equal(["department", "is_urgent"], JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject().Select(question => question.Key)); + Assert.Equal("Refreshed.\ndepartment: technical", response.Text); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task RunAsync_NewRequestAfterTerminatedToolCall_StartsANewTurnAsync(bool requiresApproval) + { + // Arrange + int executions = 0; + AIFunction refresh = AIFunctionFactory.Create(() => $"Execution {++executions}", "refresh"); + if (requiresApproval) + { + refresh = new ApprovalRequiredAIFunction(refresh); + } + + using var handler = new JevTestHttpHandler() + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))) + .Reply(HttpStatusCode.OK, JevChatTestData.Result(JevChatTestData.UserAnswers(), JevChatTestData.RouteTo("t0", "t0", "none"))); + using var httpClient = new HttpClient(handler); + using JevChatClient client = JevChatClientTests.CreateClient(httpClient); + AIAgent agent = client.AsAIAgent(new ChatClientAgentOptions { ChatOptions = new ChatOptions { Tools = [refresh] }.WithJevQuestions(JevChatTestData.Questions()) }) + .AsBuilder() + .Use(async (_, context, next, cancellationToken) => + { + context.Terminate = true; + return await next(context, cancellationToken); + }) + .Build(); + AgentSession session = await agent.CreateSessionAsync(); + + // Act + await RunAsync("Refresh the data."); + AgentResponse response = await RunAsync("Refresh it again."); + + // Assert + Assert.Equal(2, executions); + Assert.Equal(2, handler.Requests.Count); + JsonObject questions = JsonNode.Parse(handler.Requests[1].Body)!["questions"]!.AsObject(); + Assert.NotNull(questions[Route]); + Assert.DoesNotContain("already called this turn", (string?)questions[Route]!["criteria"]!["t0"], StringComparison.Ordinal); + Assert.Equal("Execution 2", Assert.Single(response.Messages.SelectMany(message => message.Contents).OfType()).Result?.ToString()); + + async Task RunAsync(string request) + { + AgentResponse response = await agent.RunAsync(request, session); + if (response.Messages.SelectMany(message => message.Contents).OfType().SingleOrDefault() is { } approval) + { + return await agent.RunAsync(new ChatMessage(ChatRole.User, [approval.CreateResponse(true), new TextContent("Yes, go ahead.")]), session); + } + + return response; + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void GetCurrentTurnStart_NewUserAfterFunctionResult_StartsANewTurn(bool withHistoryAttribution) + { + // Arrange + ChatMessage[] messages = + [ + new(ChatRole.User, "Refresh the data."), + new(ChatRole.Assistant, [new FunctionCallContent("call_1", "refresh")]), + new(ChatRole.Tool, [new FunctionResultContent("call_1", "Old result")]), + new(ChatRole.User, "Refresh it again."), + ]; + if (withHistoryAttribution) + { + for (int index = 0; index < messages.Length - 1; index++) + { + messages[index] = messages[index].WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory); + } + } + + // Act + int turnStart = JevChatState.GetCurrentTurnStart(messages); + + // Assert + Assert.Equal(messages.Length, turnStart); + Assert.Equal(0, JevChatState.CountCurrentTurnCalls(messages, turnStart)); + Assert.Empty(JevChatState.GetCurrentTurnResultTexts(messages, turnStart)); + } + + [Theory] + [InlineData("call_1", 1)] + [InlineData("unrelated_call", 5)] + public void GetCurrentTurnStart_ResumedResultBeforeSeveralUserMessages_MatchesTheHistoricalCall(string resultCallId, int expectedTurnStart) + { + // Arrange + ChatMessage[] messages = + [ + new ChatMessage(ChatRole.User, "Refresh the data.").WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory), + new ChatMessage(ChatRole.Assistant, [new FunctionCallContent("call_1", "refresh")]).WithAgentRequestMessageSource(AgentRequestMessageSourceType.ChatHistory), + new(ChatRole.Tool, [new FunctionResultContent(resultCallId, "New result")]), + new(ChatRole.User, "Yes, go ahead."), + new(ChatRole.User, "Thanks."), + ]; + + // Act + int turnStart = JevChatState.GetCurrentTurnStart(messages); + + // Assert + Assert.Equal(expectedTurnStart, turnStart); + } + [Fact] public async Task RunStreamingAsync_ThrowsAsync() { diff --git a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs index e32f3fd8538..4c0442c78a5 100644 --- a/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.TypeSafe.UnitTests/JevToolCallCompilerTests.cs @@ -154,6 +154,13 @@ public void Compile_EnumAndConstValuesMatchingTheirType_Compile() [InlineData("""{ "type": "object", "properties": { "a": { "type": "string", "const": 5 } }, "required": ["a"] }""", "required argument 'a': const value 5 does not match declared type \"string\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "enum": [1, 2.0] } } }""", "enum value 2.0 does not match declared type \"integer\"")] [InlineData("""{ "type": "object", "properties": { "a": { "type": "integer", "const": 1e2 } }, "required": ["a"] }""", "const value 1e2 does not match declared type \"integer\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "const": "x", "enum": ["y"] } } }""", "const value \"x\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["allowed"], ["forbidden"]], "items": { "type": "string", "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["forbidden"]], "items": { "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "const": ["forbidden"], "items": { "enum": ["allowed"] } } } }""", "array member \"forbidden\" does not match declared enum")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "const": [1], "items": { "type": "string" } } } }""", "array member 1 does not match declared item type \"string\"")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "array", "enum": [["x"]], "items": { "type": "string", "maxLength": 3 } } } }""", "unsupported array item schema constraints: maxLength")] + [InlineData("""{ "type": "object", "properties": { "a": { "type": "boolean", "enum": [] } } }""", "enum must be a non-empty array")] [InlineData("""{ "type": "object", "properties": {}, "required": ["missing"] }""", "required arguments are missing from properties: missing")] [InlineData("""{ "type": "array" }""", "the top-level input schema must be an object")] public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema, string reason) @@ -169,6 +176,33 @@ public void Compile_UnsupportedSchema_ExcludesTheToolWithTheReason(string schema Assert.Contains(reason, Assert.Single(logger.Entries).Message, StringComparison.Ordinal); } + [Fact] + public void Compile_ConstAndEnumWithArrayItemConstraints_ResolveAndDecode() + { + // Arrange + AIFunctionDeclaration tool = Declaration("set_tags", """ + { + "type": "object", + "properties": { + "fixed": { "type": "array", "const": ["x"], "enum": [["x"], ["y"]], "items": { "$ref": "#/$defs/Tag" } }, + "chosen": { "type": "array", "enum": [["x"], ["x", "y"]], "items": { "$ref": "#/$defs/Tag" } } + }, + "required": ["fixed", "chosen"], + "$defs": { "Tag": { "type": "string", "enum": ["x", "y"] } } + } + """); + + // Act + JevToolCallPlan plan = Compile([tool])!; + JevToolCall? call = plan.Decode(Result( + JevChatTestData.RouteTo("t0", "t0", "none"), + JevChatTestData.Choice("__af_tool__.t0.a1.value", "v1", "v0", "v1"))); + + // Assert + Assert.Equal(["x"], ((JsonElement)call!.Value.Arguments["fixed"]!).EnumerateArray().Select(item => item.GetString())); + Assert.Equal(["x", "y"], ((JsonElement)call.Value.Arguments["chosen"]!).EnumerateArray().Select(item => item.GetString())); + } + [Fact] public void Compile_SingleValueEnumAndNoArguments_NeedNoArgumentQuestions() { @@ -222,6 +256,37 @@ public void Compile_TooManyInternalQuestions_Throws() Assert.Contains("128", exception.Message, StringComparison.Ordinal); } + [Fact] + public void Compile_QuestionBudget_IncludesTheRouteQuestion() + { + // Arrange + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string array = "{ \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; + AIFunctionDeclaration tool = Declaration("large", "{ \"type\": \"object\", \"properties\": { \"a\": " + array + ", \"b\": " + array + " }, \"required\": [\"a\", \"b\"] }"); + + // Act & Assert + JevToolCallPlan plan = Compile([tool], ChatToolMode.RequireAny)!; + Assert.Equal(128, plan.Questions.Count); + Assert.Null(plan.RouteQuestionId); + Assert.Throws(() => Compile([tool])); + } + + [Fact] + public void Compile_QuestionBudgetWithRouteAtTheLimit_Compiles() + { + // Arrange + string first = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string second = string.Join(",", Enumerable.Range(0, 63).Select(index => $"\"n{index}\"")); + string schema = "{ \"type\": \"object\", \"properties\": { \"a\": { \"type\": \"array\", \"items\": { \"enum\": [" + first + "] } }, \"b\": { \"type\": \"array\", \"items\": { \"enum\": [" + second + "] } } }, \"required\": [\"a\", \"b\"] }"; + + // Act + JevToolCallPlan plan = Compile([Declaration("large", schema)])!; + + // Assert + Assert.Equal(128, plan.Questions.Count); + Assert.Equal(JevChatTestData.Route, plan.RouteQuestionId); + } + [Fact] public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() { @@ -237,6 +302,47 @@ public void Compile_UnsupportedTool_ReturnsItsQuestionBudget() Assert.Equal(125, plan.Questions.Count); } + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void Compile_UnsupportedToolExceedingQuestionBudget_IsExcludedRegardlessOfPropertyOrder(bool unsupportedFirst, bool textFirst) + { + // Arrange + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string array = "\"tags\": { \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; + string properties = textFirst ? "\"text\": { \"type\": \"string\" }, " + array : array + ", \"text\": { \"type\": \"string\" }"; + AIFunctionDeclaration supported = Declaration("supported", "{ \"type\": \"object\", \"properties\": { " + array + " }, \"required\": [\"tags\"] }"); + AIFunctionDeclaration unsupported = Declaration("unsupported", "{ \"type\": \"object\", \"properties\": { " + properties + " } }"); + var logger = new ListLoggerFactory(); + + // Act + JevToolCallPlan plan = Compile(unsupportedFirst ? [unsupported, supported] : [supported, unsupported], logger: logger.CreateLogger("test"))!; + + // Assert + Assert.Equal("supported", Assert.Single(plan.Tools).Function.Name); + Assert.Equal(65, plan.Questions.Count); + Assert.Contains("argument 'text'", Assert.Single(logger.Entries).Message, StringComparison.Ordinal); + } + + [Fact] + public void Compile_SingleUnsupportedToolExceedingQuestionBudget_IsExcluded() + { + // Arrange + string members = string.Join(",", Enumerable.Range(0, 64).Select(index => $"\"m{index}\"")); + string array = "{ \"type\": \"array\", \"items\": { \"enum\": [" + members + "] } }"; + AIFunctionDeclaration tool = Declaration("unsupported", "{ \"type\": \"object\", \"properties\": { \"a\": " + array + ", \"b\": " + array + ", \"text\": { \"type\": \"string\" } } }"); + var logger = new ListLoggerFactory(); + + // Act + JevToolCallPlan? plan = Compile([tool], logger: logger.CreateLogger("test")); + + // Assert + Assert.Null(plan); + Assert.Contains("argument 'text'", Assert.Single(logger.Entries).Message, StringComparison.Ordinal); + } + [Fact] public void Compile_TooManyEnumValuesOrProperties_ExcludeTheTool() {