diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs index 398dc061565..37911800b60 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs @@ -202,14 +202,22 @@ protected override async Task RunCoreAsync( if (iteration >= this._maxIterations) { this.LogMaxIterationsReached(iteration); - return this.BuildResult(response, transcript, aggregatedUsage); + AgentResponse capResult = this.BuildResult(response, transcript, aggregatedUsage); + (capResult.AdditionalProperties ??= new())[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.IterationCapReached; + return capResult; } // Ask the evaluators whether to continue; stop when none of them request a re-invocation. LoopNextStep step = await this.EvaluateAndBuildNextAsync(context, feedbackLog, initialSessionSnapshot, cancellationToken).ConfigureAwait(false); if (!step.ShouldContinue) { - return this.BuildResult(response, transcript, aggregatedUsage); + AgentResponse evalResult = this.BuildResult(response, transcript, aggregatedUsage); + if (context.AdditionalProperties.TryGetValue(LoopExitReason.AdditionalPropertiesKey, out object? exitReason)) + { + (evalResult.AdditionalProperties ??= new())[LoopExitReason.AdditionalPropertiesKey] = exitReason; + } + + return evalResult; } currentMessages = step.Messages; @@ -332,6 +340,9 @@ protected override async IAsyncEnumerable RunCoreStreamingA /// private async ValueTask EvaluateAndBuildNextAsync(LoopContext context, List feedbackLog, JsonElement? initialSessionSnapshot, CancellationToken cancellationToken) { + // Clear any exit reason stamped during an earlier pass so only the evaluators of the final iteration can supply it. + context.AdditionalProperties.Remove(LoopExitReason.AdditionalPropertiesKey); + // Evaluate in order; the first evaluator that requests a re-invocation wins. LoopEvaluation? winner = null; foreach (LoopEvaluator evaluator in this._evaluators) diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs index 9e1872ce8f1..e1fed678c13 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs @@ -20,7 +20,8 @@ namespace Microsoft.Agents.AI; /// /// /// Out-of-the-box implementations include , , -/// , and . Implementations should be stateless and safe to share across +/// , , and +/// . Implementations should be stateless and safe to share across /// concurrent loop runs; any per-run state must be stored on the supplied . /// /// diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs new file mode 100644 index 00000000000..55af624e596 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI; + +/// +/// Provides typed exit-reason strings for runs. +/// Values are plain strings to remain JSON-transparent. The reason is surfaced under +/// in AgentResponse.AdditionalProperties. +/// +/// +/// +/// Exit reasons are stamped by on non-streaming runs only. +/// In streaming mode the response is assembled from yielded updates by the caller, so there is no +/// single AgentResponse object available inside the generator to stamp. +/// +/// +/// Evaluators such as stamp the exit reason on +/// before returning Stop(); the loop then +/// copies that value to the returned AgentResponse.AdditionalProperties. +/// +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public static class LoopExitReason +{ + /// The key used in AgentResponse.AdditionalProperties to surface the loop exit reason. + public const string AdditionalPropertiesKey = "loop_exit_reason"; + + /// A TASK_COMPLETE: signal was detected in the agent's response. + public const string Completed = "completed"; + + /// The max-iterations cap fired before any terminal signal was emitted. + public const string IterationCapReached = "iteration_cap_reached"; + + /// A NEED_INPUT: signal was detected; the loop stopped to surface a question. + public const string NeedInput = "need_input"; + + /// External cancellation (reserved for future use). + public const string Cancelled = "cancelled"; +} diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs new file mode 100644 index 00000000000..a7160f69370 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// A implementing the GT1 signal protocol: the loop stops when an assistant message in the +/// agent's latest response contains or at the start of a +/// line, and otherwise continues with feedback reminding the agent to emit one of the signals. +/// +/// +/// +/// A token only counts when it starts its own line: it must appear at the start of the message or right after a line +/// break, optionally preceded by spaces or tabs. Mentions anywhere else on a line (for example +/// "I'll emit TASK_COMPLETE: when done", or a token quoted in backticks) are ignored, so the agent can talk about +/// the protocol without accidentally ending the loop. +/// +/// +/// takes priority over . When a signal is detected, the +/// matching is stamped into under +/// . +/// +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class SignalProtocolLoopEvaluator : LoopEvaluator +{ + /// The token signalling that the task is complete. Only recognized at the start of a line. + public const string TaskCompleteToken = "TASK_COMPLETE:"; + + /// The token signalling that the agent needs input from the user. Only recognized at the start of a line. + public const string NeedInputToken = "NEED_INPUT:"; + + /// The default feedback produced while neither signal is present. + public const string DefaultFeedbackMessage = + "Continue working on the task. When done, start a new line with '" + TaskCompleteToken + " '. " + + "If you need input, start a new line with '" + NeedInputToken + " '."; + + private readonly string _feedbackMessage; + + /// + /// Initializes a new instance of the class. + /// + /// Optional custom feedback used when no signal is present. When , is used. + public SignalProtocolLoopEvaluator(string? feedbackMessage = null) + { + this._feedbackMessage = feedbackMessage ?? DefaultFeedbackMessage; + } + + /// + public override ValueTask EvaluateAsync(LoopContext context, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(context); + + if (AssistantMessageHasSignal(context, TaskCompleteToken)) + { + context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.Completed; + return new ValueTask(LoopEvaluation.Stop()); + } + + if (AssistantMessageHasSignal(context, NeedInputToken)) + { + context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.NeedInput; + return new ValueTask(LoopEvaluation.Stop()); + } + + return new ValueTask(LoopEvaluation.Continue(this._feedbackMessage)); + } + + /// + /// Determines whether any assistant-role message in the latest response contains at the + /// start of a line. Messages are matched individually so a token cannot be formed across message boundaries, and + /// non-assistant messages (for example tool results or echoed user input) never trigger a signal. + /// + private static bool AssistantMessageHasSignal(LoopContext context, string token) + { + foreach (ChatMessage message in context.LastResponse.Messages) + { + if (message.Role == ChatRole.Assistant && ContainsTokenAtLineStart(message.Text, token)) + { + return true; + } + } + + return false; + } + + /// + /// Determines whether contains at the start of a line, allowing only + /// spaces and tabs between the preceding line break (or the start of the text) and the token. + /// + private static bool ContainsTokenAtLineStart(string text, string token) + { + int index = text.IndexOf(token, StringComparison.Ordinal); + while (index >= 0) + { + int i = index - 1; + while (i >= 0 && (text[i] == ' ' || text[i] == '\t')) + { + i--; + } + + // Checking for '\n' covers both "\n" and "\r\n" line endings. + if (i < 0 || text[i] == '\n') + { + return true; + } + + index = text.IndexOf(token, index + token.Length, StringComparison.Ordinal); + } + + return false; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt index fada975c025..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt index fada975c025..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt index fada975c025..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt index fada975c025..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt index fada975c025..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs index b6161da73a0..6cb73e7116a 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs @@ -336,6 +336,36 @@ public async Task RunAsync_AlwaysContinue_StopsAtGlobalCapAsync() // Assert Assert.Equal(3, capture.CallCount); Assert.Equal(["working", "working", "working"], response.Messages.Select(static m => m.Text)); + Assert.NotNull(response.AdditionalProperties); + Assert.Equal(LoopExitReason.IterationCapReached, response.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that an exit reason stamped during an earlier iteration is not copied onto the final response when a later + /// iteration stops without any evaluator supplying a reason. + /// + [Fact] + public async Task RunAsync_StaleExitReasonFromEarlierIteration_IsNotReturnedAsync() + { + // Arrange + // Iteration 1 emits a signal (stamping "completed") but a second evaluator still requests another iteration. + // Iteration 2 is signal-free and every evaluator stops, so no exit reason should be reported. + var capture = new InnerAgentCapture(call => new AgentResponse([new ChatMessage(ChatRole.Assistant, call == 1 ? "TASK_COMPLETE: done" : "working")])); + var signal = new SignalProtocolLoopEvaluator(); + var signalOrStop = new DelegateLoopEvaluator(async (context, ct) => + { + LoopEvaluation evaluation = await signal.EvaluateAsync(context, ct); + return evaluation.ShouldReinvoke ? LoopEvaluation.Stop() : evaluation; + }); + var evaluators = new LoopEvaluator[] { signalOrStop, While(static context => context.Iteration == 1) }; + var agent = new LoopAgent(capture.Agent, evaluators); + + // Act + var response = await agent.RunAsync([new ChatMessage(ChatRole.User, "go")], new ChatClientAgentSession()); + + // Assert + Assert.Equal(2, capture.CallCount); + Assert.False(response.AdditionalProperties?.ContainsKey(LoopExitReason.AdditionalPropertiesKey) ?? false); } /// diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs new file mode 100644 index 00000000000..c7c17e0f0cd --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs @@ -0,0 +1,398 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Moq; + +namespace Microsoft.Agents.AI.UnitTests; + +/// +/// Unit tests for the class. +/// +public class SignalProtocolLoopEvaluatorTests +{ + /// + /// Verify that the evaluator stops when TASK_COMPLETE: is present. + /// + [Fact] + public async Task EvaluateAsync_TaskCompletePresent_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("TASK_COMPLETE: all done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + } + + /// + /// Verify that the evaluator stops when NEED_INPUT: is present. + /// + [Fact] + public async Task EvaluateAsync_NeedInputPresent_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: which file?"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + } + + /// + /// Verify that the evaluator continues with feedback when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_NoSignal_ContinuesWithFeedbackAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("still working"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.NotNull(evaluation.Feedback); + } + + /// + /// Verify that TASK_COMPLETE: takes priority over NEED_INPUT: when both are present. + /// + [Fact] + public async Task EvaluateAsync_TaskCompleteWinsOverNeedInput_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: q?\nTASK_COMPLETE: done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.Completed, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that TASK_COMPLETE: stamps the completed exit reason. + /// + [Fact] + public async Task EvaluateAsync_TaskComplete_StampsCompletedReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("TASK_COMPLETE: done"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.Equal("completed", context.AdditionalProperties["loop_exit_reason"]); + } + + /// + /// Verify that NEED_INPUT: stamps the need_input exit reason. + /// + [Fact] + public async Task EvaluateAsync_NeedInput_StampsNeedInputReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: which file?"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.Equal("need_input", context.AdditionalProperties["loop_exit_reason"]); + } + + /// + /// Verify that no exit reason is stamped when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_NoSignal_DoesNotStampReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("still working"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(context.AdditionalProperties.ContainsKey("loop_exit_reason")); + } + + /// + /// Verify that a custom feedback message is used when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_CustomFeedback_UsedWhenNoSignalAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator("keep going"); + LoopContext context = CreateContext("still working"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.Equal("keep going", evaluation.Feedback); + } + + /// + /// Verify that signal tokens in non-assistant messages (tool results, user input) do not stop the loop. + /// + [Fact] + public async Task EvaluateAsync_TokenOnlyInNonAssistantMessages_ContinuesAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext( + [ + new ChatMessage(ChatRole.Tool, "TASK_COMPLETE: from tool output"), + new ChatMessage(ChatRole.User, "NEED_INPUT: echoed user text"), + new ChatMessage(ChatRole.Assistant, "still working"), + ]); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.False(context.AdditionalProperties.ContainsKey(LoopExitReason.AdditionalPropertiesKey)); + } + + /// + /// Verify that a token split across two assistant messages is not treated as a signal. + /// + [Fact] + public async Task EvaluateAsync_TokenSplitAcrossAssistantMessages_ContinuesAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext( + [ + new ChatMessage(ChatRole.Assistant, "TASK_"), + new ChatMessage(ChatRole.Assistant, "COMPLETE: done"), + ]); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + } + + /// + /// Verify that a signal in an assistant message is detected alongside non-assistant messages. + /// + [Fact] + public async Task EvaluateAsync_TokenInAssistantMessageAmongOthers_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext( + [ + new ChatMessage(ChatRole.Tool, "tool output"), + new ChatMessage(ChatRole.Assistant, "NEED_INPUT: which file?"), + ]); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.NeedInput, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that a token mentioned mid-sentence is not treated as a signal. + /// + [Theory] + [InlineData("I'll emit TASK_COMPLETE: when done.")] + [InlineData("If anything is unclear I will reply with NEED_INPUT: and a question.")] + public async Task EvaluateAsync_TokenMidSentence_ContinuesAsync(string text) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.False(context.AdditionalProperties.ContainsKey(LoopExitReason.AdditionalPropertiesKey)); + } + + /// + /// Verify that a token quoted in backticks, even at the start of a line, is not treated as a signal. + /// + [Theory] + [InlineData("`TASK_COMPLETE:` is the completion signal.")] + [InlineData("Protocol:\n`NEED_INPUT:` asks the user a question.")] + [InlineData("I'll emit `TASK_COMPLETE:` when done.")] + [InlineData("I'll emit `NEED_INPUT:` if I get stuck.")] + public async Task EvaluateAsync_TokenInBackticks_ContinuesAsync(string text) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.False(context.AdditionalProperties.ContainsKey(LoopExitReason.AdditionalPropertiesKey)); + } + + /// + /// Verify that a token at the start of a later line, after prose, stops the loop. + /// + [Theory] + [InlineData("I updated the file and ran the tests.\nTASK_COMPLETE: all tests pass", LoopExitReason.Completed)] + [InlineData("I found two config files.\nNEED_INPUT: which one should I edit?", LoopExitReason.NeedInput)] + public async Task EvaluateAsync_TokenAtStartOfLaterLine_StopsAsync(string text, string expectedReason) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(expectedReason, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that leading spaces or tabs before a token on its line are allowed. + /// + [Theory] + [InlineData(" TASK_COMPLETE: done", LoopExitReason.Completed)] + [InlineData("Summary:\n\t NEED_INPUT: which file?", LoopExitReason.NeedInput)] + public async Task EvaluateAsync_TokenWithLeadingWhitespace_StopsAsync(string text, string expectedReason) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(expectedReason, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that a token at the start of a line is detected with CRLF line endings. + /// + [Fact] + public async Task EvaluateAsync_TokenAfterCrLf_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("Done with the work.\r\nTASK_COMPLETE: summary\r\n"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.Completed, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that a mid-sentence mention earlier in a message does not hide a real signal later in the same message. + /// + [Fact] + public async Task EvaluateAsync_MentionThenRealSignal_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("I was asked to emit TASK_COMPLETE: when done.\nTASK_COMPLETE: done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.Completed, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that a mid-sentence TASK_COMPLETE: mention does not take priority over a real NEED_INPUT: signal. + /// + [Fact] + public async Task EvaluateAsync_TaskCompleteMentionWithNeedInputSignal_StampsNeedInputAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("Before I can emit TASK_COMPLETE: I need to know more.\nNEED_INPUT: which file?"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.NeedInput, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that EvaluateAsync throws when the context is null. + /// + [Fact] + public async Task EvaluateAsync_NullContext_ThrowsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + + // Act & Assert + await Assert.ThrowsAsync("context", async () => await evaluator.EvaluateAsync(null!)); + } + + /// + /// Verify the TASK_COMPLETE token constant value. + /// + [Fact] + public void TaskCompleteToken_IsExpectedValue() + { + // Assert + Assert.Equal("TASK_COMPLETE:", SignalProtocolLoopEvaluator.TaskCompleteToken); + } + + /// + /// Verify the NEED_INPUT token constant value. + /// + [Fact] + public void NeedInputToken_IsExpectedValue() + { + // Assert + Assert.Equal("NEED_INPUT:", SignalProtocolLoopEvaluator.NeedInputToken); + } + + private static LoopContext CreateContext(string responseText) => + CreateContext([new ChatMessage(ChatRole.Assistant, responseText)]); + + private static LoopContext CreateContext(List responseMessages) => new( + new Mock().Object, + new ChatClientAgentSession(), + [new ChatMessage(ChatRole.User, "go")], + new AgentResponse(responseMessages)); +}