From d64b47c3545fecae64517df63ceef25f6b7d0397 Mon Sep 17 00:00:00 2001 From: Mark Licata Date: Wed, 7 Oct 2026 08:33:48 -0700 Subject: [PATCH 1/3] .NET: add SignalProtocolLoopEvaluator for LoopAgent Co-Authored-By: Claude Opus 5.5 --- .../Harness/Loop/LoopAgent.cs | 12 +- .../Harness/Loop/LoopEvaluator.cs | 3 +- .../Harness/Loop/LoopExitReason.cs | 42 ++++ .../Loop/SignalProtocolLoopEvaluator.cs | 68 +++++++ .../PublicAPI/net10.0/PublicAPI.Unshipped.txt | 12 ++ .../PublicAPI/net472/PublicAPI.Unshipped.txt | 12 ++ .../PublicAPI/net8.0/PublicAPI.Unshipped.txt | 12 ++ .../PublicAPI/net9.0/PublicAPI.Unshipped.txt | 12 ++ .../netstandard2.0/PublicAPI.Unshipped.txt | 12 ++ .../Loop/SignalProtocolLoopEvaluatorTests.cs | 192 ++++++++++++++++++ 10 files changed, 374 insertions(+), 3 deletions(-) create mode 100644 dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs create mode 100644 dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs create mode 100644 dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs index 398dc061565..e04be399151 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs @@ -202,14 +202,22 @@ protected override async Task RunCoreAsync( if (iteration >= this._maxIterations) { this.LogMaxIterationsReached(iteration); - return this.BuildResult(response, transcript, aggregatedUsage); + AgentResponse capResult = this.BuildResult(response, transcript, aggregatedUsage); + (capResult.AdditionalProperties ??= new())[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.IterationCapReached; + return capResult; } // Ask the evaluators whether to continue; stop when none of them request a re-invocation. LoopNextStep step = await this.EvaluateAndBuildNextAsync(context, feedbackLog, initialSessionSnapshot, cancellationToken).ConfigureAwait(false); if (!step.ShouldContinue) { - return this.BuildResult(response, transcript, aggregatedUsage); + AgentResponse evalResult = this.BuildResult(response, transcript, aggregatedUsage); + if (context.AdditionalProperties.TryGetValue(LoopExitReason.AdditionalPropertiesKey, out object? exitReason)) + { + (evalResult.AdditionalProperties ??= new())[LoopExitReason.AdditionalPropertiesKey] = exitReason; + } + + return evalResult; } currentMessages = step.Messages; diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs index 9e1872ce8f1..e1fed678c13 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs @@ -20,7 +20,8 @@ namespace Microsoft.Agents.AI; /// /// /// Out-of-the-box implementations include , , -/// , and . Implementations should be stateless and safe to share across +/// , , and +/// . Implementations should be stateless and safe to share across /// concurrent loop runs; any per-run state must be stored on the supplied . /// /// diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs new file mode 100644 index 00000000000..55af624e596 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI; + +/// +/// Provides typed exit-reason strings for runs. +/// Values are plain strings to remain JSON-transparent. The reason is surfaced under +/// in AgentResponse.AdditionalProperties. +/// +/// +/// +/// Exit reasons are stamped by on non-streaming runs only. +/// In streaming mode the response is assembled from yielded updates by the caller, so there is no +/// single AgentResponse object available inside the generator to stamp. +/// +/// +/// Evaluators such as stamp the exit reason on +/// before returning Stop(); the loop then +/// copies that value to the returned AgentResponse.AdditionalProperties. +/// +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public static class LoopExitReason +{ + /// The key used in AgentResponse.AdditionalProperties to surface the loop exit reason. + public const string AdditionalPropertiesKey = "loop_exit_reason"; + + /// A TASK_COMPLETE: signal was detected in the agent's response. + public const string Completed = "completed"; + + /// The max-iterations cap fired before any terminal signal was emitted. + public const string IterationCapReached = "iteration_cap_reached"; + + /// A NEED_INPUT: signal was detected; the loop stopped to surface a question. + public const string NeedInput = "need_input"; + + /// External cancellation (reserved for future use). + public const string Cancelled = "cancelled"; +} diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs new file mode 100644 index 00000000000..f4e18d929e7 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// A implementing the GT1 signal protocol: the loop stops when the agent's latest response +/// contains or , and otherwise continues with feedback +/// reminding the agent to emit one of the signals. +/// +/// +/// takes priority over . When a signal is detected, the +/// matching is stamped into under +/// . +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class SignalProtocolLoopEvaluator : LoopEvaluator +{ + /// The token signalling that the task is complete. + public const string TaskCompleteToken = "TASK_COMPLETE:"; + + /// The token signalling that the agent needs input from the user. + public const string NeedInputToken = "NEED_INPUT:"; + + /// The default feedback produced while neither signal is present. + public const string DefaultFeedbackMessage = + "Continue working on the task. When done, emit '" + TaskCompleteToken + " '. " + + "If you need input, emit '" + NeedInputToken + " '."; + + private readonly string _feedbackMessage; + + /// + /// Initializes a new instance of the class. + /// + /// Optional custom feedback used when no signal is present. When , is used. + public SignalProtocolLoopEvaluator(string? feedbackMessage = null) + { + this._feedbackMessage = feedbackMessage ?? DefaultFeedbackMessage; + } + + /// + public override ValueTask EvaluateAsync(LoopContext context, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(context); + + string text = context.LastResponse.Text; + + if (text.Contains(TaskCompleteToken, StringComparison.Ordinal)) + { + context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.Completed; + return new ValueTask(LoopEvaluation.Stop()); + } + + if (text.Contains(NeedInputToken, StringComparison.Ordinal)) + { + context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.NeedInput; + return new ValueTask(LoopEvaluation.Stop()); + } + + return new ValueTask(LoopEvaluation.Continue(this._feedbackMessage)); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs new file mode 100644 index 00000000000..fb67281310f --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Moq; + +namespace Microsoft.Agents.AI.UnitTests; + +/// +/// Unit tests for the class. +/// +public class SignalProtocolLoopEvaluatorTests +{ + /// + /// Verify that the evaluator stops when TASK_COMPLETE: is present. + /// + [Fact] + public async Task EvaluateAsync_TaskCompletePresent_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("TASK_COMPLETE: all done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + } + + /// + /// Verify that the evaluator stops when NEED_INPUT: is present. + /// + [Fact] + public async Task EvaluateAsync_NeedInputPresent_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: which file?"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + } + + /// + /// Verify that the evaluator continues with feedback when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_NoSignal_ContinuesWithFeedbackAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("still working"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.NotNull(evaluation.Feedback); + } + + /// + /// Verify that TASK_COMPLETE: takes priority over NEED_INPUT: when both are present. + /// + [Fact] + public async Task EvaluateAsync_TaskCompleteWinsOverNeedInput_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: q? TASK_COMPLETE: done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.Completed, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that TASK_COMPLETE: stamps the completed exit reason. + /// + [Fact] + public async Task EvaluateAsync_TaskComplete_StampsCompletedReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("TASK_COMPLETE: done"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.Equal("completed", context.AdditionalProperties["loop_exit_reason"]); + } + + /// + /// Verify that NEED_INPUT: stamps the need_input exit reason. + /// + [Fact] + public async Task EvaluateAsync_NeedInput_StampsNeedInputReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: which file?"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.Equal("need_input", context.AdditionalProperties["loop_exit_reason"]); + } + + /// + /// Verify that no exit reason is stamped when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_NoSignal_DoesNotStampReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("still working"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(context.AdditionalProperties.ContainsKey("loop_exit_reason")); + } + + /// + /// Verify that a custom feedback message is used when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_CustomFeedback_UsedWhenNoSignalAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator("keep going"); + LoopContext context = CreateContext("still working"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.Equal("keep going", evaluation.Feedback); + } + + /// + /// Verify that EvaluateAsync throws when the context is null. + /// + [Fact] + public async Task EvaluateAsync_NullContext_ThrowsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + + // Act & Assert + await Assert.ThrowsAsync("context", async () => await evaluator.EvaluateAsync(null!)); + } + + /// + /// Verify the TASK_COMPLETE token constant value. + /// + [Fact] + public void TaskCompleteToken_IsExpectedValue() + { + // Assert + Assert.Equal("TASK_COMPLETE:", SignalProtocolLoopEvaluator.TaskCompleteToken); + } + + /// + /// Verify the NEED_INPUT token constant value. + /// + [Fact] + public void NeedInputToken_IsExpectedValue() + { + // Assert + Assert.Equal("NEED_INPUT:", SignalProtocolLoopEvaluator.NeedInputToken); + } + + private static LoopContext CreateContext(string responseText) => new( + new Mock().Object, + new ChatClientAgentSession(), + [new ChatMessage(ChatRole.User, "go")], + new AgentResponse([new ChatMessage(ChatRole.Assistant, responseText)])); +} From b245466d5feedf628597bd9a722387383e3abc2f Mon Sep 17 00:00:00 2001 From: Mark Licata Date: Wed, 7 Oct 2026 09:06:23 -0700 Subject: [PATCH 2/3] Address review feedback: assistant-only signal matching, clear stale exit reason Co-Authored-By: Claude Opus 5.5 --- .../Harness/Loop/LoopAgent.cs | 3 + .../Loop/SignalProtocolLoopEvaluator.cs | 31 ++++++-- .../Harness/Loop/LoopAgentTests.cs | 30 ++++++++ .../Loop/SignalProtocolLoopEvaluatorTests.cs | 74 ++++++++++++++++++- 4 files changed, 129 insertions(+), 9 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs index e04be399151..37911800b60 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs @@ -340,6 +340,9 @@ protected override async IAsyncEnumerable RunCoreStreamingA /// private async ValueTask EvaluateAndBuildNextAsync(LoopContext context, List feedbackLog, JsonElement? initialSessionSnapshot, CancellationToken cancellationToken) { + // Clear any exit reason stamped during an earlier pass so only the evaluators of the final iteration can supply it. + context.AdditionalProperties.Remove(LoopExitReason.AdditionalPropertiesKey); + // Evaluate in order; the first evaluator that requests a re-invocation wins. LoopEvaluation? winner = null; foreach (LoopEvaluator evaluator in this._evaluators) diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs index f4e18d929e7..0a248a16df0 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs @@ -4,15 +4,16 @@ using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.AI; using Microsoft.Shared.DiagnosticIds; using Microsoft.Shared.Diagnostics; namespace Microsoft.Agents.AI; /// -/// A implementing the GT1 signal protocol: the loop stops when the agent's latest response -/// contains or , and otherwise continues with feedback -/// reminding the agent to emit one of the signals. +/// A implementing the GT1 signal protocol: the loop stops when an assistant message in the +/// agent's latest response contains or , and otherwise +/// continues with feedback reminding the agent to emit one of the signals. /// /// /// takes priority over . When a signal is detected, the @@ -49,15 +50,13 @@ public override ValueTask EvaluateAsync(LoopContext context, Can { _ = Throw.IfNull(context); - string text = context.LastResponse.Text; - - if (text.Contains(TaskCompleteToken, StringComparison.Ordinal)) + if (AssistantMessageContains(context, TaskCompleteToken)) { context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.Completed; return new ValueTask(LoopEvaluation.Stop()); } - if (text.Contains(NeedInputToken, StringComparison.Ordinal)) + if (AssistantMessageContains(context, NeedInputToken)) { context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.NeedInput; return new ValueTask(LoopEvaluation.Stop()); @@ -65,4 +64,22 @@ public override ValueTask EvaluateAsync(LoopContext context, Can return new ValueTask(LoopEvaluation.Continue(this._feedbackMessage)); } + + /// + /// Determines whether any assistant-role message in the latest response contains . Messages + /// are matched individually so a token cannot be formed across message boundaries, and non-assistant messages (for + /// example tool results or echoed user input) never trigger a signal. + /// + private static bool AssistantMessageContains(LoopContext context, string token) + { + foreach (ChatMessage message in context.LastResponse.Messages) + { + if (message.Role == ChatRole.Assistant && message.Text.Contains(token, StringComparison.Ordinal)) + { + return true; + } + } + + return false; + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs index b6161da73a0..6cb73e7116a 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/LoopAgentTests.cs @@ -336,6 +336,36 @@ public async Task RunAsync_AlwaysContinue_StopsAtGlobalCapAsync() // Assert Assert.Equal(3, capture.CallCount); Assert.Equal(["working", "working", "working"], response.Messages.Select(static m => m.Text)); + Assert.NotNull(response.AdditionalProperties); + Assert.Equal(LoopExitReason.IterationCapReached, response.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that an exit reason stamped during an earlier iteration is not copied onto the final response when a later + /// iteration stops without any evaluator supplying a reason. + /// + [Fact] + public async Task RunAsync_StaleExitReasonFromEarlierIteration_IsNotReturnedAsync() + { + // Arrange + // Iteration 1 emits a signal (stamping "completed") but a second evaluator still requests another iteration. + // Iteration 2 is signal-free and every evaluator stops, so no exit reason should be reported. + var capture = new InnerAgentCapture(call => new AgentResponse([new ChatMessage(ChatRole.Assistant, call == 1 ? "TASK_COMPLETE: done" : "working")])); + var signal = new SignalProtocolLoopEvaluator(); + var signalOrStop = new DelegateLoopEvaluator(async (context, ct) => + { + LoopEvaluation evaluation = await signal.EvaluateAsync(context, ct); + return evaluation.ShouldReinvoke ? LoopEvaluation.Stop() : evaluation; + }); + var evaluators = new LoopEvaluator[] { signalOrStop, While(static context => context.Iteration == 1) }; + var agent = new LoopAgent(capture.Agent, evaluators); + + // Act + var response = await agent.RunAsync([new ChatMessage(ChatRole.User, "go")], new ChatClientAgentSession()); + + // Assert + Assert.Equal(2, capture.CallCount); + Assert.False(response.AdditionalProperties?.ContainsKey(LoopExitReason.AdditionalPropertiesKey) ?? false); } /// diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs index fb67281310f..408139ebd92 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Extensions.AI; using Moq; @@ -151,6 +152,72 @@ public async Task EvaluateAsync_CustomFeedback_UsedWhenNoSignalAsync() Assert.Equal("keep going", evaluation.Feedback); } + /// + /// Verify that signal tokens in non-assistant messages (tool results, user input) do not stop the loop. + /// + [Fact] + public async Task EvaluateAsync_TokenOnlyInNonAssistantMessages_ContinuesAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext( + [ + new ChatMessage(ChatRole.Tool, "TASK_COMPLETE: from tool output"), + new ChatMessage(ChatRole.User, "NEED_INPUT: echoed user text"), + new ChatMessage(ChatRole.Assistant, "still working"), + ]); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.False(context.AdditionalProperties.ContainsKey(LoopExitReason.AdditionalPropertiesKey)); + } + + /// + /// Verify that a token split across two assistant messages is not treated as a signal. + /// + [Fact] + public async Task EvaluateAsync_TokenSplitAcrossAssistantMessages_ContinuesAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext( + [ + new ChatMessage(ChatRole.Assistant, "TASK_"), + new ChatMessage(ChatRole.Assistant, "COMPLETE: done"), + ]); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + } + + /// + /// Verify that a signal in an assistant message is detected alongside non-assistant messages. + /// + [Fact] + public async Task EvaluateAsync_TokenInAssistantMessageAmongOthers_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext( + [ + new ChatMessage(ChatRole.Tool, "tool output"), + new ChatMessage(ChatRole.Assistant, "NEED_INPUT: which file?"), + ]); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.NeedInput, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + /// /// Verify that EvaluateAsync throws when the context is null. /// @@ -184,9 +251,12 @@ public void NeedInputToken_IsExpectedValue() Assert.Equal("NEED_INPUT:", SignalProtocolLoopEvaluator.NeedInputToken); } - private static LoopContext CreateContext(string responseText) => new( + private static LoopContext CreateContext(string responseText) => + CreateContext([new ChatMessage(ChatRole.Assistant, responseText)]); + + private static LoopContext CreateContext(List responseMessages) => new( new Mock().Object, new ChatClientAgentSession(), [new ChatMessage(ChatRole.User, "go")], - new AgentResponse([new ChatMessage(ChatRole.Assistant, responseText)])); + new AgentResponse(responseMessages)); } From 4f96e758474325435ebbec3e9ff6f341a6697417 Mon Sep 17 00:00:00 2001 From: Mark Licata Date: Tue, 13 Oct 2026 09:35:55 -0700 Subject: [PATCH 3/3] Require signal tokens at start of line SignalProtocolLoopEvaluator now only recognizes TASK_COMPLETE: / NEED_INPUT: when the token starts a line of an assistant message (optionally preceded by spaces or tabs). Mid-sentence or backtick-quoted mentions such as "I'll emit `TASK_COMPLETE:` when done" no longer stop the loop. Matches the Python signal protocol (#9162). DefaultFeedbackMessage now tells the model to start a new line with the token; the PublicAPI.Unshipped.txt entries are updated because the analyzer tracks const values. Co-Authored-By: Claude Opus 5.5 --- .../Loop/SignalProtocolLoopEvaluator.cs | 61 ++++++-- .../PublicAPI/net10.0/PublicAPI.Unshipped.txt | 2 +- .../PublicAPI/net472/PublicAPI.Unshipped.txt | 2 +- .../PublicAPI/net8.0/PublicAPI.Unshipped.txt | 2 +- .../PublicAPI/net9.0/PublicAPI.Unshipped.txt | 2 +- .../netstandard2.0/PublicAPI.Unshipped.txt | 2 +- .../Loop/SignalProtocolLoopEvaluatorTests.cs | 138 +++++++++++++++++- 7 files changed, 190 insertions(+), 19 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs index 0a248a16df0..a7160f69370 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs @@ -12,27 +12,35 @@ namespace Microsoft.Agents.AI; /// /// A implementing the GT1 signal protocol: the loop stops when an assistant message in the -/// agent's latest response contains or , and otherwise -/// continues with feedback reminding the agent to emit one of the signals. +/// agent's latest response contains or at the start of a +/// line, and otherwise continues with feedback reminding the agent to emit one of the signals. /// /// +/// +/// A token only counts when it starts its own line: it must appear at the start of the message or right after a line +/// break, optionally preceded by spaces or tabs. Mentions anywhere else on a line (for example +/// "I'll emit TASK_COMPLETE: when done", or a token quoted in backticks) are ignored, so the agent can talk about +/// the protocol without accidentally ending the loop. +/// +/// /// takes priority over . When a signal is detected, the /// matching is stamped into under /// . +/// /// [Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] public sealed class SignalProtocolLoopEvaluator : LoopEvaluator { - /// The token signalling that the task is complete. + /// The token signalling that the task is complete. Only recognized at the start of a line. public const string TaskCompleteToken = "TASK_COMPLETE:"; - /// The token signalling that the agent needs input from the user. + /// The token signalling that the agent needs input from the user. Only recognized at the start of a line. public const string NeedInputToken = "NEED_INPUT:"; /// The default feedback produced while neither signal is present. public const string DefaultFeedbackMessage = - "Continue working on the task. When done, emit '" + TaskCompleteToken + " '. " + - "If you need input, emit '" + NeedInputToken + " '."; + "Continue working on the task. When done, start a new line with '" + TaskCompleteToken + " '. " + + "If you need input, start a new line with '" + NeedInputToken + " '."; private readonly string _feedbackMessage; @@ -50,13 +58,13 @@ public override ValueTask EvaluateAsync(LoopContext context, Can { _ = Throw.IfNull(context); - if (AssistantMessageContains(context, TaskCompleteToken)) + if (AssistantMessageHasSignal(context, TaskCompleteToken)) { context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.Completed; return new ValueTask(LoopEvaluation.Stop()); } - if (AssistantMessageContains(context, NeedInputToken)) + if (AssistantMessageHasSignal(context, NeedInputToken)) { context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.NeedInput; return new ValueTask(LoopEvaluation.Stop()); @@ -66,15 +74,15 @@ public override ValueTask EvaluateAsync(LoopContext context, Can } /// - /// Determines whether any assistant-role message in the latest response contains . Messages - /// are matched individually so a token cannot be formed across message boundaries, and non-assistant messages (for - /// example tool results or echoed user input) never trigger a signal. + /// Determines whether any assistant-role message in the latest response contains at the + /// start of a line. Messages are matched individually so a token cannot be formed across message boundaries, and + /// non-assistant messages (for example tool results or echoed user input) never trigger a signal. /// - private static bool AssistantMessageContains(LoopContext context, string token) + private static bool AssistantMessageHasSignal(LoopContext context, string token) { foreach (ChatMessage message in context.LastResponse.Messages) { - if (message.Role == ChatRole.Assistant && message.Text.Contains(token, StringComparison.Ordinal)) + if (message.Role == ChatRole.Assistant && ContainsTokenAtLineStart(message.Text, token)) { return true; } @@ -82,4 +90,31 @@ private static bool AssistantMessageContains(LoopContext context, string token) return false; } + + /// + /// Determines whether contains at the start of a line, allowing only + /// spaces and tabs between the preceding line break (or the start of the text) and the token. + /// + private static bool ContainsTokenAtLineStart(string text, string token) + { + int index = text.IndexOf(token, StringComparison.Ordinal); + while (index >= 0) + { + int i = index - 1; + while (i >= 0 && (text[i] == ' ' || text[i] == '\t')) + { + i--; + } + + // Checking for '\n' covers both "\n" and "\r\n" line endings. + if (i < 0 || text[i] == '\n') + { + return true; + } + + index = text.IndexOf(token, index + token.Length, StringComparison.Ordinal); + } + + return false; + } } diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt index 44a683d0d7f..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt @@ -32,7 +32,7 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! -[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! [MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt index 44a683d0d7f..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt @@ -32,7 +32,7 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! -[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! [MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt index 44a683d0d7f..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt @@ -32,7 +32,7 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! -[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! [MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt index 44a683d0d7f..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt @@ -32,7 +32,7 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! -[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! [MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt index 44a683d0d7f..1e7e002e32d 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -32,7 +32,7 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! [MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! -[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, start a new line with 'TASK_COMPLETE: '. If you need input, start a new line with 'NEED_INPUT: '." -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! [MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! [MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs index 408139ebd92..c7c17e0f0cd 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs @@ -73,7 +73,7 @@ public async Task EvaluateAsync_TaskCompleteWinsOverNeedInput_StopsAsync() { // Arrange var evaluator = new SignalProtocolLoopEvaluator(); - LoopContext context = CreateContext("NEED_INPUT: q? TASK_COMPLETE: done"); + LoopContext context = CreateContext("NEED_INPUT: q?\nTASK_COMPLETE: done"); // Act LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); @@ -218,6 +218,142 @@ public async Task EvaluateAsync_TokenInAssistantMessageAmongOthers_StopsAsync() Assert.Equal(LoopExitReason.NeedInput, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); } + /// + /// Verify that a token mentioned mid-sentence is not treated as a signal. + /// + [Theory] + [InlineData("I'll emit TASK_COMPLETE: when done.")] + [InlineData("If anything is unclear I will reply with NEED_INPUT: and a question.")] + public async Task EvaluateAsync_TokenMidSentence_ContinuesAsync(string text) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.False(context.AdditionalProperties.ContainsKey(LoopExitReason.AdditionalPropertiesKey)); + } + + /// + /// Verify that a token quoted in backticks, even at the start of a line, is not treated as a signal. + /// + [Theory] + [InlineData("`TASK_COMPLETE:` is the completion signal.")] + [InlineData("Protocol:\n`NEED_INPUT:` asks the user a question.")] + [InlineData("I'll emit `TASK_COMPLETE:` when done.")] + [InlineData("I'll emit `NEED_INPUT:` if I get stuck.")] + public async Task EvaluateAsync_TokenInBackticks_ContinuesAsync(string text) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.False(context.AdditionalProperties.ContainsKey(LoopExitReason.AdditionalPropertiesKey)); + } + + /// + /// Verify that a token at the start of a later line, after prose, stops the loop. + /// + [Theory] + [InlineData("I updated the file and ran the tests.\nTASK_COMPLETE: all tests pass", LoopExitReason.Completed)] + [InlineData("I found two config files.\nNEED_INPUT: which one should I edit?", LoopExitReason.NeedInput)] + public async Task EvaluateAsync_TokenAtStartOfLaterLine_StopsAsync(string text, string expectedReason) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(expectedReason, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that leading spaces or tabs before a token on its line are allowed. + /// + [Theory] + [InlineData(" TASK_COMPLETE: done", LoopExitReason.Completed)] + [InlineData("Summary:\n\t NEED_INPUT: which file?", LoopExitReason.NeedInput)] + public async Task EvaluateAsync_TokenWithLeadingWhitespace_StopsAsync(string text, string expectedReason) + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext(text); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(expectedReason, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that a token at the start of a line is detected with CRLF line endings. + /// + [Fact] + public async Task EvaluateAsync_TokenAfterCrLf_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("Done with the work.\r\nTASK_COMPLETE: summary\r\n"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.Completed, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that a mid-sentence mention earlier in a message does not hide a real signal later in the same message. + /// + [Fact] + public async Task EvaluateAsync_MentionThenRealSignal_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("I was asked to emit TASK_COMPLETE: when done.\nTASK_COMPLETE: done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.Completed, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that a mid-sentence TASK_COMPLETE: mention does not take priority over a real NEED_INPUT: signal. + /// + [Fact] + public async Task EvaluateAsync_TaskCompleteMentionWithNeedInputSignal_StampsNeedInputAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("Before I can emit TASK_COMPLETE: I need to know more.\nNEED_INPUT: which file?"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.NeedInput, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + /// /// Verify that EvaluateAsync throws when the context is null. ///