diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs index 398dc061565..e04be399151 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopAgent.cs @@ -202,14 +202,22 @@ protected override async Task RunCoreAsync( if (iteration >= this._maxIterations) { this.LogMaxIterationsReached(iteration); - return this.BuildResult(response, transcript, aggregatedUsage); + AgentResponse capResult = this.BuildResult(response, transcript, aggregatedUsage); + (capResult.AdditionalProperties ??= new())[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.IterationCapReached; + return capResult; } // Ask the evaluators whether to continue; stop when none of them request a re-invocation. LoopNextStep step = await this.EvaluateAndBuildNextAsync(context, feedbackLog, initialSessionSnapshot, cancellationToken).ConfigureAwait(false); if (!step.ShouldContinue) { - return this.BuildResult(response, transcript, aggregatedUsage); + AgentResponse evalResult = this.BuildResult(response, transcript, aggregatedUsage); + if (context.AdditionalProperties.TryGetValue(LoopExitReason.AdditionalPropertiesKey, out object? exitReason)) + { + (evalResult.AdditionalProperties ??= new())[LoopExitReason.AdditionalPropertiesKey] = exitReason; + } + + return evalResult; } currentMessages = step.Messages; diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs index 9e1872ce8f1..e1fed678c13 100644 --- a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopEvaluator.cs @@ -20,7 +20,8 @@ namespace Microsoft.Agents.AI; /// /// /// Out-of-the-box implementations include , , -/// , and . Implementations should be stateless and safe to share across +/// , , and +/// . Implementations should be stateless and safe to share across /// concurrent loop runs; any per-run state must be stored on the supplied . /// /// diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs new file mode 100644 index 00000000000..55af624e596 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/LoopExitReason.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI; + +/// +/// Provides typed exit-reason strings for runs. +/// Values are plain strings to remain JSON-transparent. The reason is surfaced under +/// in AgentResponse.AdditionalProperties. +/// +/// +/// +/// Exit reasons are stamped by on non-streaming runs only. +/// In streaming mode the response is assembled from yielded updates by the caller, so there is no +/// single AgentResponse object available inside the generator to stamp. +/// +/// +/// Evaluators such as stamp the exit reason on +/// before returning Stop(); the loop then +/// copies that value to the returned AgentResponse.AdditionalProperties. +/// +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public static class LoopExitReason +{ + /// The key used in AgentResponse.AdditionalProperties to surface the loop exit reason. + public const string AdditionalPropertiesKey = "loop_exit_reason"; + + /// A TASK_COMPLETE: signal was detected in the agent's response. + public const string Completed = "completed"; + + /// The max-iterations cap fired before any terminal signal was emitted. + public const string IterationCapReached = "iteration_cap_reached"; + + /// A NEED_INPUT: signal was detected; the loop stopped to surface a question. + public const string NeedInput = "need_input"; + + /// External cancellation (reserved for future use). + public const string Cancelled = "cancelled"; +} diff --git a/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs new file mode 100644 index 00000000000..f4e18d929e7 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Harness/Loop/SignalProtocolLoopEvaluator.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics.CodeAnalysis; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Shared.DiagnosticIds; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Agents.AI; + +/// +/// A implementing the GT1 signal protocol: the loop stops when the agent's latest response +/// contains or , and otherwise continues with feedback +/// reminding the agent to emit one of the signals. +/// +/// +/// takes priority over . When a signal is detected, the +/// matching is stamped into under +/// . +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class SignalProtocolLoopEvaluator : LoopEvaluator +{ + /// The token signalling that the task is complete. + public const string TaskCompleteToken = "TASK_COMPLETE:"; + + /// The token signalling that the agent needs input from the user. + public const string NeedInputToken = "NEED_INPUT:"; + + /// The default feedback produced while neither signal is present. + public const string DefaultFeedbackMessage = + "Continue working on the task. When done, emit '" + TaskCompleteToken + " '. " + + "If you need input, emit '" + NeedInputToken + " '."; + + private readonly string _feedbackMessage; + + /// + /// Initializes a new instance of the class. + /// + /// Optional custom feedback used when no signal is present. When , is used. + public SignalProtocolLoopEvaluator(string? feedbackMessage = null) + { + this._feedbackMessage = feedbackMessage ?? DefaultFeedbackMessage; + } + + /// + public override ValueTask EvaluateAsync(LoopContext context, CancellationToken cancellationToken = default) + { + _ = Throw.IfNull(context); + + string text = context.LastResponse.Text; + + if (text.Contains(TaskCompleteToken, StringComparison.Ordinal)) + { + context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.Completed; + return new ValueTask(LoopEvaluation.Stop()); + } + + if (text.Contains(NeedInputToken, StringComparison.Ordinal)) + { + context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey] = LoopExitReason.NeedInput; + return new ValueTask(LoopEvaluation.Stop()); + } + + return new ValueTask(LoopEvaluation.Continue(this._feedbackMessage)); + } +} diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net10.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net472/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net8.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/net9.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt index fada975c025..44a683d0d7f 100644 --- a/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt +++ b/dotnet/src/Microsoft.Agents.AI/PublicAPI/netstandard2.0/PublicAPI.Unshipped.txt @@ -24,3 +24,15 @@ Microsoft.Agents.AI.AgentModeProviderOptions.DisableModeSetTool.set -> void [MAAI001]virtual Microsoft.Agents.AI.AgentFileStore.SearchAsync(string! directory, string! regexPattern, string? globPattern = null, bool recursive = false, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task!>! [MAAI001]Microsoft.Agents.AI.FunctionInvocationContextExtensions [MAAI001]static Microsoft.Agents.AI.FunctionInvocationContextExtensions.WrapWithPendingMiddleware(this Microsoft.Extensions.AI.FunctionInvocationContext! context, Microsoft.Extensions.AI.AIFunction! function) -> Microsoft.Extensions.AI.AIFunction! +[MAAI001]Microsoft.Agents.AI.LoopExitReason +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator +[MAAI001]Microsoft.Agents.AI.SignalProtocolLoopEvaluator.SignalProtocolLoopEvaluator(string? feedbackMessage = null) -> void +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.AdditionalPropertiesKey = "loop_exit_reason" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Cancelled = "cancelled" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.Completed = "completed" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.IterationCapReached = "iteration_cap_reached" -> string! +[MAAI001]const Microsoft.Agents.AI.LoopExitReason.NeedInput = "need_input" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.DefaultFeedbackMessage = "Continue working on the task. When done, emit 'TASK_COMPLETE: '. If you need input, emit 'NEED_INPUT: '." -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.NeedInputToken = "NEED_INPUT:" -> string! +[MAAI001]const Microsoft.Agents.AI.SignalProtocolLoopEvaluator.TaskCompleteToken = "TASK_COMPLETE:" -> string! +[MAAI001]override Microsoft.Agents.AI.SignalProtocolLoopEvaluator.EvaluateAsync(Microsoft.Agents.AI.LoopContext! context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs new file mode 100644 index 00000000000..fb67281310f --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/Harness/Loop/SignalProtocolLoopEvaluatorTests.cs @@ -0,0 +1,192 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using Moq; + +namespace Microsoft.Agents.AI.UnitTests; + +/// +/// Unit tests for the class. +/// +public class SignalProtocolLoopEvaluatorTests +{ + /// + /// Verify that the evaluator stops when TASK_COMPLETE: is present. + /// + [Fact] + public async Task EvaluateAsync_TaskCompletePresent_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("TASK_COMPLETE: all done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + } + + /// + /// Verify that the evaluator stops when NEED_INPUT: is present. + /// + [Fact] + public async Task EvaluateAsync_NeedInputPresent_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: which file?"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + } + + /// + /// Verify that the evaluator continues with feedback when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_NoSignal_ContinuesWithFeedbackAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("still working"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.NotNull(evaluation.Feedback); + } + + /// + /// Verify that TASK_COMPLETE: takes priority over NEED_INPUT: when both are present. + /// + [Fact] + public async Task EvaluateAsync_TaskCompleteWinsOverNeedInput_StopsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: q? TASK_COMPLETE: done"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(evaluation.ShouldReinvoke); + Assert.Equal(LoopExitReason.Completed, context.AdditionalProperties[LoopExitReason.AdditionalPropertiesKey]); + } + + /// + /// Verify that TASK_COMPLETE: stamps the completed exit reason. + /// + [Fact] + public async Task EvaluateAsync_TaskComplete_StampsCompletedReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("TASK_COMPLETE: done"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.Equal("completed", context.AdditionalProperties["loop_exit_reason"]); + } + + /// + /// Verify that NEED_INPUT: stamps the need_input exit reason. + /// + [Fact] + public async Task EvaluateAsync_NeedInput_StampsNeedInputReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("NEED_INPUT: which file?"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.Equal("need_input", context.AdditionalProperties["loop_exit_reason"]); + } + + /// + /// Verify that no exit reason is stamped when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_NoSignal_DoesNotStampReasonAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + LoopContext context = CreateContext("still working"); + + // Act + await evaluator.EvaluateAsync(context); + + // Assert + Assert.False(context.AdditionalProperties.ContainsKey("loop_exit_reason")); + } + + /// + /// Verify that a custom feedback message is used when no signal is present. + /// + [Fact] + public async Task EvaluateAsync_CustomFeedback_UsedWhenNoSignalAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator("keep going"); + LoopContext context = CreateContext("still working"); + + // Act + LoopEvaluation evaluation = await evaluator.EvaluateAsync(context); + + // Assert + Assert.True(evaluation.ShouldReinvoke); + Assert.Equal("keep going", evaluation.Feedback); + } + + /// + /// Verify that EvaluateAsync throws when the context is null. + /// + [Fact] + public async Task EvaluateAsync_NullContext_ThrowsAsync() + { + // Arrange + var evaluator = new SignalProtocolLoopEvaluator(); + + // Act & Assert + await Assert.ThrowsAsync("context", async () => await evaluator.EvaluateAsync(null!)); + } + + /// + /// Verify the TASK_COMPLETE token constant value. + /// + [Fact] + public void TaskCompleteToken_IsExpectedValue() + { + // Assert + Assert.Equal("TASK_COMPLETE:", SignalProtocolLoopEvaluator.TaskCompleteToken); + } + + /// + /// Verify the NEED_INPUT token constant value. + /// + [Fact] + public void NeedInputToken_IsExpectedValue() + { + // Assert + Assert.Equal("NEED_INPUT:", SignalProtocolLoopEvaluator.NeedInputToken); + } + + private static LoopContext CreateContext(string responseText) => new( + new Mock().Object, + new ChatClientAgentSession(), + [new ChatMessage(ChatRole.User, "go")], + new AgentResponse([new ChatMessage(ChatRole.Assistant, responseText)])); +} diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index f9b01907f22..021c8de5d20 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- **agent-framework-core**: Add GT1 Signal Protocol for `AgentLoopMiddleware` autonomous loops: `SignalParser`, `signal_should_continue()`, `LoopExitReason`, `get_loop_exit_reason()`, and the `TASK_COMPLETE:` / `NEED_INPUT:` text-signal convention. `AgentLoopMiddleware` now stamps `loop_exit_reason` in `AdditionalProperties` when the iteration cap fires. + ## [1.20.0] - 2026-10-02 ### Added diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 13361bbeaa9..3686222311d 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -303,6 +303,7 @@ The vector store API is experimental under the shared `VECTOR_STORES` feature ID - **`todos_remaining(*, looping_modes=None)`** / **`todos_remaining_message`** - Helper factories for todo-driven loops (the Python counterpart of .NET's `TodoCompletionLoopEvaluator`), designed for `create_harness_agent` but usable with any agent that registers a `TodoProvider` via `context_providers`. They resolve the `TodoProvider`/`AgentModeProvider` from the *running agent* (`agent.context_providers`, via `_resolve_context_provider`) rather than taking the provider as an argument, so they can be wired directly into `loop_should_continue`/`loop_next_message`. `todos_remaining` returns a `should_continue` predicate that loops while any todo is open; pass `looping_modes=[...]` to gate looping to specific operating modes (case-insensitive; honors the `AgentModeProvider`'s `source_id`/`available_modes`), `looping_modes=None` (default) applies in every mode, and an empty sequence raises `ValueError`. `todos_remaining_message` is a `next_message` callable that lists the still-open todo titles and tells the agent to finish them, returning `None` when the session/agent/provider is unavailable or nothing is open (in which case the middleware's default `None` handling applies: reuse the previous iteration's messages verbatim under the default `fresh_context=False`, or `DEFAULT_NEXT_MESSAGE` only when `fresh_context=True`). - **`background_tasks_running()`** / **`background_tasks_running_message`** - Helper factories for background-agent-driven loops, mirroring the `todos_remaining` pair. They resolve the `BackgroundAgentsProvider` from the *running agent* (`agent.context_providers`, via `_resolve_context_provider`) rather than taking the provider as an argument, so they can be wired directly into `create_harness_agent`'s `loop_should_continue`/`loop_next_message`. `background_tasks_running` returns a `should_continue` predicate that loops while the provider's persisted state shows any task with `status == RUNNING` (pair it with `max_iterations` so the loop is bounded even if a task's persisted status is never refreshed). `background_tasks_running_message` is a `next_message` callable that lists the still-running tasks (`# (): `) and tells the agent to wait for them to finish and retrieve their results, returning `None` when the session/agent/provider is unavailable or no task is running. - **Approval escape hatch** - `_has_pending_approval_request(result)` checks whether an iteration's response carries a pending tool-approval request (any content with `type == "function_approval_request"`). Both the streaming and non-streaming loops stop and return that response to the caller *before* evaluating `should_continue`/`max_iterations` or injecting `next_message`, so the loop is HITL-safe even when wrapped outermost around a `ToolApprovalMiddleware` (mirrors the C# `LoopAgent`'s `HasPendingApprovalRequests`). + - **Signal protocol** (`_harness/_signals.py`) - `SignalParser` and `signal_should_continue()` provide a drop-in `should_continue` predicate that stops the loop when the agent emits `TASK_COMPLETE: ` (sets exit reason `"completed"`) or `NEED_INPUT: ` (sets exit reason `"need_input"`). Only assistant-role messages are scanned; user-role nudges injected between iterations are ignored. `LoopExitReason` holds the typed string constants. `get_loop_exit_reason(response)` scans assistant text first, then falls back to `response.additional_properties["loop_exit_reason"]` (set by the middleware when the iteration cap fires). Signal tokens always win over the cap. `_loop.py` deliberately avoids importing `_signals.py` and stamps `"iteration_cap_reached"` as a string literal, keeping the modules decoupled. - **Harness integration** - `create_harness_agent` enables the loop when a `loop_should_continue` callable is passed; it prepends `AgentLoopMiddleware(loop_should_continue, max_iterations=loop_max_iterations, next_message=loop_next_message)` ahead of `ToolApprovalMiddleware` so the loop is the outermost middleware (each iteration is a full agent run including tool approval, and the escape hatch hands pending approvals back to the caller). `loop_next_message` and `loop_max_iterations` only take effect together with `loop_should_continue` (with no `loop_should_continue` there is no loop, so they are ignored); `loop_max_iterations` defaults to the loop's default cap (`None` → unbounded). ### Workflows (`_workflows/`) diff --git a/python/packages/core/agent_framework/__init__.py b/python/packages/core/agent_framework/__init__.py index 3953187578e..f366ad98682 100644 --- a/python/packages/core/agent_framework/__init__.py +++ b/python/packages/core/agent_framework/__init__.py @@ -148,6 +148,16 @@ "MemoryTopicRecord", ), "._harness._mode": ("DEFAULT_MODE_SOURCE_ID", "AgentModeProvider", "get_agent_mode", "set_agent_mode"), + "._harness._signals": ( + "LoopExitReason", + "LOOP_EXIT_REASON_KEY", + "LOOP_NEED_INPUT_KEY", + "NEED_INPUT_TOKEN", + "SignalParser", + "TASK_COMPLETE_TOKEN", + "get_loop_exit_reason", + "signal_should_continue", + ), "._harness._todo": ( "DEFAULT_TODO_SOURCE_ID", "TodoFileStore", @@ -424,11 +434,15 @@ "GROUP_KIND_KEY", "GROUP_TOKEN_COUNT_KEY", "INTERNAL_SOURCE_ID", + "LOOP_EXIT_REASON_KEY", + "LOOP_NEED_INPUT_KEY", "MESSAGE_INJECTION_PENDING_MESSAGES_STATE_KEY", + "NEED_INPUT_TOKEN", "SKIP_PARSING", "SUMMARIZED_BY_SUMMARY_ID_KEY", "SUMMARY_OF_GROUP_IDS_KEY", "SUMMARY_OF_MESSAGE_IDS_KEY", + "TASK_COMPLETE_TOKEN", "USER_AGENT_KEY", "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", @@ -554,6 +568,7 @@ "InlineSkillScript", "JudgeVerdict", "LocalEvaluator", + "LoopExitReason", "MCPSkill", "MCPSkillResource", "MCPSkillsSource", @@ -596,6 +611,7 @@ "ServiceSessionId", "SessionContext", "SessionStore", + "SignalParser", "SingleEdgeGroup", "Skill", "SkillFrontmatter", @@ -696,6 +712,7 @@ "executor", "function_middleware", "get_agent_mode", + "get_loop_exit_reason", "get_run_context", "handler", "included_messages", @@ -715,6 +732,7 @@ "resolve_agent_id", "response_handler", "set_agent_mode", + "signal_should_continue", "step", "todos_remaining", "todos_remaining_message", diff --git a/python/packages/core/agent_framework/__init__.pyi b/python/packages/core/agent_framework/__init__.pyi index e8d1bc6e411..64c5b139e4d 100644 --- a/python/packages/core/agent_framework/__init__.pyi +++ b/python/packages/core/agent_framework/__init__.pyi @@ -106,6 +106,16 @@ from ._harness._memory import ( MemoryTopicRecord, ) from ._harness._mode import DEFAULT_MODE_SOURCE_ID, AgentModeProvider, get_agent_mode, set_agent_mode +from ._harness._signals import ( + LOOP_EXIT_REASON_KEY, + LOOP_NEED_INPUT_KEY, + NEED_INPUT_TOKEN, + TASK_COMPLETE_TOKEN, + LoopExitReason, + SignalParser, + get_loop_exit_reason, + signal_should_continue, +) from ._harness._todo import ( DEFAULT_TODO_SOURCE_ID, TodoFileStore, @@ -379,11 +389,15 @@ __all__ = [ "GROUP_KIND_KEY", "GROUP_TOKEN_COUNT_KEY", "INTERNAL_SOURCE_ID", + "LOOP_EXIT_REASON_KEY", + "LOOP_NEED_INPUT_KEY", "MESSAGE_INJECTION_PENDING_MESSAGES_STATE_KEY", + "NEED_INPUT_TOKEN", "SKIP_PARSING", "SUMMARIZED_BY_SUMMARY_ID_KEY", "SUMMARY_OF_GROUP_IDS_KEY", "SUMMARY_OF_MESSAGE_IDS_KEY", + "TASK_COMPLETE_TOKEN", "USER_AGENT_KEY", "USER_AGENT_TELEMETRY_DISABLED_ENV_VAR", "Agent", @@ -509,6 +523,7 @@ __all__ = [ "InlineSkillScript", "JudgeVerdict", "LocalEvaluator", + "LoopExitReason", "MCPSkill", "MCPSkillResource", "MCPSkillsSource", @@ -551,6 +566,7 @@ __all__ = [ "ServiceSessionId", "SessionContext", "SessionStore", + "SignalParser", "SingleEdgeGroup", "Skill", "SkillFrontmatter", @@ -651,6 +667,7 @@ __all__ = [ "executor", "function_middleware", "get_agent_mode", + "get_loop_exit_reason", "get_run_context", "handler", "included_messages", @@ -670,6 +687,7 @@ __all__ = [ "resolve_agent_id", "response_handler", "set_agent_mode", + "signal_should_continue", "step", "todos_remaining", "todos_remaining_message", diff --git a/python/packages/core/agent_framework/_harness/_loop.py b/python/packages/core/agent_framework/_harness/_loop.py index b4617a23e01..101a23956e8 100644 --- a/python/packages/core/agent_framework/_harness/_loop.py +++ b/python/packages/core/agent_framework/_harness/_loop.py @@ -22,6 +22,15 @@ structured output; provider-specific formats can be supplied with a parser that converts the response into ``JudgeVerdict``. The loop continues while the answer is "no". This is a convenience wrapper that builds an async ``should_continue`` predicate, so it is a special case of (1). +3. Signal protocol (via :func:`~agent_framework.signal_should_continue` from ``_signals.py``) - the + agent emits ``TASK_COMPLETE: `` or ``NEED_INPUT: `` in its response text to + signal terminal intent. Only assistant-role messages are scanned; user-role nudges injected between + iterations do not trigger false positives. ``TASK_COMPLETE:`` is checked first. Read the result + with :func:`~agent_framework.get_loop_exit_reason`, which returns a + :class:`~agent_framework.LoopExitReason` string (``"completed"``, ``"iteration_cap_reached"``, + ``"need_input"``). When ``max_iterations`` fires the loop stamps + ``additional_properties["loop_exit_reason"] = "iteration_cap_reached"``; signal tokens always win + over the cap. In every case, the input for the next iteration is controlled by the ``next_message`` callable. """ @@ -538,6 +547,7 @@ async def _process_non_streaming( snapshot: dict[str, Any] | None, ) -> None: iteration = 0 + cap_fired = False work_iterations = 0 progress: list[str] = [] # Aggregated transcript across iterations: each iteration's response messages plus the @@ -584,6 +594,8 @@ async def _process_non_streaming( # Decide whether to stop and capture any feedback from should_continue first, so the # feedback is available to both the progress and next-message callables this iteration. stop, feedback = await self._evaluate_stop(loop_kwargs, work_iterations) + if stop and self.max_iterations is not None and work_iterations >= self.max_iterations: + cap_fired = True loop_kwargs = self._build_loop_kwargs( context=context, iteration=iteration, @@ -617,6 +629,9 @@ async def _process_non_streaming( finally: context.options.pop(_LOOP_ITERATION_TOKEN_KEY, None) + if cap_fired: + final_result.additional_properties.setdefault("loop_exit_reason", "iteration_cap_reached") + if not self.return_final_only: context.result = self._aggregate_response(final_result, aggregated, aggregated_usage) await self._fire_turn_scoped_after_providers( @@ -638,6 +653,7 @@ def _process_streaming( async def _generator() -> Any: iteration = 0 + cap_fired = False work_iterations = 0 progress: list[str] = [] stamped_options = dict(context.options) if context.options is not None else {} @@ -686,6 +702,8 @@ async def _generator() -> Any: # Decide whether to stop and capture any feedback from should_continue first, so the # feedback is available to both the progress and next-message callables this iteration. stop, feedback = await self._evaluate_stop(loop_kwargs, work_iterations) + if stop and self.max_iterations is not None and work_iterations >= self.max_iterations: + cap_fired = True loop_kwargs = self._build_loop_kwargs( context=context, iteration=iteration, @@ -721,6 +739,8 @@ async def _generator() -> Any: yield self._message_to_update(message) finally: context.options.pop(_LOOP_ITERATION_TOKEN_KEY, None) + if cap_fired and holder["final"] is not None: + holder["final"].additional_properties.setdefault("loop_exit_reason", "iteration_cap_reached") await self._fire_turn_scoped_after_providers(context, holder["final"], original_messages) def _finalize(updates: Sequence[AgentResponseUpdate]) -> AgentResponse: diff --git a/python/packages/core/agent_framework/_harness/_signals.py b/python/packages/core/agent_framework/_harness/_signals.py new file mode 100644 index 00000000000..e7d4d9c8879 --- /dev/null +++ b/python/packages/core/agent_framework/_harness/_signals.py @@ -0,0 +1,174 @@ +# Copyright (c) Microsoft. All rights reserved. + +"""Signal protocol utilities for AgentLoopMiddleware autonomous loops. + +Provides a text-based convention for agents to signal loop termination: + TASK_COMPLETE: -- agent is done; loop should stop + NEED_INPUT: -- agent needs human input; loop should stop + +Use :func:`signal_should_continue` to get a :class:`SignalParser` instance +ready to pass as ``should_continue`` to :class:`~agent_framework.AgentLoopMiddleware`. +Use :func:`get_loop_exit_reason` to read the typed exit reason after the loop. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .._types import AgentResponse + +__all__ = [ + "LOOP_EXIT_REASON_KEY", + "LOOP_NEED_INPUT_KEY", + "NEED_INPUT_TOKEN", + "TASK_COMPLETE_TOKEN", + "LoopExitReason", + "SignalParser", + "get_loop_exit_reason", + "signal_should_continue", +] + +# Text tokens the agent emits in its response to signal loop state. +# Case-sensitive: the agent is instructed to emit the exact tokens. +TASK_COMPLETE_TOKEN: str = "TASK_COMPLETE:" # ruff: ignore[hardcoded-password-string] +NEED_INPUT_TOKEN: str = "NEED_INPUT:" # ruff: ignore[hardcoded-password-string] + +# Keys used in AgentResponse.additional_properties to surface loop metadata. +LOOP_EXIT_REASON_KEY: str = "loop_exit_reason" +LOOP_NEED_INPUT_KEY: str = "loop_need_input" + + +class LoopExitReason: + """Typed exit reason strings for AgentLoopMiddleware runs. + + Values are plain strings (not an enum) to match the ``FinishReason`` + convention in this codebase and remain JSON-transparent. + + Examples:: + + from agent_framework import get_loop_exit_reason, LoopExitReason + + exit_reason = get_loop_exit_reason(response) + if exit_reason == LoopExitReason.completed: + print("Task done!") + elif exit_reason == LoopExitReason.need_input: + # question is in response.text after NEED_INPUT: + """ + + completed: str = "completed" + """``TASK_COMPLETE:`` signal detected in the agent's response.""" + + iteration_cap_reached: str = "iteration_cap_reached" + """``max_iterations`` cap fired before any terminal signal was emitted.""" + + need_input: str = "need_input" + """``NEED_INPUT:`` signal detected; loop stopped to surface a question.""" + + cancelled: str = "cancelled" + """External cancellation (reserved for future use).""" + + token_budget_exceeded: str = "token_budget_exceeded" # ruff: ignore[hardcoded-password-string] + """Token budget (``max_tokens``) exhausted before the loop completed.""" + + time_budget_exceeded: str = "time_budget_exceeded" + """Wall-clock time budget (``max_duration``) exhausted before the loop completed.""" + + +class SignalParser: + """A ``should_continue`` predicate that scans for signal tokens. + + Scans the agent's latest response text for :data:`TASK_COMPLETE_TOKEN` + and :data:`NEED_INPUT_TOKEN` and returns the appropriate continuation + decision for :class:`~agent_framework.AgentLoopMiddleware`. + + ``TASK_COMPLETE:`` is checked before ``NEED_INPUT:``; if an agent emits + both, the loop treats the run as completed. + + The extracted text after the token (up to the first newline) is returned + as the feedback string so ``next_message`` and ``record_feedback`` + callbacks can reference it. + + Examples:: + + from agent_framework import AgentLoopMiddleware, signal_should_continue + + agent = Agent( + client=client, + middleware=[AgentLoopMiddleware(signal_should_continue())], + ) + """ + + def __call__(self, *, last_result: AgentResponse, **kwargs: Any) -> tuple[bool, str | None]: + """Return ``(False, extracted_text)`` on a terminal signal, ``(True, None)`` otherwise.""" + text = "\n".join(m.text or "" for m in last_result.messages if m.role == "assistant") + + idx = text.find(TASK_COMPLETE_TOKEN) + if idx != -1: + summary = text[idx + len(TASK_COMPLETE_TOKEN) :].split("\n", 1)[0].strip() + return (False, summary or None) + + idx = text.find(NEED_INPUT_TOKEN) + if idx != -1: + question = text[idx + len(NEED_INPUT_TOKEN) :].split("\n", 1)[0].strip() + return (False, question or None) + + return (True, None) + + +def signal_should_continue() -> SignalParser: + """Return a :class:`SignalParser` instance ready for :class:`~agent_framework.AgentLoopMiddleware`. + + Pass the returned callable as ``should_continue``:: + + middleware = AgentLoopMiddleware(signal_should_continue()) + """ + return SignalParser() + + +def get_loop_exit_reason(response: AgentResponse) -> str | None: + """Return the typed exit reason for a completed loop run, or ``None`` if unknown. + + Scans the agent's own (assistant-role) response messages for signal tokens first, + then falls back to ``response.additional_properties[LOOP_EXIT_REASON_KEY]`` + (set by :class:`~agent_framework.AgentLoopMiddleware` when the iteration cap fires). + + Signal tokens take priority: an agent that emitted ``TASK_COMPLETE:`` on the final + allowed iteration is treated as completed, not cap-reached. + + Args: + response: The :class:`~agent_framework.AgentResponse` returned by the loop. + + Returns: + A :class:`LoopExitReason` string, or ``None`` when the reason cannot + be determined (e.g. the loop was stopped by a custom predicate with no + signal tokens in the response). + + Examples: + + .. code-block:: python + + from agent_framework import get_loop_exit_reason, LoopExitReason + + response = await agent.run(task) + match get_loop_exit_reason(response): + case LoopExitReason.completed: + print("Done:", response.text) + case LoopExitReason.need_input: + # Question is in response.text after NEED_INPUT: + # LOOP_NEED_INPUT_KEY is a convention key reserved for future use. + print("Agent needs input; see response.text for the question.") + case LoopExitReason.iteration_cap_reached: + print("Hit cap - partial result") + case _: + print("Unknown exit reason") + """ + # Scan only assistant messages - the agent's own output. User-role nudges injected + # by the loop between iterations must not trigger a false completion signal. + assistant_text = "\n".join(m.text or "" for m in response.messages if m.role == "assistant") + if TASK_COMPLETE_TOKEN in assistant_text: + return LoopExitReason.completed + if NEED_INPUT_TOKEN in assistant_text: + return LoopExitReason.need_input + # Fall back to additional_properties for framework-set reasons (e.g. iteration cap). + return response.additional_properties.get(LOOP_EXIT_REASON_KEY) or None diff --git a/python/packages/core/tests/core/test_harness_loop.py b/python/packages/core/tests/core/test_harness_loop.py index 3b87544d804..f8a94b5cf76 100644 --- a/python/packages/core/tests/core/test_harness_loop.py +++ b/python/packages/core/tests/core/test_harness_loop.py @@ -9,6 +9,7 @@ from pydantic import BaseModel from agent_framework import ( + LOOP_EXIT_REASON_KEY, Agent, AgentContext, AgentMiddleware, @@ -24,14 +25,18 @@ Content, ContextProvider, JudgeVerdict, + LoopExitReason, Message, MiddlewareTermination, ResponseStream, + SignalParser, TodoItem, TodoProvider, background_tasks_running, background_tasks_running_message, + get_loop_exit_reason, set_agent_mode, + signal_should_continue, todos_remaining, todos_remaining_message, tool, @@ -1697,3 +1702,159 @@ async def test_concurrent_same_agent_run_during_stream_pause_is_not_suppressed() # once for the loop boundary, once for the concurrent run's own boundary assert turn_scoped.after_calls == 2 + + +# --- Signal Protocol Tests --- + + +def _make_signal_response(text: str, additional_properties: dict[str, Any] | None = None) -> AgentResponse: + return AgentResponse( + messages=[Message(role="assistant", contents=[text])], + additional_properties=additional_properties, + ) + + +def test_signal_parser_task_complete_stops_and_extracts_summary() -> None: + response = _make_signal_response("TASK_COMPLETE: finished the job") + assert SignalParser()(last_result=response) == (False, "finished the job") + + +def test_signal_parser_need_input_stops_and_extracts_question() -> None: + response = _make_signal_response("NEED_INPUT: which file should I edit?") + assert SignalParser()(last_result=response) == (False, "which file should I edit?") + + +def test_signal_parser_no_signal_continues() -> None: + response = _make_signal_response("I am still working on it.") + assert SignalParser()(last_result=response) == (True, None) + + +def test_signal_parser_task_complete_wins_over_need_input() -> None: + response = _make_signal_response("TASK_COMPLETE: done\nNEED_INPUT: are you sure?") + assert SignalParser()(last_result=response) == (False, "done") + + +def test_signal_parser_extracts_only_first_line_after_token() -> None: + response = _make_signal_response("Some preamble\nTASK_COMPLETE: line1\nmore stuff after") + assert SignalParser()(last_result=response) == (False, "line1") + + +def test_signal_parser_empty_response_continues() -> None: + response = AgentResponse(messages=[]) + assert SignalParser()(last_result=response) == (True, None) + + +def test_signal_parser_token_with_no_following_text_returns_none_feedback() -> None: + response = _make_signal_response("TASK_COMPLETE:") + assert SignalParser()(last_result=response) == (False, None) + + +def test_get_loop_exit_reason_completed_from_text() -> None: + response = _make_signal_response("TASK_COMPLETE: done") + assert get_loop_exit_reason(response) == LoopExitReason.completed + + +def test_get_loop_exit_reason_need_input_from_text() -> None: + response = _make_signal_response("NEED_INPUT: which approach?") + assert get_loop_exit_reason(response) == LoopExitReason.need_input + + +def test_get_loop_exit_reason_cap_from_additional_properties() -> None: + response = _make_signal_response( + "still going", additional_properties={LOOP_EXIT_REASON_KEY: "iteration_cap_reached"} + ) + assert get_loop_exit_reason(response) == LoopExitReason.iteration_cap_reached + + +def test_get_loop_exit_reason_signal_wins_over_additional_properties() -> None: + response = _make_signal_response( + "TASK_COMPLETE: done", additional_properties={LOOP_EXIT_REASON_KEY: "iteration_cap_reached"} + ) + assert get_loop_exit_reason(response) == LoopExitReason.completed + + +def test_get_loop_exit_reason_unknown_returns_none() -> None: + response = _make_signal_response("nothing special") + assert get_loop_exit_reason(response) is None + + +def test_signal_should_continue_factory_returns_signal_parser() -> None: + assert isinstance(signal_should_continue(), SignalParser) + + +async def test_loop_stops_on_task_complete_signal() -> None: + client = RecordingChatClient(texts=["TASK_COMPLETE: all done"]) + agent = Agent(client=client, middleware=[AgentLoopMiddleware(signal_should_continue(), max_iterations=10)]) + + response = await agent.run("task") + + assert client.call_count == 1 + assert get_loop_exit_reason(response) == LoopExitReason.completed + assert LOOP_EXIT_REASON_KEY not in response.additional_properties + + +async def test_loop_stops_on_need_input_signal() -> None: + client = RecordingChatClient(texts=["NEED_INPUT: which approach?"]) + agent = Agent(client=client, middleware=[AgentLoopMiddleware(signal_should_continue(), max_iterations=10)]) + + response = await agent.run("task") + + assert client.call_count == 1 + assert get_loop_exit_reason(response) == LoopExitReason.need_input + assert LOOP_EXIT_REASON_KEY not in response.additional_properties + + +async def test_loop_sets_iteration_cap_reason_when_no_signal() -> None: + client = RecordingChatClient() + agent = Agent(client=client, middleware=[AgentLoopMiddleware(signal_should_continue(), max_iterations=2)]) + + response = await agent.run("task") + + assert client.call_count == 2 + assert response.additional_properties.get(LOOP_EXIT_REASON_KEY) == LoopExitReason.iteration_cap_reached + assert get_loop_exit_reason(response) == LoopExitReason.iteration_cap_reached + + +async def test_loop_sets_iteration_cap_reason_streaming() -> None: + client = RecordingChatClient() + agent = Agent(client=client, middleware=[AgentLoopMiddleware(signal_should_continue(), max_iterations=2)]) + + stream = agent.run("task", stream=True) + async for _ in stream: + pass + response = await stream.get_final_response() + + assert client.call_count == 2 + assert response.additional_properties.get(LOOP_EXIT_REASON_KEY) == LoopExitReason.iteration_cap_reached + + +async def test_get_loop_exit_reason_ignores_nudge_text_with_token() -> None: + # A next_message nudge that mentions TASK_COMPLETE: must not trigger a false positive. + # Scripted texts keep the agent from echoing the nudge back in its own messages. + client = RecordingChatClient(texts=["working...", "still working..."]) + agent = Agent( + client=client, + middleware=[ + AgentLoopMiddleware( + signal_should_continue(), + max_iterations=2, + next_message=lambda **kwargs: "Reminder: emit TASK_COMPLETE: when done.", + ) + ], + ) + + response = await agent.run("task") + + assert client.call_count == 2 + assert response.additional_properties.get(LOOP_EXIT_REASON_KEY) == LoopExitReason.iteration_cap_reached + assert get_loop_exit_reason(response) == LoopExitReason.iteration_cap_reached + + +async def test_loop_continues_when_no_signal() -> None: + client = RecordingChatClient(texts=["working...", "still working...", "TASK_COMPLETE: finally done"]) + agent = Agent(client=client, middleware=[AgentLoopMiddleware(signal_should_continue(), max_iterations=10)]) + + response = await agent.run("task") + + assert client.call_count == 3 + assert get_loop_exit_reason(response) == LoopExitReason.completed