diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index 6206022f043..c19e227c6ff 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -724,21 +724,37 @@ async def create_progress_ledger(self, magentic_context: MagenticContext) -> Mag # Include full context to help the model decide current stage, with small retry loop attempts = 0 last_error: Exception | None = None + unknown_speaker_ledger: MagenticProgressLedger | None = None while attempts < self.progress_ledger_retry_count: raw = await self._complete([*magentic_context.chat_history, user_message]) try: ledger_dict = _extract_json(raw.text) - return _coerce_model(MagenticProgressLedger, ledger_dict) + ledger = _coerce_model(MagenticProgressLedger, ledger_dict) except Exception as ex: last_error = ex - attempts += 1 - logger.warning( - f"Progress ledger JSON parse failed (attempt {attempts}/{self.progress_ledger_retry_count}): {ex}" - ) - if attempts < self.progress_ledger_retry_count: - # brief backoff before next try - await asyncio.sleep(0.25 * attempts) + unknown_speaker_ledger = None + else: + next_speaker = ledger.next_speaker.answer + if ( + ledger.is_request_satisfied.answer + or not isinstance(next_speaker, str) + or next_speaker in agent_names + ): + return ledger + # Retry an unknown name, but keep the ledger so running out of attempts ends the run + # through the orchestrator's invalid speaker handling instead of another reset. + unknown_speaker_ledger = ledger + last_error = ValueError(f"Unknown next speaker '{next_speaker}'. Valid names: {names_csv}.") + attempts += 1 + logger.warning( + f"Progress ledger attempt failed ({attempts}/{self.progress_ledger_retry_count}): {last_error}" + ) + if attempts < self.progress_ledger_retry_count: + # brief backoff before next try + await asyncio.sleep(0.25 * attempts) + if unknown_speaker_ledger is not None: + return unknown_speaker_ledger raise RuntimeError( f"Progress ledger parse failed after {self.progress_ledger_retry_count} attempt(s): {last_error}" ) diff --git a/python/packages/orchestrations/tests/test_magentic.py b/python/packages/orchestrations/tests/test_magentic.py index 38a3dcc718c..a9834834690 100644 --- a/python/packages/orchestrations/tests/test_magentic.py +++ b/python/packages/orchestrations/tests/test_magentic.py @@ -581,6 +581,74 @@ async def fake_complete_bad(messages: list[Message], **kwargs: Any) -> Message: await mgr.create_progress_ledger(ctx.clone()) +async def test_standard_manager_progress_ledger_retries_unknown_next_speaker(): + mgr = StandardMagenticManager(agent=StubManagerAgent(), progress_ledger_retry_count=2) + ctx = MagenticContext(task="task", participant_descriptions={"alice": "desc"}) + speakers = ["Alice", "alice"] + + async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: + json_text = ( + '{"is_request_satisfied": {"reason": "r", "answer": false}, ' + '"is_in_loop": {"reason": "r", "answer": false}, ' + '"is_progress_being_made": {"reason": "r", "answer": true}, ' + f'"next_speaker": {{"reason": "r", "answer": "{speakers.pop(0)}"}}, ' + '"instruction_or_question": {"reason": "r", "answer": "do"}}' + ) + return Message("assistant", [json_text]) + + mgr._complete = fake_complete # type: ignore[method-assign] # ty: ignore[invalid-assignment] + ledger = await mgr.create_progress_ledger(ctx.clone()) + + assert ledger.next_speaker.answer == "alice" + assert not speakers + + +async def test_standard_manager_progress_ledger_returns_unknown_speaker_after_retries(): + """Exhausted retries on an unknown name end the run instead of triggering another reset.""" + mgr = StandardMagenticManager(agent=StubManagerAgent(), progress_ledger_retry_count=3) + ctx = MagenticContext(task="task", participant_descriptions={"alice": "desc"}) + calls = 0 + + async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: + nonlocal calls + calls += 1 + json_text = ( + '{"is_request_satisfied": {"reason": "r", "answer": false}, ' + '"is_in_loop": {"reason": "r", "answer": false}, ' + '"is_progress_being_made": {"reason": "r", "answer": true}, ' + '"next_speaker": {"reason": "r", "answer": "Alice"}, ' + '"instruction_or_question": {"reason": "r", "answer": "do"}}' + ) + return Message("assistant", [json_text]) + + mgr._complete = fake_complete # type: ignore[method-assign] # ty: ignore[invalid-assignment] + ledger = await mgr.create_progress_ledger(ctx.clone()) + + assert ledger.next_speaker.answer == "Alice" + assert calls == 3 + + +async def test_standard_manager_progress_ledger_reports_parse_failure_after_unknown_speaker(): + """A saved unknown-speaker ledger does not hide a parse failure on the final attempt.""" + mgr = StandardMagenticManager(agent=StubManagerAgent(), progress_ledger_retry_count=3) + ctx = MagenticContext(task="task", participant_descriptions={"alice": "desc"}) + unknown_speaker = ( + '{"is_request_satisfied": {"reason": "r", "answer": false}, ' + '"is_in_loop": {"reason": "r", "answer": false}, ' + '"is_progress_being_made": {"reason": "r", "answer": true}, ' + '"next_speaker": {"reason": "r", "answer": "Alice"}, ' + '"instruction_or_question": {"reason": "r", "answer": "do"}}' + ) + replies = [unknown_speaker, "not-json", "not-json"] + + async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: + return Message("assistant", [replies.pop(0)]) + + mgr._complete = fake_complete # type: ignore[method-assign] # ty: ignore[invalid-assignment] + with pytest.raises(RuntimeError, match="No JSON object found"): + await mgr.create_progress_ledger(ctx.clone()) + + class InvokeOnceManager(MagenticManagerBase): def __init__(self) -> None: super().__init__(max_round_count=5, max_stall_count=3, max_reset_count=2)