From 45c45d921517631352ca0d23cba48f1e952e0aad Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Wed, 7 Oct 2026 17:00:30 +0000 Subject: [PATCH 1/3] Python: Retry the Magentic progress ledger when it names an unknown agent --- .../_magentic.py | 10 ++++++++- .../orchestrations/tests/test_magentic.py | 22 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index 6206022f043..a5569324502 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -728,7 +728,15 @@ async def create_progress_ledger(self, magentic_context: MagenticContext) -> Mag raw = await self._complete([*magentic_context.chat_history, user_message]) try: ledger_dict = _extract_json(raw.text) - return _coerce_model(MagenticProgressLedger, ledger_dict) + ledger = _coerce_model(MagenticProgressLedger, ledger_dict) + next_speaker = ledger.next_speaker.answer + if ( + not ledger.is_request_satisfied.answer + and isinstance(next_speaker, str) + and next_speaker not in agent_names + ): + raise ValueError(f"Unknown next speaker '{next_speaker}'. Valid names: {names_csv}.") + return ledger except Exception as ex: last_error = ex attempts += 1 diff --git a/python/packages/orchestrations/tests/test_magentic.py b/python/packages/orchestrations/tests/test_magentic.py index 38a3dcc718c..0e7ffe4c4ce 100644 --- a/python/packages/orchestrations/tests/test_magentic.py +++ b/python/packages/orchestrations/tests/test_magentic.py @@ -581,6 +581,28 @@ async def fake_complete_bad(messages: list[Message], **kwargs: Any) -> Message: await mgr.create_progress_ledger(ctx.clone()) +async def test_standard_manager_progress_ledger_retries_unknown_next_speaker(): + mgr = StandardMagenticManager(agent=StubManagerAgent(), progress_ledger_retry_count=2) + ctx = MagenticContext(task="task", participant_descriptions={"alice": "desc"}) + speakers = ["Alice", "alice"] + + async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: + json_text = ( + '{"is_request_satisfied": {"reason": "r", "answer": false}, ' + '"is_in_loop": {"reason": "r", "answer": false}, ' + '"is_progress_being_made": {"reason": "r", "answer": true}, ' + f'"next_speaker": {{"reason": "r", "answer": "{speakers.pop(0)}"}}, ' + '"instruction_or_question": {"reason": "r", "answer": "do"}}' + ) + return Message("assistant", [json_text]) + + mgr._complete = fake_complete # type: ignore[method-assign] # ty: ignore[invalid-assignment] + ledger = await mgr.create_progress_ledger(ctx.clone()) + + assert ledger.next_speaker.answer == "alice" + assert not speakers + + class InvokeOnceManager(MagenticManagerBase): def __init__(self) -> None: super().__init__(max_round_count=5, max_stall_count=3, max_reset_count=2) From b1b51425aa16705c921590cc526e21187451f895 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:21:02 +0000 Subject: [PATCH 2/3] Python: End the Magentic run when every ledger retry names an unknown agent --- .../_magentic.py | 35 +++++++++++-------- .../orchestrations/tests/test_magentic.py | 25 +++++++++++++ 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index a5569324502..154c628cd81 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -724,29 +724,36 @@ async def create_progress_ledger(self, magentic_context: MagenticContext) -> Mag # Include full context to help the model decide current stage, with small retry loop attempts = 0 last_error: Exception | None = None + unknown_speaker_ledger: MagenticProgressLedger | None = None while attempts < self.progress_ledger_retry_count: raw = await self._complete([*magentic_context.chat_history, user_message]) try: ledger_dict = _extract_json(raw.text) ledger = _coerce_model(MagenticProgressLedger, ledger_dict) + except Exception as ex: + last_error = ex + else: next_speaker = ledger.next_speaker.answer if ( - not ledger.is_request_satisfied.answer - and isinstance(next_speaker, str) - and next_speaker not in agent_names + ledger.is_request_satisfied.answer + or not isinstance(next_speaker, str) + or next_speaker in agent_names ): - raise ValueError(f"Unknown next speaker '{next_speaker}'. Valid names: {names_csv}.") - return ledger - except Exception as ex: - last_error = ex - attempts += 1 - logger.warning( - f"Progress ledger JSON parse failed (attempt {attempts}/{self.progress_ledger_retry_count}): {ex}" - ) - if attempts < self.progress_ledger_retry_count: - # brief backoff before next try - await asyncio.sleep(0.25 * attempts) + return ledger + # Retry an unknown name, but keep the ledger so running out of attempts ends the run + # through the orchestrator's invalid speaker handling instead of another reset. + unknown_speaker_ledger = ledger + last_error = ValueError(f"Unknown next speaker '{next_speaker}'. Valid names: {names_csv}.") + attempts += 1 + logger.warning( + f"Progress ledger attempt failed ({attempts}/{self.progress_ledger_retry_count}): {last_error}" + ) + if attempts < self.progress_ledger_retry_count: + # brief backoff before next try + await asyncio.sleep(0.25 * attempts) + if unknown_speaker_ledger is not None: + return unknown_speaker_ledger raise RuntimeError( f"Progress ledger parse failed after {self.progress_ledger_retry_count} attempt(s): {last_error}" ) diff --git a/python/packages/orchestrations/tests/test_magentic.py b/python/packages/orchestrations/tests/test_magentic.py index 0e7ffe4c4ce..ed392118a6f 100644 --- a/python/packages/orchestrations/tests/test_magentic.py +++ b/python/packages/orchestrations/tests/test_magentic.py @@ -603,6 +603,31 @@ async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: assert not speakers +async def test_standard_manager_progress_ledger_returns_unknown_speaker_after_retries(): + """Exhausted retries on an unknown name end the run instead of triggering another reset.""" + mgr = StandardMagenticManager(agent=StubManagerAgent(), progress_ledger_retry_count=3) + ctx = MagenticContext(task="task", participant_descriptions={"alice": "desc"}) + calls = 0 + + async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: + nonlocal calls + calls += 1 + json_text = ( + '{"is_request_satisfied": {"reason": "r", "answer": false}, ' + '"is_in_loop": {"reason": "r", "answer": false}, ' + '"is_progress_being_made": {"reason": "r", "answer": true}, ' + '"next_speaker": {"reason": "r", "answer": "Alice"}, ' + '"instruction_or_question": {"reason": "r", "answer": "do"}}' + ) + return Message("assistant", [json_text]) + + mgr._complete = fake_complete # type: ignore[method-assign] # ty: ignore[invalid-assignment] + ledger = await mgr.create_progress_ledger(ctx.clone()) + + assert ledger.next_speaker.answer == "Alice" + assert calls == 3 + + class InvokeOnceManager(MagenticManagerBase): def __init__(self) -> None: super().__init__(max_round_count=5, max_stall_count=3, max_reset_count=2) From 1db7ac8c781b581886a8489fd20ff1f46a514132 Mon Sep 17 00:00:00 2001 From: Lanre Shittu <136805224+Shizoqua@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:24:07 +0000 Subject: [PATCH 3/3] Python: Report the parse failure when the last Magentic ledger attempt is malformed --- .../_magentic.py | 1 + .../orchestrations/tests/test_magentic.py | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py index 154c628cd81..c19e227c6ff 100644 --- a/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py +++ b/python/packages/orchestrations/agent_framework_orchestrations/_magentic.py @@ -732,6 +732,7 @@ async def create_progress_ledger(self, magentic_context: MagenticContext) -> Mag ledger = _coerce_model(MagenticProgressLedger, ledger_dict) except Exception as ex: last_error = ex + unknown_speaker_ledger = None else: next_speaker = ledger.next_speaker.answer if ( diff --git a/python/packages/orchestrations/tests/test_magentic.py b/python/packages/orchestrations/tests/test_magentic.py index ed392118a6f..a9834834690 100644 --- a/python/packages/orchestrations/tests/test_magentic.py +++ b/python/packages/orchestrations/tests/test_magentic.py @@ -628,6 +628,27 @@ async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: assert calls == 3 +async def test_standard_manager_progress_ledger_reports_parse_failure_after_unknown_speaker(): + """A saved unknown-speaker ledger does not hide a parse failure on the final attempt.""" + mgr = StandardMagenticManager(agent=StubManagerAgent(), progress_ledger_retry_count=3) + ctx = MagenticContext(task="task", participant_descriptions={"alice": "desc"}) + unknown_speaker = ( + '{"is_request_satisfied": {"reason": "r", "answer": false}, ' + '"is_in_loop": {"reason": "r", "answer": false}, ' + '"is_progress_being_made": {"reason": "r", "answer": true}, ' + '"next_speaker": {"reason": "r", "answer": "Alice"}, ' + '"instruction_or_question": {"reason": "r", "answer": "do"}}' + ) + replies = [unknown_speaker, "not-json", "not-json"] + + async def fake_complete(messages: list[Message], **kwargs: Any) -> Message: + return Message("assistant", [replies.pop(0)]) + + mgr._complete = fake_complete # type: ignore[method-assign] # ty: ignore[invalid-assignment] + with pytest.raises(RuntimeError, match="No JSON object found"): + await mgr.create_progress_ledger(ctx.clone()) + + class InvokeOnceManager(MagenticManagerBase): def __init__(self) -> None: super().__init__(max_round_count=5, max_stall_count=3, max_reset_count=2)