From 4ba0a5cfbd3d997e243bbff9d8854b49bea1fb47 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Sun, 4 Oct 2026 19:30:51 +0530 Subject: [PATCH 1/7] Python: fix(core): serialize Pydantic tool results in JSON mode Co-Authored-By: Claude Opus 5.5 (1M context) --- python/packages/core/agent_framework/_tools.py | 3 ++- python/packages/core/tests/core/test_types.py | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 602da9dc4cb..81d9354a16b 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -1256,7 +1256,8 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, Content): return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): - return value.model_dump() + # JSON mode, so enums, datetimes and UUIDs come out as Pydantic serializes them. + return value.model_dump(mode="json") if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 05d474687f4..32c123b80d6 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3821,6 +3821,22 @@ def test_parse_result_nested_pydantic_model(): assert "18.0" in parsed[0].text or "18" in parsed[0].text # type: ignore[operator] # pyrefly: ignore[not-iterable] # ty: ignore[unsupported-operator] +def test_parse_result_pydantic_model_uses_json_mode(): + """Enum and datetime fields are serialized the way Pydantic writes them to JSON.""" + from enum import Enum + + class Status(Enum): + OPEN = "open" + + class Ticket(BaseModel): + status: Status + created: datetime + + parsed = FunctionTool.parse_result(Ticket(status=Status.OPEN, created=datetime(2024, 1, 1, 12))) + + assert json.loads(parsed[0].text) == {"status": "open", "created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From c9a68d954caf15293b3840b5c537a2d35f83a6e6 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Mon, 5 Oct 2026 22:35:20 +0530 Subject: [PATCH 2/7] Python: fall back to the python dump when a model has no JSON serializer --- python/packages/core/agent_framework/_tools.py | 7 ++++++- python/packages/core/tests/core/test_types.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 81d9354a16b..f52dde564d9 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -45,6 +45,7 @@ from opentelemetry import trace from opentelemetry.metrics import Histogram, NoOpHistogram from pydantic import BaseModel, Field, ValidationError, create_model +from pydantic_core import PydanticSerializationError from ._serialization import SerializationMixin from .exceptions import ResponseInvalidatedException, ToolException, UserInputRequiredException @@ -1257,7 +1258,11 @@ def _make_dumpable(value: Any) -> Any: return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): # JSON mode, so enums, datetimes and UUIDs come out as Pydantic serializes them. - return value.model_dump(mode="json") + # Arbitrary types have no JSON serializer, so those models keep the python dump. + try: + return value.model_dump(mode="json") + except PydanticSerializationError: + return value.model_dump() if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 32c123b80d6..c941ae5be88 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3837,6 +3837,24 @@ class Ticket(BaseModel): assert json.loads(parsed[0].text) == {"status": "open", "created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def test_parse_result_pydantic_model_with_arbitrary_type(): + """A field Pydantic cannot serialize to JSON falls back to str, as before.""" + + class Handle: + def __str__(self) -> str: + return "handle-1" + + class Result(BaseModel): + model_config = {"arbitrary_types_allowed": True} + + handle: Handle + created: datetime + + parsed = FunctionTool.parse_result(Result(handle=Handle(), created=datetime(2024, 1, 1, 12))) + + assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01 12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From ee3a266d0ddf2c58f03eb3f92a4d3a591cec0455 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Wed, 7 Oct 2026 14:18:32 +0530 Subject: [PATCH 3/7] Python: fall back to str per value instead of dumping the whole model in python mode --- python/packages/core/agent_framework/_tools.py | 10 +++------- python/packages/core/tests/core/test_types.py | 12 +++++++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index f52dde564d9..44fa646b4b5 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -45,7 +45,6 @@ from opentelemetry import trace from opentelemetry.metrics import Histogram, NoOpHistogram from pydantic import BaseModel, Field, ValidationError, create_model -from pydantic_core import PydanticSerializationError from ._serialization import SerializationMixin from .exceptions import ResponseInvalidatedException, ToolException, UserInputRequiredException @@ -1257,12 +1256,9 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, Content): return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): - # JSON mode, so enums, datetimes and UUIDs come out as Pydantic serializes them. - # Arbitrary types have no JSON serializer, so those models keep the python dump. - try: - return value.model_dump(mode="json") - except PydanticSerializationError: - return value.model_dump() + # JSON mode, so enums, datetimes, UUIDs and JSON-only field serializers apply. Values + # Pydantic cannot serialize to JSON (arbitrary types) fall back to str individually. + return value.model_dump(mode="json", fallback=str) if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index c941ae5be88..18805dfd3a0 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3838,7 +3838,8 @@ class Ticket(BaseModel): def test_parse_result_pydantic_model_with_arbitrary_type(): - """A field Pydantic cannot serialize to JSON falls back to str, as before.""" + """Only the field Pydantic cannot serialize to JSON falls back to str; the others stay in JSON mode.""" + from pydantic import field_serializer class Handle: def __str__(self) -> str: @@ -3849,10 +3850,15 @@ class Result(BaseModel): handle: Handle created: datetime + token: str - parsed = FunctionTool.parse_result(Result(handle=Handle(), created=datetime(2024, 1, 1, 12))) + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" - assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01 12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + parsed = FunctionTool.parse_result(Result(handle=Handle(), created=datetime(2024, 1, 1, 12), token="secret")) + + assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01T12:00:00", "token": "***"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] # region FunctionTool.parse_result with MCP TextContent-like objects From 603ec936b646a3b6fed0bbccf8cb3760e94d4d04 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Tue, 13 Oct 2026 14:00:27 +0530 Subject: [PATCH 4/7] Fall back to the python-mode dump when JSON mode can't encode a value --- python/packages/core/agent_framework/_tools.py | 7 ++++++- python/packages/core/tests/core/test_types.py | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 44fa646b4b5..e1e6136e9af 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -1258,7 +1258,12 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, BaseModel): # JSON mode, so enums, datetimes, UUIDs and JSON-only field serializers apply. Values # Pydantic cannot serialize to JSON (arbitrary types) fall back to str individually. - return value.model_dump(mode="json", fallback=str) + try: + return value.model_dump(mode="json", fallback=str) + except ValueError: + # e.g. bytes that aren't valid UTF-8. The python-mode dump still drops excluded + # fields, unlike the str(result) fallback in invoke. + return value.model_dump() if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 18805dfd3a0..744bf46b9db 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3861,6 +3861,20 @@ def _redact(self, value: str) -> str: assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01T12:00:00", "token": "***"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def test_parse_result_pydantic_model_with_non_utf8_bytes_keeps_excluded_fields_out(): + """Bytes that JSON mode cannot encode don't fail the whole result, and excluded fields stay out.""" + from pydantic import Field + + class Result(BaseModel): + data: bytes + api_key: str = Field(exclude=True) + + parsed = FunctionTool.parse_result(Result(data=b"\xff\xfe", api_key="secret")) + + assert "secret" not in parsed[0].text # type: ignore[operator] # pyrefly: ignore[not-iterable] # ty: ignore[unsupported-operator] + assert set(json.loads(parsed[0].text)) == {"data"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From ffd46edfc341ba9528a2741942708fba3472826e Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Thu, 15 Oct 2026 03:20:00 +0530 Subject: [PATCH 5/7] Fall back per field when JSON mode can't encode a value, and only pass fallback where Pydantic supports it --- .../packages/core/agent_framework/_tools.py | 41 ++++++++++++---- python/packages/core/tests/core/test_types.py | 48 ++++++++++++++++--- 2 files changed, 75 insertions(+), 14 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index e1e6136e9af..da3fb68868e 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -411,6 +411,10 @@ def _annotation_includes_function_invocation_context(annotation: Any) -> bool: _ToolDescriptionFormat: TypeAlias = _ToolParameterFormat | Mapping[str, _ToolParameterFormat] _NormalizedToolDescriptionFormat: TypeAlias = _ToolParameterFormat | dict[str, _ToolParameterFormat] _TOOL_DESCRIPTION_FORMAT_ERROR = "tool_description_format must be 'compact', 'json', or a tool-name mapping." +# model_dump's fallback (str for values JSON mode doesn't know) only exists on newer Pydantic 2 versions. +_MODEL_DUMP_JSON_FALLBACK: dict[str, Any] = ( + {"fallback": str} if "fallback" in inspect.signature(BaseModel.model_dump).parameters else {} +) def _normalize_tool_description_format( # pyright: ignore[reportUnusedFunction] @@ -1256,20 +1260,41 @@ def _make_dumpable(value: Any) -> Any: if isinstance(value, Content): return value.to_dict(exclude={"raw_representation", "additional_properties"}) if isinstance(value, BaseModel): - # JSON mode, so enums, datetimes, UUIDs and JSON-only field serializers apply. Values - # Pydantic cannot serialize to JSON (arbitrary types) fall back to str individually. - try: - return value.model_dump(mode="json", fallback=str) - except ValueError: - # e.g. bytes that aren't valid UTF-8. The python-mode dump still drops excluded - # fields, unlike the str(result) fallback in invoke. - return value.model_dump() + return FunctionTool._dump_model(value) if hasattr(value, "to_dict"): return value.to_dict() if hasattr(value, "text") and isinstance(value.text, str): return value.text return value + @staticmethod + def _dump_model(model: BaseModel) -> Any: + """Dump a Pydantic model in JSON mode, falling back to python mode only for the fields that need it. + + JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. A field + JSON mode cannot encode (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic + versions without ``fallback``) is dumped in python mode on its own, so the other fields keep + their JSON serializers. Excluded fields stay out either way, unlike the str(result) fallback + in invoke. + """ + try: + return model.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) + except ValueError: + pass + model_type = type(model) + names = [*model_type.model_fields, *model_type.model_computed_fields, *(model.model_extra or {})] + dumped: dict[str, Any] = {} + for name in names: + try: + dumped.update(model.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) + except ValueError: + field_dump = model.model_dump(include={name}) + field_value = getattr(model, name, None) + if isinstance(field_value, BaseModel): + field_dump = {key: FunctionTool._dump_model(field_value) for key in field_dump} + dumped.update(field_dump) + return dumped + @staticmethod def parse_result(result: Any) -> list[Content]: """Convert a raw function return value to a list of Content items. diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 744bf46b9db..2ab53d42b52 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3861,18 +3861,54 @@ def _redact(self, value: str) -> str: assert json.loads(parsed[0].text) == {"handle": "handle-1", "created": "2024-01-01T12:00:00", "token": "***"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] -def test_parse_result_pydantic_model_with_non_utf8_bytes_keeps_excluded_fields_out(): - """Bytes that JSON mode cannot encode don't fail the whole result, and excluded fields stay out.""" - from pydantic import Field +def test_parse_result_pydantic_model_with_non_utf8_bytes_falls_back_per_field(): + """Only the bytes fall back; JSON-only serializers still run and excluded fields stay out.""" + from pydantic import Field, field_serializer class Result(BaseModel): data: bytes + created: datetime + token: str api_key: str = Field(exclude=True) - parsed = FunctionTool.parse_result(Result(data=b"\xff\xfe", api_key="secret")) + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" + + result = Result(data=b"\xff\xfe", created=datetime(2024, 1, 1, 12), token="secret", api_key="key") + parsed = FunctionTool.parse_result(result) + + assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + "data": str(b"\xff\xfe"), + "created": "2024-01-01T12:00:00", + "token": "***", + } + + +def test_parse_result_nested_pydantic_model_with_non_utf8_bytes(): + """A nested model with bytes JSON mode cannot encode keeps its JSON-only serializers.""" + from pydantic import field_serializer + + class Inner(BaseModel): + data: bytes + token: str + + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" + + class Outer(BaseModel): + inner: Inner + created: datetime + + parsed = FunctionTool.parse_result( + Outer(inner=Inner(data=b"\xff\xfe", token="secret"), created=datetime(2024, 1, 1, 12)) + ) - assert "secret" not in parsed[0].text # type: ignore[operator] # pyrefly: ignore[not-iterable] # ty: ignore[unsupported-operator] - assert set(json.loads(parsed[0].text)) == {"data"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + "inner": {"data": str(b"\xff\xfe"), "token": "***"}, + "created": "2024-01-01T12:00:00", + } # region FunctionTool.parse_result with MCP TextContent-like objects From 4295bcd3ad9e7f7a9c08b1e8bbbd5b6068137ad9 Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Thu, 15 Oct 2026 16:43:15 +0530 Subject: [PATCH 6/7] Take fallback field names from the python dump and keep JSON mode for models in lists and dicts --- .../packages/core/agent_framework/_tools.py | 69 +++++++++++++++---- python/packages/core/tests/core/test_types.py | 43 ++++++++++++ 2 files changed, 97 insertions(+), 15 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index da3fb68868e..4b7b1c725d3 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -44,7 +44,8 @@ from opentelemetry import trace from opentelemetry.metrics import Histogram, NoOpHistogram -from pydantic import BaseModel, Field, ValidationError, create_model +from pydantic import BaseModel, Field, PlainSerializer, ValidationError, WrapSerializer, create_model +from pydantic_core import to_jsonable_python from ._serialization import SerializationMixin from .exceptions import ResponseInvalidatedException, ToolException, UserInputRequiredException @@ -417,6 +418,17 @@ def _annotation_includes_function_invocation_context(annotation: Any) -> bool: ) +def _has_field_serializer(model_type: type[BaseModel], name: str) -> bool: + """Whether a model field has its own serializer, from a decorator or an annotation.""" + decorators = model_type.__pydantic_decorators__.field_serializers.values() + if any(name in decorator.info.fields or "*" in decorator.info.fields for decorator in decorators): + return True + field = model_type.model_fields.get(name) + return field is not None and any( + isinstance(metadata, (PlainSerializer, WrapSerializer)) for metadata in field.metadata + ) + + def _normalize_tool_description_format( # pyright: ignore[reportUnusedFunction] value: object, ) -> _NormalizedToolDescriptionFormat: @@ -1269,32 +1281,59 @@ def _make_dumpable(value: Any) -> Any: @staticmethod def _dump_model(model: BaseModel) -> Any: - """Dump a Pydantic model in JSON mode, falling back to python mode only for the fields that need it. - - JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. A field - JSON mode cannot encode (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic - versions without ``fallback``) is dumped in python mode on its own, so the other fields keep - their JSON serializers. Excluded fields stay out either way, unlike the str(result) fallback - in invoke. + """Dump a Pydantic model in JSON mode, falling back to python mode only for the values that need it. + + JSON mode applies enum, datetime and UUID encoding and JSON-only field serializers. When it + fails (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic versions without + ``fallback``), each field is dumped in JSON mode on its own and only a failing field falls + back. Models inside a failing field, directly or in a list or dict, are dumped the same way, + so their JSON serializers and excluded fields still apply. Excluded fields stay out either + way, unlike the str(result) fallback in invoke. """ try: return model.model_dump(mode="json", **_MODEL_DUMP_JSON_FALLBACK) except ValueError: pass + # The python-mode dump gives the serialized field names without the excluded ones. The class's + # model_computed_fields can't be used for that, it's a property on Pydantic 2.0. + python_dump = model.model_dump(by_alias=False) + if not isinstance(python_dump, dict): + return model.model_dump() model_type = type(model) - names = [*model_type.model_fields, *model_type.model_computed_fields, *(model.model_extra or {})] dumped: dict[str, Any] = {} - for name in names: + for name in python_dump: try: dumped.update(model.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) + continue except ValueError: - field_dump = model.model_dump(include={name}) - field_value = getattr(model, name, None) - if isinstance(field_value, BaseModel): - field_dump = {key: FunctionTool._dump_model(field_value) for key in field_dump} - dumped.update(field_dump) + pass + for key, python_value in model.model_dump(include={name}).items(): + value = getattr(model, name, None) + if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)) and not _has_field_serializer( + model_type, name + ): + dumped[key] = FunctionTool._dump_fallback_value(value) + else: + dumped[key] = python_value return dumped + @staticmethod + def _dump_fallback_value(value: Any) -> Any: + """Dump a field value JSON mode could not encode, keeping JSON mode for what inside it can be.""" + if isinstance(value, BaseModel): + return FunctionTool._dump_model(value) + if isinstance(value, dict): + dict_value = cast(dict[object, object], value) + return {key: FunctionTool._dump_fallback_value(item) for key, item in dict_value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + items = cast(Iterable[object], value) + return [FunctionTool._dump_fallback_value(item) for item in items] + try: + return to_jsonable_python(value) + except ValueError: + # parse_result's json.dumps(default=str) turns it into a string + return value + @staticmethod def parse_result(result: Any) -> list[Content]: """Convert a raw function return value to a list of Content items. diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 2ab53d42b52..b30cf55ce23 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -3911,6 +3911,49 @@ class Outer(BaseModel): } +def test_parse_result_pydantic_model_with_non_utf8_bytes_keeps_computed_fields(): + """The per-field fallback includes computed fields and leaves excluded fields out.""" + from pydantic import Field, computed_field + + class Result(BaseModel): + data: bytes + api_key: str = Field(exclude=True) + + @computed_field # type: ignore[prop-decorator] + @property + def size(self) -> int: + return len(self.data) + + parsed = FunctionTool.parse_result(Result(data=b"\xff\xfe", api_key="key")) + + assert json.loads(parsed[0].text) == {"data": str(b"\xff\xfe"), "size": 2} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + +def test_parse_result_pydantic_models_in_list_and_dict_with_non_utf8_bytes(): + """Models inside a list or dict field that falls back keep their JSON-only serializers and exclusions.""" + from pydantic import Field, field_serializer + + class Inner(BaseModel): + data: bytes + created: datetime + token: str + api_key: str = Field(exclude=True) + + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> str: + return "***" + + class Outer(BaseModel): + items: list[Inner] + by_name: dict[str, Inner] + + inner = Inner(data=b"\xff\xfe", created=datetime(2024, 1, 1, 12), token="secret", api_key="key") + parsed = FunctionTool.parse_result(Outer(items=[inner], by_name={"a": inner})) + + expected_inner = {"data": str(b"\xff\xfe"), "created": "2024-01-01T12:00:00", "token": "***"} + assert json.loads(parsed[0].text) == {"items": [expected_inner], "by_name": {"a": expected_inner}} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects From 0a73949d4420daaaf807bf1181337fdf81c204ca Mon Sep 17 00:00:00 2001 From: Anish Mehta Date: Thu, 22 Oct 2026 23:53:56 +0530 Subject: [PATCH 7/7] Keep JSON mode for a field with its own serializer when its output has invalid UTF-8 bytes --- .../packages/core/agent_framework/_tools.py | 40 ++++++++++++++++--- python/packages/core/tests/core/test_types.py | 38 ++++++++++++++++++ 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/python/packages/core/agent_framework/_tools.py b/python/packages/core/agent_framework/_tools.py index 7e1d48d3564..d11417e7cfa 100644 --- a/python/packages/core/agent_framework/_tools.py +++ b/python/packages/core/agent_framework/_tools.py @@ -517,6 +517,24 @@ def _has_field_serializer(model_type: type[BaseModel], name: str) -> bool: ) +def _escape_invalid_utf8(value: Any) -> Any: + """Replace bytes that aren't valid UTF-8 with their str() form, inside models, dicts and sequences too.""" + if isinstance(value, bytes): + try: + value.decode("utf-8") + except UnicodeDecodeError: + return str(value).encode() + return value + if isinstance(value, BaseModel): + return value.model_copy(update={name: _escape_invalid_utf8(item) for name, item in value.__dict__.items()}) + if isinstance(value, dict): + dict_value = cast(dict[object, object], value) + return {key: _escape_invalid_utf8(item) for key, item in dict_value.items()} + if type(value) in (list, tuple, set, frozenset): + return type(value)(_escape_invalid_utf8(item) for item in value) + return value + + def _normalize_tool_description_format( # pyright: ignore[reportUnusedFunction] value: object, ) -> _NormalizedToolDescriptionFormat: @@ -1402,7 +1420,9 @@ def _dump_model(model: BaseModel) -> Any: fails (bytes that aren't valid UTF-8, or an arbitrary type on Pydantic versions without ``fallback``), each field is dumped in JSON mode on its own and only a failing field falls back. Models inside a failing field, directly or in a list or dict, are dumped the same way, - so their JSON serializers and excluded fields still apply. Excluded fields stay out either + so their JSON serializers and excluded fields still apply. A failing field with its own + serializer is dumped in JSON mode again with its invalid UTF-8 bytes escaped, so the + serializer still runs, and is left out if that fails too. Excluded fields stay out either way, unlike the str(result) fallback in invoke. """ try: @@ -1422,11 +1442,21 @@ def _dump_model(model: BaseModel) -> Any: continue except ValueError: pass + value = getattr(model, name, None) + if _has_field_serializer(model_type, name): + # The python-mode value would skip the field's serializer, and with it any redaction it does. + escaped = model.model_copy(update={name: _escape_invalid_utf8(value)}) + try: + dumped.update(escaped.model_dump(mode="json", include={name}, **_MODEL_DUMP_JSON_FALLBACK)) + except ValueError: + logger.warning( + "Leaving %s.%s out of the tool result: its serializer's output can't be encoded as JSON.", + model_type.__name__, + name, + ) + continue for key, python_value in model.model_dump(include={name}).items(): - value = getattr(model, name, None) - if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)) and not _has_field_serializer( - model_type, name - ): + if isinstance(value, (BaseModel, dict, list, tuple, set, frozenset)): dumped[key] = FunctionTool._dump_fallback_value(value) else: dumped[key] = python_value diff --git a/python/packages/core/tests/core/test_types.py b/python/packages/core/tests/core/test_types.py index 1da72d71082..1d417e529c0 100644 --- a/python/packages/core/tests/core/test_types.py +++ b/python/packages/core/tests/core/test_types.py @@ -4660,6 +4660,44 @@ class Outer(BaseModel): assert json.loads(parsed[0].text) == {"items": [expected_inner], "by_name": {"a": expected_inner}} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] +def test_parse_result_pydantic_field_serializer_output_with_non_utf8_bytes(): + """A JSON-only serializer whose output has bytes JSON mode cannot encode still runs; only the bytes fall back.""" + from pydantic import field_serializer + + class Result(BaseModel): + info: dict[str, Any] + created: datetime + + @field_serializer("info", when_used="json") + def _redact(self, value: dict[str, Any]) -> dict[str, Any]: + return {**value, "token": "***"} + + result = Result(info={"token": "secret", "data": b"\xff\xfe"}, created=datetime(2024, 1, 1, 12)) + parsed = FunctionTool.parse_result(result) + + assert json.loads(parsed[0].text) == { # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + "info": {"token": "***", "data": str(b"\xff\xfe")}, + "created": "2024-01-01T12:00:00", + } + + +def test_parse_result_pydantic_field_serializer_output_that_cannot_be_encoded_is_left_out(): + """A field whose serializer output JSON mode cannot encode is left out instead of sent without the serializer.""" + from pydantic import field_serializer + + class Result(BaseModel): + token: str + created: datetime + + @field_serializer("token", when_used="json") + def _redact(self, value: str) -> bytes: + return b"\xff***" + + parsed = FunctionTool.parse_result(Result(token="secret", created=datetime(2024, 1, 1, 12))) + + assert json.loads(parsed[0].text) == {"created": "2024-01-01T12:00:00"} # type: ignore[arg-type] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] + + # region FunctionTool.parse_result with MCP TextContent-like objects