From dc2cf01868f8c632167449b895a062f8e28a86df Mon Sep 17 00:00:00 2001 From: Andy <152056884+jesko2004@users.noreply.github.com> Date: Sat, 19 Sep 2026 23:18:35 +0800 Subject: [PATCH] Fix charset parsing of quoted HTTP Content-Type parameters --- .../markitdown/src/markitdown/_markitdown.py | 15 ++--- .../tests/test_response_content_type.py | 55 +++++++++++++++++++ 2 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 packages/markitdown/tests/test_response_content_type.py diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 888d1e6260..3e533a02d3 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -544,13 +544,14 @@ def convert_response( charset: Optional[str] = None if "content-type" in response.headers: - parts = response.headers["content-type"].split(";") - mimetype = parts.pop(0).strip() - for part in parts: - if part.strip().startswith("charset="): - _charset = part.split("=")[1].strip() - if len(_charset) > 0: - charset = _charset + content_type = response.headers["content-type"] + mimetype = content_type.split(";", 1)[0].strip() + # A semicolon inside a quoted parameter is not a delimiter. + message = Message() + message["content-type"] = content_type + _charset = message.get_param("charset") + if isinstance(_charset, str) and _charset.strip(): + charset = _charset.strip() # If there is a content-disposition header, get the filename and possibly the extension filename: Optional[str] = None diff --git a/packages/markitdown/tests/test_response_content_type.py b/packages/markitdown/tests/test_response_content_type.py new file mode 100644 index 0000000000..2281701cc9 --- /dev/null +++ b/packages/markitdown/tests/test_response_content_type.py @@ -0,0 +1,55 @@ +import io + +import pytest +import requests + +from markitdown import MarkItDown, StreamInfo + + +def _response(content_type: str | None, body: bytes) -> requests.Response: + response = requests.Response() + response.status_code = 200 + response.url = "https://example.com/document.txt" + response.raw = io.BytesIO(body) + if content_type is not None: + response.headers["Content-Type"] = content_type + return response + + +@pytest.fixture(scope="module") +def converter() -> MarkItDown: + return MarkItDown(enable_plugins=False) + + +@pytest.mark.parametrize( + "content_type", + [ + 'text/plain; charset=iso-8859-1; profile="urn:example;charset=utf-8"', + 'text/plain; charset=iso-8859-1; profile="urn:example;charset=ascii;v=1"', + 'text/plain; profile="urn:example;charset=utf-8"; charset=iso-8859-1', + 'text/plain; charset="iso-8859-1"', + "text/plain; charset=iso-8859-1", + ], +) +def test_quoted_parameter_cannot_override_charset(converter, content_type): + response = _response(content_type, b"Caf\xe9") + + assert converter.convert_response(response).markdown == "Caf\u00e9" + + +@pytest.mark.parametrize("content_type", [None, "text/plain", 'text/plain; charset=""']) +def test_response_without_charset_still_converts(converter, content_type): + assert ( + converter.convert_response(_response(content_type, b"hello")).markdown + == "hello" + ) + + +def test_explicit_stream_info_charset_overrides_header(converter): + response = _response("text/plain; charset=utf-8", b"Caf\xe9") + + result = converter.convert_response( + response, stream_info=StreamInfo(charset="iso-8859-1") + ) + + assert result.markdown == "Caf\u00e9"