From 170e781b4cc49cf7877807ed3f86e029471b0596 Mon Sep 17 00:00:00 2001 From: RainieLLM Date: Fri, 18 Sep 2026 17:25:03 -0700 Subject: [PATCH 1/2] fix(bing-serp): do not rewrite result URLs that use their own u parameter Bing puts each search result behind a /ck/a redirect link. The real destination sits in a base64 `u` query parameter, and the converter decodes it. The decoder did not check the link, so it decoded the `u` parameter of every in the result. A normal result that uses `u=` for its own purpose was therefore rewritten into garbage. Decode the `u` parameter only when the link is one that Bing wrapped. --- .../converters/_bing_serp_converter.py | 20 ++++- .../tests/test_bing_serp_converter.py | 79 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 packages/markitdown/tests/test_bing_serp_converter.py diff --git a/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py b/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py index fd00a70ab7..787b0c5019 100644 --- a/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py +++ b/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py @@ -1,7 +1,7 @@ import re import base64 import binascii -from urllib.parse import parse_qs, urlparse +from urllib.parse import ParseResult, parse_qs, urlparse from typing import Any, BinaryIO from bs4 import BeautifulSoup @@ -20,6 +20,18 @@ ] +def _is_bing_redirect(parsed_href: ParseResult) -> bool: + """Return True only for Bing's own `/ck/a` redirect links. + + A normal result may use `u=` for its own purpose. So the base64 payload is + decoded only when the link is one that Bing wrapped. + """ + host = (parsed_href.netloc or "").lower() + if host != "bing.com" and not host.endswith(".bing.com"): + return False + return parsed_href.path.startswith("/ck/a") + + class BingSerpConverter(DocumentConverter): """ Handle Bing results pages (only the organic search results). @@ -87,6 +99,12 @@ def convert( # Rewrite redirect urls for a in result.find_all("a", href=True): parsed_href = urlparse(a["href"]) + + # Keep normal result URLs as they are. Only Bing's redirect + # links carry the base64 `u` payload. + if not _is_bing_redirect(parsed_href): + continue + qs = parse_qs(parsed_href.query) # The destination is contained in the u parameter, diff --git a/packages/markitdown/tests/test_bing_serp_converter.py b/packages/markitdown/tests/test_bing_serp_converter.py new file mode 100644 index 0000000000..e1594e320f --- /dev/null +++ b/packages/markitdown/tests/test_bing_serp_converter.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 -m pytest +"""A result URL that uses `u=` for its own purpose must survive conversion. + +Bing puts each search result behind a `/ck/a` redirect link. The real +destination sits in a base64 `u` query parameter. The converter decodes that +parameter, but it did so for every link that has `u=`. A normal result that +uses `u=` for its own purpose was therefore rewritten into garbage. +""" + +import io + +from markitdown import MarkItDown, StreamInfo + +SERP_URL = "https://www.bing.com/search?q=python" + +REDIRECT_TARGET = "https://docs.python.org/3/" +REDIRECT_HREF = ( + "https://www.bing.com/ck/a?!&&p=xyz" + "&u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy8=" +) + +NORMAL_HREF = "https://example.com/profile?u=abcdef" + +LOOKALIKE_HREF = "https://not-bing.com/ck/a?u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy8=" + + +def _serp(*hrefs: str) -> io.BytesIO: + items = "".join( + '

  • Result %d

    Snippet %d.

    • ' + % (href, i, i) + for i, href in enumerate(hrefs, start=1) + ) + html = ( + "python - Search" + '
      %s
    ' % items + ) + return io.BytesIO(html.encode("utf-8")) + + +def _convert(*hrefs: str) -> str: + return ( + MarkItDown() + .convert_stream( + _serp(*hrefs), + stream_info=StreamInfo( + url=SERP_URL, + mimetype="text/html", + extension=".html", + charset="utf-8", + ), + ) + .markdown + ) + + +def test_bing_redirect_is_decoded_to_its_destination() -> None: + markdown = _convert(REDIRECT_HREF) + + assert "](%s)" % REDIRECT_TARGET in markdown + assert "ck/a" not in markdown + + +def test_normal_url_keeps_its_own_u_parameter() -> None: + markdown = _convert(NORMAL_HREF) + + assert "](%s)" % NORMAL_HREF in markdown + + +def test_normal_url_survives_next_to_a_redirect() -> None: + markdown = _convert(REDIRECT_HREF, NORMAL_HREF) + + assert "](%s)" % REDIRECT_TARGET in markdown + assert "](%s)" % NORMAL_HREF in markdown + + +def test_lookalike_host_is_not_treated_as_a_bing_redirect() -> None: + markdown = _convert(LOOKALIKE_HREF) + + assert "](%s)" % LOOKALIKE_HREF in markdown From 2ab314b6d6d5c400a6b6f44fab8b7688cf390a4a Mon Sep 17 00:00:00 2001 From: RainieLLM Date: Fri, 18 Sep 2026 18:47:47 -0700 Subject: [PATCH 2/2] fix(bing-serp): match the redirect path exactly --- .../src/markitdown/converters/_bing_serp_converter.py | 2 +- packages/markitdown/tests/test_bing_serp_converter.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py b/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py index 787b0c5019..c58e359490 100644 --- a/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py +++ b/packages/markitdown/src/markitdown/converters/_bing_serp_converter.py @@ -29,7 +29,7 @@ def _is_bing_redirect(parsed_href: ParseResult) -> bool: host = (parsed_href.netloc or "").lower() if host != "bing.com" and not host.endswith(".bing.com"): return False - return parsed_href.path.startswith("/ck/a") + return parsed_href.path == "/ck/a" class BingSerpConverter(DocumentConverter): diff --git a/packages/markitdown/tests/test_bing_serp_converter.py b/packages/markitdown/tests/test_bing_serp_converter.py index e1594e320f..9d80589bce 100644 --- a/packages/markitdown/tests/test_bing_serp_converter.py +++ b/packages/markitdown/tests/test_bing_serp_converter.py @@ -23,6 +23,10 @@ LOOKALIKE_HREF = "https://not-bing.com/ck/a?u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy8=" +LOOKALIKE_PATH_HREF = ( + "https://www.bing.com/ck/abc?u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy8=" +) + def _serp(*hrefs: str) -> io.BytesIO: items = "".join( @@ -77,3 +81,9 @@ def test_lookalike_host_is_not_treated_as_a_bing_redirect() -> None: markdown = _convert(LOOKALIKE_HREF) assert "](%s)" % LOOKALIKE_HREF in markdown + + +def test_lookalike_path_is_not_treated_as_a_bing_redirect() -> None: + markdown = _convert(LOOKALIKE_PATH_HREF) + + assert "](%s)" % LOOKALIKE_PATH_HREF in markdown