Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import re
import base64
import binascii
from urllib.parse import parse_qs, urlparse
from urllib.parse import ParseResult, parse_qs, urlparse
from typing import Any, BinaryIO
from bs4 import BeautifulSoup

Expand All @@ -20,6 +20,18 @@
]


def _is_bing_redirect(parsed_href: ParseResult) -> bool:
"""Return True only for Bing's own `/ck/a` redirect links.

A normal result may use `u=` for its own purpose. So the base64 payload is
decoded only when the link is one that Bing wrapped.
"""
host = (parsed_href.netloc or "").lower()
if host != "bing.com" and not host.endswith(".bing.com"):
return False
return parsed_href.path == "/ck/a"


class BingSerpConverter(DocumentConverter):
"""
Handle Bing results pages (only the organic search results).
Expand Down Expand Up @@ -87,6 +99,12 @@ def convert(
# Rewrite redirect urls
for a in result.find_all("a", href=True):
parsed_href = urlparse(a["href"])

# Keep normal result URLs as they are. Only Bing's redirect
# links carry the base64 `u` payload.
if not _is_bing_redirect(parsed_href):
continue

qs = parse_qs(parsed_href.query)

# The destination is contained in the u parameter,
Expand Down
89 changes: 89 additions & 0 deletions packages/markitdown/tests/test_bing_serp_converter.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
#!/usr/bin/env python3 -m pytest
"""A result URL that uses `u=` for its own purpose must survive conversion.

Bing puts each search result behind a `/ck/a` redirect link. The real
destination sits in a base64 `u` query parameter. The converter decodes that
parameter, but it did so for every link that has `u=`. A normal result that
uses `u=` for its own purpose was therefore rewritten into garbage.
"""

import io

from markitdown import MarkItDown, StreamInfo

SERP_URL = "https://www.bing.com/search?q=python"

REDIRECT_TARGET = "https://docs.python.org/3/"
REDIRECT_HREF = (
"https://www.bing.com/ck/a?!&&p=xyz"
"&u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy8="
)

NORMAL_HREF = "https://example.com/profile?u=abcdef"

LOOKALIKE_HREF = "https://not-bing.com/ck/a?u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy8="

LOOKALIKE_PATH_HREF = (
"https://www.bing.com/ck/abc?u=a1aHR0cHM6Ly9kb2NzLnB5dGhvbi5vcmcvMy8="
)


def _serp(*hrefs: str) -> io.BytesIO:
items = "".join(
'<li class="b_algo"><h2><a href="%s">Result %d</a></h2><p>Snippet %d.</p></li>'
% (href, i, i)
for i, href in enumerate(hrefs, start=1)
)
html = (
"<html><head><title>python - Search</title></head><body>"
'<ol id="b_results">%s</ol></body></html>' % items
)
return io.BytesIO(html.encode("utf-8"))


def _convert(*hrefs: str) -> str:
return (
MarkItDown()
.convert_stream(
_serp(*hrefs),
stream_info=StreamInfo(
url=SERP_URL,
mimetype="text/html",
extension=".html",
charset="utf-8",
),
)
.markdown
)


def test_bing_redirect_is_decoded_to_its_destination() -> None:
markdown = _convert(REDIRECT_HREF)

assert "](%s)" % REDIRECT_TARGET in markdown
assert "ck/a" not in markdown


def test_normal_url_keeps_its_own_u_parameter() -> None:
markdown = _convert(NORMAL_HREF)

assert "](%s)" % NORMAL_HREF in markdown


def test_normal_url_survives_next_to_a_redirect() -> None:
markdown = _convert(REDIRECT_HREF, NORMAL_HREF)

assert "](%s)" % REDIRECT_TARGET in markdown
assert "](%s)" % NORMAL_HREF in markdown


def test_lookalike_host_is_not_treated_as_a_bing_redirect() -> None:
markdown = _convert(LOOKALIKE_HREF)

assert "](%s)" % LOOKALIKE_HREF in markdown


def test_lookalike_path_is_not_treated_as_a_bing_redirect() -> None:
markdown = _convert(LOOKALIKE_PATH_HREF)

assert "](%s)" % LOOKALIKE_PATH_HREF in markdown