Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

---

## [0.5.16] - 2026-10-28

### Changed

- Google parsing is about 40% faster on desktop pages (42-53% on the 1-2 MB Chrome fixtures), with
identical output on every fixture. The AI-overview splice of `jsl.dh` fragments looked each
fragment's target up with a walk of the whole tree, per fragment and round; it now uses an id
index built once. `parse_extras` takes an optional `soup` and `parse` passes its own, so the page
is no longer parsed into a second tree

---

## [0.5.15] - 2026-10-28

### Fixed
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "search-parser"
version = "0.5.15"
version = "0.5.16"
description = "Parse Google, Bing, DuckDuckGo, eBay, and Google Maps results into typed JSON, Markdown, or dict output"
readme = "README.md"
requires-python = ">=3.10"
Expand Down
2 changes: 1 addition & 1 deletion src/search_parser/__version__.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
"""Version information."""

__version__ = "0.5.15"
__version__ = "0.5.16"
29 changes: 26 additions & 3 deletions src/search_parser/parsers/google.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ def _clean_search_link(result: SearchResult) -> SearchResult:
def _parse_metadata(self, soup: BeautifulSoup, html: str) -> dict[str, object]:
"""Page metadata plus the ``extras`` from :meth:`parse_extras` (when any were found)."""
metadata = self._extract_page_metadata(soup)
extras = self.parse_extras(html)
extras = self.parse_extras(html, soup)
if extras:
metadata["extras"] = extras
return metadata
Expand Down Expand Up @@ -802,16 +802,39 @@ def _splice_jsl_dh(self, soup: BeautifulSoup, root: Tag) -> None:
for m in self._JSL_DH_ANY.finditer(text):
body = m.group(3) if m.group(3) is not None else m.group(4)
frags.append((m.group(1) or m.group(2), self._js_unescape(body)))
if not frags:
return
# id -> elements carrying it, so each fragment does not walk the whole tree
# (root.find per fragment and round was most of a desktop parse).
by_id: dict[str, list[Tag]] = {}

def index(tree: Tag) -> None:
for el in tree.find_all(id=True):
by_id.setdefault(str(el.get("id")), []).append(el)

def target(fid: str) -> Tag | None:
# still under root: a cleared element's old descendants are not (identity, not Tag ==)
live = [el for el in by_id.get(fid, ()) if any(p is root for p in el.parents)]
if len(live) > 1: # a repeated id: the first in document order, as find() gives
found = root.find(id=fid)
return found if isinstance(found, Tag) else None
return live[0] if live else None

index(root)
for _ in range(6): # fragments can target ids created by other fragments
left = []
for fid, body in frags:
el = root.find(id=fid)
if not isinstance(el, Tag):
el = target(fid)
if el is None:
left.append((fid, body))
continue
el.clear()
for child in list(make_soup(body).contents):
el.append(child)
if isinstance(child, Tag):
if child.get("id") is not None:
by_id.setdefault(str(child.get("id")), []).append(child)
index(child)
if len(left) == len(frags):
break
frags = left
Expand Down
11 changes: 8 additions & 3 deletions src/search_parser/parsers/google_extras.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,12 +300,17 @@ def _x_ads(self, soup: BeautifulSoup) -> list[dict[str, Any]]:
)
return out

def parse_extras(self, content: str) -> dict[str, Any]:
def parse_extras(self, content: str, soup: BeautifulSoup | None = None) -> dict[str, Any]:
"""Fields the search-parser/_parse pair leave out. Returns only keys it
found; every part is independent so one failing selector costs only itself."""
found; every part is independent so one failing selector costs only itself.

``soup`` is ``content`` already parsed (``parse`` passes its own), which saves
building a second tree of a page that can be megabytes; none of the parts
change it."""
if not content:
return {}
soup = BeautifulSoup(content, "lxml")
if soup is None:
soup = BeautifulSoup(content, "lxml")
out = {}
parts = (
("total", lambda: self._x_total(content)),
Expand Down
7 changes: 7 additions & 0 deletions tests/unit/test_google_extras.py
Original file line number Diff line number Diff line change
Expand Up @@ -152,3 +152,10 @@ def test_parse_puts_extras_in_metadata() -> None:
assert extras["pagination"]["has_next"] is True
bare = PARSER.parse(_page("<p>no extras</p>"))
assert "extras" not in bare.metadata


def test_a_given_soup_gives_the_same_extras() -> None:
from search_parser.utils import make_soup

html = _page(TOTAL_RAW)
assert PARSER.parse_extras(html, make_soup(html)) == PARSER.parse_extras(html)
62 changes: 62 additions & 0 deletions tests/unit/test_google_jsl_dh.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""Unit tests for GoogleParser._splice_jsl_dh: which element each jsl.dh fragment lands in."""

from __future__ import annotations

from bs4 import BeautifulSoup

from search_parser.parsers.google import GoogleParser
from search_parser.utils import make_soup

PARSER = GoogleParser()


def _spliced(body: str, *frags: tuple[str, str]) -> BeautifulSoup:
scripts = "".join(f"<script>window.jsl.dh('{fid}','{html}');</script>" for fid, html in frags)
soup = make_soup(f"<html><body>{body}{scripts}</body></html>")
PARSER._splice_jsl_dh(soup, soup)
for script in soup.find_all("script"):
script.decompose()
return soup


def _texts(soup: BeautifulSoup, fid: str) -> list[str]:
return [el.get_text() for el in soup.find_all(id=fid)]


def test_a_fragment_replaces_its_targets_content() -> None:
soup = _spliced('<div id="a">old</div>', ("a", "<p>new</p>"))
assert _texts(soup, "a") == ["new"]


def test_a_fragment_can_target_an_id_another_fragment_created() -> None:
# listed before the fragment that creates its target: lands on the next round
soup = _spliced('<div id="a"></div>', ("b", "<i>inner</i>"), ("a", '<span id="b"></span>'))
assert _texts(soup, "b") == ["inner"]
assert _texts(soup, "a") == ["inner"]


def test_a_repeated_id_takes_the_first_in_document_order() -> None:
soup = _spliced('<div id="a">1</div><div id="a">2</div>', ("a", "x"))
assert _texts(soup, "a") == ["x", "2"]


def test_an_id_cleared_away_is_no_longer_a_target() -> None:
# #b goes with #a's old content, so its fragment has nowhere to land
soup = _spliced(
'<div id="a"><div id="b">b</div></div>', ("a", "<p>new</p>"), ("b", "<p>lost</p>")
)
assert _texts(soup, "a") == ["new"]
assert "lost" not in soup.get_text()


def test_a_new_element_with_a_cleared_id_is_the_target() -> None:
soup = _spliced(
'<div id="a"><div id="b">old</div></div>',
("a", '<div id="b"></div>'),
("b", "<p>new</p>"),
)
assert _texts(soup, "b") == ["new"]


def test_no_fragments_leaves_the_tree_alone() -> None:
assert _texts(_spliced('<div id="a">old</div>'), "a") == ["old"]
2 changes: 1 addition & 1 deletion uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading