From 0f1f45ea1a9c9f3b2b301d8495c293700ec289f3 Mon Sep 17 00:00:00 2001 From: "Link.sc" <58829603+getlinksc@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:28:32 -0400 Subject: [PATCH 1/2] Google: index ids for the jsl.dh splice, reuse the soup for extras (~40% faster parse) The AI-overview splice found each fragment's target with root.find(id=...), a walk of the whole tree per fragment and round: a third of a desktop parse. An id index built once gives the same element (first live one in document order; a repeated id still goes through find). parse_extras takes the soup parse already built instead of parsing the page again. Same output on all 29 fixtures; 3.67 s -> 2.24 s over the 12 largest. --- CHANGELOG.md | 8 +++ src/search_parser/parsers/google.py | 29 ++++++++-- src/search_parser/parsers/google_extras.py | 11 ++-- tests/unit/test_google_extras.py | 7 +++ tests/unit/test_google_jsl_dh.py | 62 ++++++++++++++++++++++ 5 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_google_jsl_dh.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7573788..56e6b04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Google parsing is about 40% faster on desktop pages (42-53% on the 1-2 MB Chrome fixtures), with + identical output on every fixture. The AI-overview splice of `jsl.dh` fragments looked each + fragment's target up with a walk of the whole tree, per fragment and round; it now uses an id + index built once. `parse_extras` takes an optional `soup` and `parse` passes its own, so the page + is no longer parsed into a second tree + --- ## [0.5.15] - 2026-10-28 diff --git a/src/search_parser/parsers/google.py b/src/search_parser/parsers/google.py index dd4f0fb..728f4e4 100644 --- a/src/search_parser/parsers/google.py +++ b/src/search_parser/parsers/google.py @@ -116,7 +116,7 @@ def _clean_search_link(result: SearchResult) -> SearchResult: def _parse_metadata(self, soup: BeautifulSoup, html: str) -> dict[str, object]: """Page metadata plus the ``extras`` from :meth:`parse_extras` (when any were found).""" metadata = self._extract_page_metadata(soup) - extras = self.parse_extras(html) + extras = self.parse_extras(html, soup) if extras: metadata["extras"] = extras return metadata @@ -802,16 +802,39 @@ def _splice_jsl_dh(self, soup: BeautifulSoup, root: Tag) -> None: for m in self._JSL_DH_ANY.finditer(text): body = m.group(3) if m.group(3) is not None else m.group(4) frags.append((m.group(1) or m.group(2), self._js_unescape(body))) + if not frags: + return + # id -> elements carrying it, so each fragment does not walk the whole tree + # (root.find per fragment and round was most of a desktop parse). + by_id: dict[str, list[Tag]] = {} + + def index(tree: Tag) -> None: + for el in tree.find_all(id=True): + by_id.setdefault(str(el.get("id")), []).append(el) + + def target(fid: str) -> Tag | None: + # still under root: a cleared element's old descendants are not (identity, not Tag ==) + live = [el for el in by_id.get(fid, ()) if any(p is root for p in el.parents)] + if len(live) > 1: # a repeated id: the first in document order, as find() gives + found = root.find(id=fid) + return found if isinstance(found, Tag) else None + return live[0] if live else None + + index(root) for _ in range(6): # fragments can target ids created by other fragments left = [] for fid, body in frags: - el = root.find(id=fid) - if not isinstance(el, Tag): + el = target(fid) + if el is None: left.append((fid, body)) continue el.clear() for child in list(make_soup(body).contents): el.append(child) + if isinstance(child, Tag): + if child.get("id") is not None: + by_id.setdefault(str(child.get("id")), []).append(child) + index(child) if len(left) == len(frags): break frags = left diff --git a/src/search_parser/parsers/google_extras.py b/src/search_parser/parsers/google_extras.py index 1f9cf39..a384581 100644 --- a/src/search_parser/parsers/google_extras.py +++ b/src/search_parser/parsers/google_extras.py @@ -300,12 +300,17 @@ def _x_ads(self, soup: BeautifulSoup) -> list[dict[str, Any]]: ) return out - def parse_extras(self, content: str) -> dict[str, Any]: + def parse_extras(self, content: str, soup: BeautifulSoup | None = None) -> dict[str, Any]: """Fields the search-parser/_parse pair leave out. Returns only keys it - found; every part is independent so one failing selector costs only itself.""" + found; every part is independent so one failing selector costs only itself. + + ``soup`` is ``content`` already parsed (``parse`` passes its own), which saves + building a second tree of a page that can be megabytes; none of the parts + change it.""" if not content: return {} - soup = BeautifulSoup(content, "lxml") + if soup is None: + soup = BeautifulSoup(content, "lxml") out = {} parts = ( ("total", lambda: self._x_total(content)), diff --git a/tests/unit/test_google_extras.py b/tests/unit/test_google_extras.py index b3f7576..559ec42 100644 --- a/tests/unit/test_google_extras.py +++ b/tests/unit/test_google_extras.py @@ -152,3 +152,10 @@ def test_parse_puts_extras_in_metadata() -> None: assert extras["pagination"]["has_next"] is True bare = PARSER.parse(_page("

no extras

")) assert "extras" not in bare.metadata + + +def test_a_given_soup_gives_the_same_extras() -> None: + from search_parser.utils import make_soup + + html = _page(TOTAL_RAW) + assert PARSER.parse_extras(html, make_soup(html)) == PARSER.parse_extras(html) diff --git a/tests/unit/test_google_jsl_dh.py b/tests/unit/test_google_jsl_dh.py new file mode 100644 index 0000000..31a7509 --- /dev/null +++ b/tests/unit/test_google_jsl_dh.py @@ -0,0 +1,62 @@ +"""Unit tests for GoogleParser._splice_jsl_dh: which element each jsl.dh fragment lands in.""" + +from __future__ import annotations + +from bs4 import BeautifulSoup + +from search_parser.parsers.google import GoogleParser +from search_parser.utils import make_soup + +PARSER = GoogleParser() + + +def _spliced(body: str, *frags: tuple[str, str]) -> BeautifulSoup: + scripts = "".join(f"" for fid, html in frags) + soup = make_soup(f"{body}{scripts}") + PARSER._splice_jsl_dh(soup, soup) + for script in soup.find_all("script"): + script.decompose() + return soup + + +def _texts(soup: BeautifulSoup, fid: str) -> list[str]: + return [el.get_text() for el in soup.find_all(id=fid)] + + +def test_a_fragment_replaces_its_targets_content() -> None: + soup = _spliced('
old
', ("a", "

new

")) + assert _texts(soup, "a") == ["new"] + + +def test_a_fragment_can_target_an_id_another_fragment_created() -> None: + # listed before the fragment that creates its target: lands on the next round + soup = _spliced('
', ("b", "inner"), ("a", '')) + assert _texts(soup, "b") == ["inner"] + assert _texts(soup, "a") == ["inner"] + + +def test_a_repeated_id_takes_the_first_in_document_order() -> None: + soup = _spliced('
1
2
', ("a", "x")) + assert _texts(soup, "a") == ["x", "2"] + + +def test_an_id_cleared_away_is_no_longer_a_target() -> None: + # #b goes with #a's old content, so its fragment has nowhere to land + soup = _spliced( + '
b
', ("a", "

new

"), ("b", "

lost

") + ) + assert _texts(soup, "a") == ["new"] + assert "lost" not in soup.get_text() + + +def test_a_new_element_with_a_cleared_id_is_the_target() -> None: + soup = _spliced( + '
old
', + ("a", '
'), + ("b", "

new

"), + ) + assert _texts(soup, "b") == ["new"] + + +def test_no_fragments_leaves_the_tree_alone() -> None: + assert _texts(_spliced('
old
'), "a") == ["old"] From be11cfa7e8a79879822d529a9de7accddec959e6 Mon Sep 17 00:00:00 2001 From: "Link.sc" <58829603+getlinksc@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:40:46 -0400 Subject: [PATCH 2/2] Bump to 0.5.16 Moves the Unreleased parse speed-up under 0.5.16. --- CHANGELOG.md | 4 ++++ pyproject.toml | 2 +- src/search_parser/__version__.py | 2 +- uv.lock | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56e6b04..8ffa02f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +--- + +## [0.5.16] - 2026-10-28 + ### Changed - Google parsing is about 40% faster on desktop pages (42-53% on the 1-2 MB Chrome fixtures), with diff --git a/pyproject.toml b/pyproject.toml index b232104..b97a956 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "search-parser" -version = "0.5.15" +version = "0.5.16" description = "Parse Google, Bing, DuckDuckGo, eBay, and Google Maps results into typed JSON, Markdown, or dict output" readme = "README.md" requires-python = ">=3.10" diff --git a/src/search_parser/__version__.py b/src/search_parser/__version__.py index 4b70b99..cca1d19 100644 --- a/src/search_parser/__version__.py +++ b/src/search_parser/__version__.py @@ -1,3 +1,3 @@ """Version information.""" -__version__ = "0.5.15" +__version__ = "0.5.16" diff --git a/uv.lock b/uv.lock index fe7aaf3..e6b509e 100644 --- a/uv.lock +++ b/uv.lock @@ -2001,7 +2001,7 @@ wheels = [ [[package]] name = "search-parser" -version = "0.5.15" +version = "0.5.16" source = { editable = "." } dependencies = [ { name = "beautifulsoup4" },