From 0f1f45ea1a9c9f3b2b301d8495c293700ec289f3 Mon Sep 17 00:00:00 2001 From: "Link.sc" <58829603+getlinksc@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:28:32 -0400 Subject: [PATCH 1/2] Google: index ids for the jsl.dh splice, reuse the soup for extras (~40% faster parse) The AI-overview splice found each fragment's target with root.find(id=...), a walk of the whole tree per fragment and round: a third of a desktop parse. An id index built once gives the same element (first live one in document order; a repeated id still goes through find). parse_extras takes the soup parse already built instead of parsing the page again. Same output on all 29 fixtures; 3.67 s -> 2.24 s over the 12 largest. --- CHANGELOG.md | 8 +++ src/search_parser/parsers/google.py | 29 ++++++++-- src/search_parser/parsers/google_extras.py | 11 ++-- tests/unit/test_google_extras.py | 7 +++ tests/unit/test_google_jsl_dh.py | 62 ++++++++++++++++++++++ 5 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 tests/unit/test_google_jsl_dh.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7573788..56e6b04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Google parsing is about 40% faster on desktop pages (42-53% on the 1-2 MB Chrome fixtures), with + identical output on every fixture. The AI-overview splice of `jsl.dh` fragments looked each + fragment's target up with a walk of the whole tree, per fragment and round; it now uses an id + index built once. `parse_extras` takes an optional `soup` and `parse` passes its own, so the page + is no longer parsed into a second tree + --- ## [0.5.15] - 2026-10-28 diff --git a/src/search_parser/parsers/google.py b/src/search_parser/parsers/google.py index dd4f0fb..728f4e4 100644 --- a/src/search_parser/parsers/google.py +++ b/src/search_parser/parsers/google.py @@ -116,7 +116,7 @@ def _clean_search_link(result: SearchResult) -> SearchResult: def _parse_metadata(self, soup: BeautifulSoup, html: str) -> dict[str, object]: """Page metadata plus the ``extras`` from :meth:`parse_extras` (when any were found).""" metadata = self._extract_page_metadata(soup) - extras = self.parse_extras(html) + extras = self.parse_extras(html, soup) if extras: metadata["extras"] = extras return metadata @@ -802,16 +802,39 @@ def _splice_jsl_dh(self, soup: BeautifulSoup, root: Tag) -> None: for m in self._JSL_DH_ANY.finditer(text): body = m.group(3) if m.group(3) is not None else m.group(4) frags.append((m.group(1) or m.group(2), self._js_unescape(body))) + if not frags: + return + # id -> elements carrying it, so each fragment does not walk the whole tree + # (root.find per fragment and round was most of a desktop parse). + by_id: dict[str, list[Tag]] = {} + + def index(tree: Tag) -> None: + for el in tree.find_all(id=True): + by_id.setdefault(str(el.get("id")), []).append(el) + + def target(fid: str) -> Tag | None: + # still under root: a cleared element's old descendants are not (identity, not Tag ==) + live = [el for el in by_id.get(fid, ()) if any(p is root for p in el.parents)] + if len(live) > 1: # a repeated id: the first in document order, as find() gives + found = root.find(id=fid) + return found if isinstance(found, Tag) else None + return live[0] if live else None + + index(root) for _ in range(6): # fragments can target ids created by other fragments left = [] for fid, body in frags: - el = root.find(id=fid) - if not isinstance(el, Tag): + el = target(fid) + if el is None: left.append((fid, body)) continue el.clear() for child in list(make_soup(body).contents): el.append(child) + if isinstance(child, Tag): + if child.get("id") is not None: + by_id.setdefault(str(child.get("id")), []).append(child) + index(child) if len(left) == len(frags): break frags = left diff --git a/src/search_parser/parsers/google_extras.py b/src/search_parser/parsers/google_extras.py index 1f9cf39..a384581 100644 --- a/src/search_parser/parsers/google_extras.py +++ b/src/search_parser/parsers/google_extras.py @@ -300,12 +300,17 @@ def _x_ads(self, soup: BeautifulSoup) -> list[dict[str, Any]]: ) return out - def parse_extras(self, content: str) -> dict[str, Any]: + def parse_extras(self, content: str, soup: BeautifulSoup | None = None) -> dict[str, Any]: """Fields the search-parser/_parse pair leave out. Returns only keys it - found; every part is independent so one failing selector costs only itself.""" + found; every part is independent so one failing selector costs only itself. + + ``soup`` is ``content`` already parsed (``parse`` passes its own), which saves + building a second tree of a page that can be megabytes; none of the parts + change it.""" if not content: return {} - soup = BeautifulSoup(content, "lxml") + if soup is None: + soup = BeautifulSoup(content, "lxml") out = {} parts = ( ("total", lambda: self._x_total(content)), diff --git a/tests/unit/test_google_extras.py b/tests/unit/test_google_extras.py index b3f7576..559ec42 100644 --- a/tests/unit/test_google_extras.py +++ b/tests/unit/test_google_extras.py @@ -152,3 +152,10 @@ def test_parse_puts_extras_in_metadata() -> None: assert extras["pagination"]["has_next"] is True bare = PARSER.parse(_page("
no extras
")) assert "extras" not in bare.metadata + + +def test_a_given_soup_gives_the_same_extras() -> None: + from search_parser.utils import make_soup + + html = _page(TOTAL_RAW) + assert PARSER.parse_extras(html, make_soup(html)) == PARSER.parse_extras(html) diff --git a/tests/unit/test_google_jsl_dh.py b/tests/unit/test_google_jsl_dh.py new file mode 100644 index 0000000..31a7509 --- /dev/null +++ b/tests/unit/test_google_jsl_dh.py @@ -0,0 +1,62 @@ +"""Unit tests for GoogleParser._splice_jsl_dh: which element each jsl.dh fragment lands in.""" + +from __future__ import annotations + +from bs4 import BeautifulSoup + +from search_parser.parsers.google import GoogleParser +from search_parser.utils import make_soup + +PARSER = GoogleParser() + + +def _spliced(body: str, *frags: tuple[str, str]) -> BeautifulSoup: + scripts = "".join(f"" for fid, html in frags) + soup = make_soup(f"{body}{scripts}") + PARSER._splice_jsl_dh(soup, soup) + for script in soup.find_all("script"): + script.decompose() + return soup + + +def _texts(soup: BeautifulSoup, fid: str) -> list[str]: + return [el.get_text() for el in soup.find_all(id=fid)] + + +def test_a_fragment_replaces_its_targets_content() -> None: + soup = _spliced('new
")) + assert _texts(soup, "a") == ["new"] + + +def test_a_fragment_can_target_an_id_another_fragment_created() -> None: + # listed before the fragment that creates its target: lands on the next round + soup = _spliced('', ("b", "inner"), ("a", '')) + assert _texts(soup, "b") == ["inner"] + assert _texts(soup, "a") == ["inner"] + + +def test_a_repeated_id_takes_the_first_in_document_order() -> None: + soup = _spliced('new
"), ("b", "lost
") + ) + assert _texts(soup, "a") == ["new"] + assert "lost" not in soup.get_text() + + +def test_a_new_element_with_a_cleared_id_is_the_target() -> None: + soup = _spliced( + 'new
"), + ) + assert _texts(soup, "b") == ["new"] + + +def test_no_fragments_leaves_the_tree_alone() -> None: + assert _texts(_spliced('