From 366a0c41121a5227f8c0b3fd1a0d18f7a2128794 Mon Sep 17 00:00:00 2001 From: itsafal <141961256+itsafal@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:48:10 -0400 Subject: [PATCH 1/2] fix: preserve inline CSS background images --- crawl4ai/content_scraping_strategy.py | 57 +++++++++++++++++++++ tests/test_css_background_images.py | 72 +++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 tests/test_css_background_images.py diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py index 67e87250d..1e8300a6e 100644 --- a/crawl4ai/content_scraping_strategy.py +++ b/crawl4ai/content_scraping_strategy.py @@ -36,6 +36,12 @@ OG_REGEX = re.compile(r"^og:") TWITTER_REGEX = re.compile(r"^twitter:") DIMENSION_REGEX = re.compile(r"(\d+)(\D*)") +CSS_BACKGROUND_DECLARATION_REGEX = re.compile( + r"(?:^|;)\s*background(?:-image)?\s*:\s*([^;]+)", re.IGNORECASE +) +CSS_URL_REGEX = re.compile( + r"url\(\s*(?:([\"'])(.*?)\1|([^)]*?))\s*\)", re.IGNORECASE +) # Function to parse srcset @@ -119,6 +125,47 @@ def _log(self, level, message, tag="SCRAPE", **kwargs): log_method = getattr(self.logger, level) log_method(message=message, tag=tag, **kwargs) + def _materialize_inline_css_background_images( + self, element: lhtml.HtmlElement, page_url: str + ) -> None: + """Convert inline CSS background URLs into images for media and markdown.""" + seen_urls = { + urljoin(page_url, src.strip()) + for src in element.xpath(".//img/@src") + if src and src.strip() + } + + styled_elements = element.xpath(".//*[@style]") + if element.get("style"): + styled_elements.insert(0, element) + + for styled_element in styled_elements: + style = styled_element.get("style", "") + for declaration in CSS_BACKGROUND_DECLARATION_REGEX.findall(style): + for quote, quoted_url, unquoted_url in CSS_URL_REGEX.findall( + declaration + ): + src = (quoted_url if quote else unquoted_url).strip() + if not src or src.lower().startswith( + ("data:", "blob:", "javascript:") + ): + continue + + canonical_url = urljoin(page_url, src) + if canonical_url in seen_urls: + continue + seen_urls.add(canonical_url) + + image = lhtml.Element("img") + image.set("src", src) + image.set("data-c4a-source", "css-background") + if alt := ( + styled_element.get("aria-label") + or styled_element.get("title") + ): + image.set("alt", alt) + styled_element.append(image) + def scrap(self, url: str, html: str, **kwargs) -> ScrapingResult: """ Main entry point for content scraping. @@ -310,6 +357,11 @@ def _process_element( self._log("error", f"Error processing link: {str(e)}", "SCRAPE") continue + # Materialize inline CSS backgrounds so they reach both media extraction + # and the cleaned HTML used by markdown generation. + if not kwargs.get("exclude_all_images", False): + self._materialize_inline_css_background_images(element, url) + # Process images images = element.xpath(".//img") total_images = len(images) @@ -464,6 +516,11 @@ def process_image( if picture := img.xpath("./ancestor::picture[1]"): score += 1 + if img.get("data-c4a-source") == "css-background": + # CSS backgrounds do not expose intrinsic dimensions or extensions, + # but an explicit background URL is itself a strong image signal. + score += 3 + if score <= kwargs.get("image_score_threshold", IMAGE_SCORE_THRESHOLD): return None diff --git a/tests/test_css_background_images.py b/tests/test_css_background_images.py new file mode 100644 index 000000000..cd95f2db0 --- /dev/null +++ b/tests/test_css_background_images.py @@ -0,0 +1,72 @@ +from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy +from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator + + +def test_inline_css_background_images_are_preserved_for_markdown_and_media(): + image_url = "https://cdn.example.com/property/photo-34753.jpg" + html = f""" + + +
+

Property listing

+
+
+
+ + + """ + + result = LXMLWebScrapingStrategy()._scrap( + url="https://example.com/listings/123", + html=html, + ) + + assert result["success"] is True + assert [image["src"] for image in result["media"]["images"]] == [image_url] + assert result["cleaned_html"].count(f'src="{image_url}"') == 1 + + markdown = DefaultMarkdownGenerator().generate_markdown( + result["cleaned_html"], + base_url="https://example.com/listings/123", + citations=False, + ) + assert image_url in markdown.raw_markdown + + +def test_css_background_images_respect_image_exclusions(): + image_url = "https://cdn.external.test/property.jpg" + html = f'
' + strategy = LXMLWebScrapingStrategy() + + external_result = strategy._scrap( + url="https://example.com/listings/123", + html=html, + exclude_external_images=True, + ) + all_images_result = strategy._scrap( + url="https://example.com/listings/123", + html=html, + exclude_all_images=True, + ) + + for result in (external_result, all_images_result): + assert result["media"]["images"] == [] + assert image_url not in result["cleaned_html"] + + +def test_css_background_images_do_not_require_file_extensions(): + image_urls = [ + "https://cdn.example.com/media/34753", + "https://cdn.example.com/media/34754", + ] + html = "".join( + f'
' + for image_url in image_urls + ) + + result = LXMLWebScrapingStrategy()._scrap( + url="https://example.com/listings/123", + html=html, + ) + + assert [image["src"] for image in result["media"]["images"]] == image_urls From c4c483a74d24d55eb5e3886e943f7802a61d13a2 Mon Sep 17 00:00:00 2001 From: itsafal <141961256+itsafal@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:35:55 -0400 Subject: [PATCH 2/2] fix: harden CSS background image extraction --- crawl4ai/content_scraping_strategy.py | 107 +++++++++++++++++++------- tests/test_css_background_images.py | 56 ++++++++++++++ 2 files changed, 135 insertions(+), 28 deletions(-) diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py index 1e8300a6e..054350425 100644 --- a/crawl4ai/content_scraping_strategy.py +++ b/crawl4ai/content_scraping_strategy.py @@ -36,12 +36,7 @@ OG_REGEX = re.compile(r"^og:") TWITTER_REGEX = re.compile(r"^twitter:") DIMENSION_REGEX = re.compile(r"(\d+)(\D*)") -CSS_BACKGROUND_DECLARATION_REGEX = re.compile( - r"(?:^|;)\s*background(?:-image)?\s*:\s*([^;]+)", re.IGNORECASE -) -CSS_URL_REGEX = re.compile( - r"url\(\s*(?:([\"'])(.*?)\1|([^)]*?))\s*\)", re.IGNORECASE -) +CSS_URL_START_REGEX = re.compile(r"\burl\s*\(", re.IGNORECASE) # Function to parse srcset @@ -141,30 +136,86 @@ def _materialize_inline_css_background_images( for styled_element in styled_elements: style = styled_element.get("style", "") - for declaration in CSS_BACKGROUND_DECLARATION_REGEX.findall(style): - for quote, quoted_url, unquoted_url in CSS_URL_REGEX.findall( - declaration + for src in self._extract_css_background_urls(style): + if src.lower().startswith(("data:", "blob:", "javascript:")): + continue + + canonical_url = urljoin(page_url, src) + if canonical_url in seen_urls: + continue + seen_urls.add(canonical_url) + + image = lhtml.Element("img") + image.set("src", src) + image.set("data-c4a-source", "css-background") + if alt := ( + styled_element.get("aria-label") + or styled_element.get("title") ): - src = (quoted_url if quote else unquoted_url).strip() - if not src or src.lower().startswith( - ("data:", "blob:", "javascript:") - ): - continue + image.set("alt", alt) + styled_element.append(image) + + @staticmethod + def _extract_css_background_urls(style: str) -> List[str]: + """Extract URLs from background declarations without splitting quoted values.""" + declarations = [] + start = 0 + quote = None + escaped = False + depth = 0 + + for index, char in enumerate(style): + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif quote: + if char == quote: + quote = None + elif char in "\"'": + quote = char + elif char == "(": + depth += 1 + elif char == ")": + depth = max(0, depth - 1) + elif char == ";" and depth == 0: + declarations.append(style[start:index]) + start = index + 1 + declarations.append(style[start:]) + + urls = [] + for declaration in declarations: + property_end = declaration.find(":") + if property_end < 0: + continue + property_name = declaration[:property_end].strip().lower() + if property_name not in {"background", "background-image"}: + continue - canonical_url = urljoin(page_url, src) - if canonical_url in seen_urls: - continue - seen_urls.add(canonical_url) - - image = lhtml.Element("img") - image.set("src", src) - image.set("data-c4a-source", "css-background") - if alt := ( - styled_element.get("aria-label") - or styled_element.get("title") - ): - image.set("alt", alt) - styled_element.append(image) + value = declaration[property_end + 1 :] + for match in CSS_URL_START_REGEX.finditer(value): + opening_paren = match.end() - 1 + quote = None + escaped = False + for index in range(opening_paren + 1, len(value)): + char = value[index] + if escaped: + escaped = False + elif char == "\\": + escaped = True + elif quote: + if char == quote: + quote = None + elif char in "\"'": + quote = char + elif char == ")": + src = value[opening_paren + 1 : index].strip() + if len(src) >= 2 and src[0] in "\"'" and src[-1] == src[0]: + src = src[1:-1] + if src: + urls.append(src.strip()) + break + return urls def scrap(self, url: str, html: str, **kwargs) -> ScrapingResult: """ diff --git a/tests/test_css_background_images.py b/tests/test_css_background_images.py index cd95f2db0..b3f8e4f4d 100644 --- a/tests/test_css_background_images.py +++ b/tests/test_css_background_images.py @@ -70,3 +70,59 @@ def test_css_background_images_do_not_require_file_extensions(): ) assert [image["src"] for image in result["media"]["images"]] == image_urls + + +def test_css_background_images_skip_non_http_schemes(): + html = """ +
+
+
+ """ + + result = LXMLWebScrapingStrategy()._scrap( + url="https://example.com/listings/123", + html=html, + ) + + assert result["media"]["images"] == [] + assert "', + ) + + assert [image["src"] for image in result["media"]["images"]] == [image_url] + + +def test_css_background_images_deduplicate_repeated_urls_globally(): + image_url = "https://cdn.example.com/property.jpg" + result = LXMLWebScrapingStrategy()._scrap( + url="https://example.com/listings/123", + html=( + f'
' + f'' + ), + ) + + assert [image["src"] for image in result["media"]["images"]] == [image_url]