fix: preserve inline CSS background images

crawl4ai/content_scraping_strategy.py

@@ -36,6 +36,7 @@
 OG_REGEX = re.compile(r"^og:")
 TWITTER_REGEX = re.compile(r"^twitter:")
 DIMENSION_REGEX = re.compile(r"(\d+)(\D*)")
+CSS_URL_START_REGEX = re.compile(r"\burl\s*\(", re.IGNORECASE)
 
 
 # Function to parse srcset
@@ -119,6 +120,103 @@ def _log(self, level, message, tag="SCRAPE", **kwargs):
             log_method = getattr(self.logger, level)
             log_method(message=message, tag=tag, **kwargs)
 
+    def _materialize_inline_css_background_images(
+        self, element: lhtml.HtmlElement, page_url: str
+    ) -> None:
+        """Convert inline CSS background URLs into images for media and markdown."""
+        seen_urls = {
+            urljoin(page_url, src.strip())
+            for src in element.xpath(".//img/@src")
+            if src and src.strip()
+        }
+
+        styled_elements = element.xpath(".//*[@style]")
+        if element.get("style"):
+            styled_elements.insert(0, element)
+
+        for styled_element in styled_elements:
+            style = styled_element.get("style", "")
+            for src in self._extract_css_background_urls(style):
+                if src.lower().startswith(("data:", "blob:", "javascript:")):
+                    continue
+
+                canonical_url = urljoin(page_url, src)
+                if canonical_url in seen_urls:
+                    continue
+                seen_urls.add(canonical_url)
+
+                image = lhtml.Element("img")
+                image.set("src", src)
+                image.set("data-c4a-source", "css-background")
+                if alt := (
+                    styled_element.get("aria-label")
+                    or styled_element.get("title")
+                ):
+                    image.set("alt", alt)
+                styled_element.append(image)
+
+    @staticmethod
+    def _extract_css_background_urls(style: str) -> List[str]:
+        """Extract URLs from background declarations without splitting quoted values."""
+        declarations = []
+        start = 0
+        quote = None
+        escaped = False
+        depth = 0
+
+        for index, char in enumerate(style):
+            if escaped:
+                escaped = False
+            elif char == "\\":
+                escaped = True
+            elif quote:
+                if char == quote:
+                    quote = None
+            elif char in "\"'":
+                quote = char
+            elif char == "(":
+                depth += 1
+            elif char == ")":
+                depth = max(0, depth - 1)
+            elif char == ";" and depth == 0:
+                declarations.append(style[start:index])
+                start = index + 1
+        declarations.append(style[start:])
+
+        urls = []
+        for declaration in declarations:
+            property_end = declaration.find(":")
+            if property_end < 0:
+                continue
+            property_name = declaration[:property_end].strip().lower()
+            if property_name not in {"background", "background-image"}:
+                continue
+
+            value = declaration[property_end + 1 :]
+            for match in CSS_URL_START_REGEX.finditer(value):
+                opening_paren = match.end() - 1
+                quote = None
+                escaped = False
+                for index in range(opening_paren + 1, len(value)):
+                    char = value[index]
+                    if escaped:
+                        escaped = False
+                    elif char == "\\":
+                        escaped = True
+                    elif quote:
+                        if char == quote:
+                            quote = None
+                    elif char in "\"'":
+                        quote = char
+                    elif char == ")":
+                        src = value[opening_paren + 1 : index].strip()
+                        if len(src) >= 2 and src[0] in "\"'" and src[-1] == src[0]:
+                            src = src[1:-1]
+                        if src:
+                            urls.append(src.strip())
+                        break
+        return urls
+
     def scrap(self, url: str, html: str, **kwargs) -> ScrapingResult:
         """
         Main entry point for content scraping.
@@ -310,6 +408,11 @@ def _process_element(
                 self._log("error", f"Error processing link: {str(e)}", "SCRAPE")
                 continue
 
+        # Materialize inline CSS backgrounds so they reach both media extraction
+        # and the cleaned HTML used by markdown generation.
+        if not kwargs.get("exclude_all_images", False):
+            self._materialize_inline_css_background_images(element, url)
+
         # Process images
         images = element.xpath(".//img")
         total_images = len(images)
@@ -464,6 +567,11 @@ def process_image(
         if picture := img.xpath("./ancestor::picture[1]"):
             score += 1
 
+        if img.get("data-c4a-source") == "css-background":
+            # CSS backgrounds do not expose intrinsic dimensions or extensions,
+            # but an explicit background URL is itself a strong image signal.
+            score += 3
+
         if score <= kwargs.get("image_score_threshold", IMAGE_SCORE_THRESHOLD):
             return None
 

tests/test_css_background_images.py

@@ -0,0 +1,128 @@
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
+from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
+
+
+def test_inline_css_background_images_are_preserved_for_markdown_and_media():
+    image_url = "https://cdn.example.com/property/photo-34753.jpg"
+    html = f"""
+    <html>
+      <body>
+        <main>
+          <h1>Property listing</h1>
+          <div class="slick-slide" style="background: url('{image_url}') center / cover no-repeat"></div>
+          <div class="slick-slide slick-cloned" style='background-image: url("{image_url}")'></div>
+        </main>
+      </body>
+    </html>
+    """
+
+    result = LXMLWebScrapingStrategy()._scrap(
+        url="https://example.com/listings/123",
+        html=html,
+    )
+
+    assert result["success"] is True
+    assert [image["src"] for image in result["media"]["images"]] == [image_url]
+    assert result["cleaned_html"].count(f'src="{image_url}"') == 1
+
+    markdown = DefaultMarkdownGenerator().generate_markdown(
+        result["cleaned_html"],
+        base_url="https://example.com/listings/123",
+        citations=False,
+    )
+    assert image_url in markdown.raw_markdown
+
+
+def test_css_background_images_respect_image_exclusions():
+    image_url = "https://cdn.external.test/property.jpg"
+    html = f'<div style="background-image: url({image_url})"></div>'
+    strategy = LXMLWebScrapingStrategy()
+
+    external_result = strategy._scrap(
+        url="https://example.com/listings/123",
+        html=html,
+        exclude_external_images=True,
+    )
+    all_images_result = strategy._scrap(
+        url="https://example.com/listings/123",
+        html=html,
+        exclude_all_images=True,
+    )
+
+    for result in (external_result, all_images_result):
+        assert result["media"]["images"] == []
+        assert image_url not in result["cleaned_html"]
+
+
+def test_css_background_images_do_not_require_file_extensions():
+    image_urls = [
+        "https://cdn.example.com/media/34753",
+        "https://cdn.example.com/media/34754",
+    ]
+    html = "".join(
+        f'<div style="background: url({image_url}) center/cover no-repeat"></div>'
+        for image_url in image_urls
+    )
+
+    result = LXMLWebScrapingStrategy()._scrap(
+        url="https://example.com/listings/123",
+        html=html,
+    )
+
+    assert [image["src"] for image in result["media"]["images"]] == image_urls
+
+
+def test_css_background_images_skip_non_http_schemes():
+    html = """
+    <div style="background: url(data:image/png;base64,abc)"></div>
+    <div style="background-image: url('blob:https://example.com/id')"></div>
+    <div style="background: url(javascript:alert(1))"></div>
+    """
+
+    result = LXMLWebScrapingStrategy()._scrap(
+        url="https://example.com/listings/123",
+        html=html,
+    )
+
+    assert result["media"]["images"] == []
+    assert "<img" not in result["cleaned_html"]
+
+
+def test_css_background_images_resolve_relative_urls_for_markdown():
+    result = LXMLWebScrapingStrategy()._scrap(
+        url="https://example.com/listings/123",
+        html='<div style="background-image: url(../images/house.jpg)"></div>',
+    )
+
+    assert [image["src"] for image in result["media"]["images"]] == [
+        "../images/house.jpg"
+    ]
+    markdown = DefaultMarkdownGenerator().generate_markdown(
+        result["cleaned_html"],
+        base_url="https://example.com/listings/123",
+        citations=False,
+    )
+    assert "https://example.com/images/house.jpg" in markdown.raw_markdown
+
+
+def test_css_background_images_allow_semicolons_in_quoted_urls():
+    image_url = "https://cdn.example.com/property.jpg?variant=hero;size=large"
+    result = LXMLWebScrapingStrategy()._scrap(
+        url="https://example.com/listings/123",
+        html=f'<div style="background-image: url(\'{image_url}\')"></div>',
+    )
+
+    assert [image["src"] for image in result["media"]["images"]] == [image_url]
+
+
+def test_css_background_images_deduplicate_repeated_urls_globally():
+    image_url = "https://cdn.example.com/property.jpg"
+    result = LXMLWebScrapingStrategy()._scrap(
+        url="https://example.com/listings/123",
+        html=(
+            f'<section style="background: url({image_url})"></section>'
+            f'<aside style="background-image: url({image_url})"></aside>'
+        ),
+    )
+
+    assert [image["src"] for image in result["media"]["images"]] == [image_url]