From c7fcb1f72f8ba3258220dc48daf12301c876cfe2 Mon Sep 17 00:00:00 2001 From: Zsanz3 <268133725+Zsanz3@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:54:46 +0000 Subject: [PATCH 1/3] fix(robots): Disallow /*? should only match URLs with query strings urllib.robotparser normalizes rule paths with urlparse/urlunparse, which drops an empty query. Combined with the wildcard patch, Disallow: /*? collapsed to Disallow: /* and blocked the entire site. Preserve the trailing ? on RuleLine paths and require a query marker before treating those rules as a match. Fixes unclecode/crawl4ai#2225 Co-authored-by: Zsanz3 --- crawl4ai/utils.py | 54 +++++++++++++++++++++++------- tests/general/test_robot_parser.py | 41 ++++++++++++++++++++++- 2 files changed, 81 insertions(+), 14 deletions(-) diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 279c27708..137bc8213 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -55,21 +55,43 @@ import re original_applies_to = RuleLine.applies_to +original_ruleline_init = RuleLine.__init__ + + +def _has_query_marker(value: str) -> bool: + """True if a quoted or raw robots path/URL still carries a '?' query marker.""" + if not value: + return False + return "?" in value or "%3F" in value.upper() + + +def patched_ruleline_init(self, path, allowance): + # urlparse/urlunparse drops an empty query, so '/*?' collapses to '/*'. + # Preserve a trailing '?' so query-only rules keep their meaning (#2225). + keep_query_marker = isinstance(path, str) and path.endswith("?") + original_ruleline_init(self, path, allowance) + if keep_query_marker and not _has_query_marker(self.path): + self.path += "%3F" + def patched_applies_to(self, filename): - # Handle wildcards in paths - if '*' in self.path or '%2A' in self.path or self.path in ("*", "%2A"): - pattern = self.path.replace('%2A', '*') - pattern = re.escape(pattern).replace('\\*', '.*') - pattern = '^' + pattern - if pattern.endswith('\\$'): - pattern = pattern[:-2] + '$' - try: - return bool(re.match(pattern, filename)) - except re.error: - return original_applies_to(self, filename) - return original_applies_to(self, filename) + # Disallow: /*? (and similar) must only match URLs that actually carry a query. + if _has_query_marker(self.path) and not _has_query_marker(filename): + return False + # Handle wildcards in paths + if '*' in self.path or '%2A' in self.path or self.path in ("*", "%2A"): + pattern = self.path.replace('%2A', '*') + pattern = re.escape(pattern).replace('\\*', '.*') + pattern = '^' + pattern + if pattern.endswith('\\$'): + pattern = pattern[:-2] + '$' + try: + return bool(re.match(pattern, filename)) + except re.error: + return original_applies_to(self, filename) + return original_applies_to(self, filename) +RuleLine.__init__ = patched_ruleline_init RuleLine.applies_to = patched_applies_to # Monkey patch ends @@ -361,7 +383,13 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool: # If parser can't read rules, allow access if not parser.mtime(): return True - + + # urllib.robotparser drops an empty query component, so a URL like + # https://host/page? is normalized to /page. Keep a query marker so + # rules such as Disallow: /*? still see that the URL carries '?'. + if "?" in url and not parsed.query: + url = f"{url}=" + return parser.can_fetch(user_agent, url) def clear_cache(self): diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.py index a2fc30f1a..bb57dba79 100644 --- a/tests/general/test_robot_parser.py +++ b/tests/general/test_robot_parser.py @@ -1,5 +1,6 @@ from crawl4ai.utils import RobotsParser - +from urllib.robotparser import RobotFileParser + import asyncio import aiohttp from aiohttp import web @@ -8,6 +9,12 @@ import os, sys, time, json +# Ecommerce-style rule: block URLs that carry a query string, not the whole site. +DISALLOW_QUERY_WILDCARD_ROBOTS = """User-agent: * +Disallow: /*? +""" + + async def test_robots_parser(): print("\n=== Testing RobotsParser ===\n") @@ -148,8 +155,40 @@ async def giant_robots(request): shutil.rmtree(temp_dir) print("\nTest cleanup completed") +def test_robotfileparser_disallow_query_wildcard(): + """RuleLine patch: Disallow: /*? only matches URLs that carry a query string.""" + parser = RobotFileParser() + parser.parse(DISALLOW_QUERY_WILDCARD_ROBOTS.splitlines()) + assert parser.can_fetch("*", "https://shop.example/page") is True + assert parser.can_fetch("*", "https://shop.example/") is True + assert parser.can_fetch("*", "https://shop.example/products/shoes") is True + assert parser.can_fetch("*", "https://shop.example/page?q=1") is False + assert parser.can_fetch("*", "https://shop.example/?s=search") is False + + +async def test_disallow_query_wildcard_only_blocks_query_urls(): + """RobotsParser.can_fetch: Disallow: /*? must not collapse to Disallow: /* (#2225).""" + temp_dir = tempfile.mkdtemp() + try: + parser = RobotsParser(cache_dir=temp_dir) + parser._cache_rules("shop.example", DISALLOW_QUERY_WILDCARD_ROBOTS) + + assert await parser.can_fetch("https://shop.example/page", "*") is True + assert await parser.can_fetch("https://shop.example/", "*") is True + assert await parser.can_fetch("https://shop.example/products/shoes", "*") is True + assert await parser.can_fetch("https://shop.example/page?q=1", "*") is False + assert await parser.can_fetch("https://shop.example/?s=search", "*") is False + # Empty-but-present query still counts as carrying a query string + assert await parser.can_fetch("https://shop.example/page?", "*") is False + print("✓ Disallow: /*? blocks query URLs only (issue #2225)") + finally: + shutil.rmtree(temp_dir) + + async def main(): try: + test_robotfileparser_disallow_query_wildcard() + await test_disallow_query_wildcard_only_blocks_query_urls() await test_robots_parser() except Exception as e: print(f"Test failed: {str(e)}") From dc3a9e95fb087f12a3abaf7d90c4739dce194984 Mon Sep 17 00:00:00 2001 From: Zsanz3 <268133725+Zsanz3@users.noreply.github.com> Date: Wed, 2 Sep 2026 01:56:36 +0000 Subject: [PATCH 2/3] test(robots): mark Disallow /*? async test for pytest-asyncio Co-authored-by: Zsanz3 --- tests/general/test_robot_parser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.py index bb57dba79..91ce90581 100644 --- a/tests/general/test_robot_parser.py +++ b/tests/general/test_robot_parser.py @@ -7,6 +7,7 @@ import tempfile import shutil import os, sys, time, json +import pytest # Ecommerce-style rule: block URLs that carry a query string, not the whole site. @@ -166,6 +167,7 @@ def test_robotfileparser_disallow_query_wildcard(): assert parser.can_fetch("*", "https://shop.example/?s=search") is False +@pytest.mark.asyncio async def test_disallow_query_wildcard_only_blocks_query_urls(): """RobotsParser.can_fetch: Disallow: /*? must not collapse to Disallow: /* (#2225).""" temp_dir = tempfile.mkdtemp() From bd2f4d817d9dfbd62114d7d1582f4709251caf4b Mon Sep 17 00:00:00 2001 From: Zsanz3 <268133725+Zsanz3@users.noreply.github.com> Date: Wed, 2 Sep 2026 02:02:10 +0000 Subject: [PATCH 3/3] fix(robots): preserve empty query without corrupting fragments RobotsParser.can_fetch appended '=' to the raw URL, which turned page?#frag into page?#frag= and could treat a '?' that lives only in the fragment as a query. Rebuild via urlunparse when '?' is in the pre-# part so the query becomes '=' and the fragment is kept. Co-authored-by: Zsanz3 --- crawl4ai/utils.py | 9 +++++++-- tests/general/test_robot_parser.py | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crawl4ai/utils.py b/crawl4ai/utils.py index 137bc8213..b59ee6d4f 100644 --- a/crawl4ai/utils.py +++ b/crawl4ai/utils.py @@ -387,8 +387,13 @@ async def can_fetch(self, url: str, user_agent: str = "*") -> bool: # urllib.robotparser drops an empty query component, so a URL like # https://host/page? is normalized to /page. Keep a query marker so # rules such as Disallow: /*? still see that the URL carries '?'. - if "?" in url and not parsed.query: - url = f"{url}=" + # Only look at the pre-fragment part so "? in fragment" is not treated + # as a query, and rebuild via urlunparse so page?#frag becomes page?=#frag + # instead of corrupting the fragment. Bare RobotFileParser.can_fetch + # still allows page? — this rewrite is RobotsParser-only. + before_hash = url.split("#", 1)[0] + if "?" in before_hash and not parsed.query: + url = urlunparse(parsed._replace(query="=")) return parser.can_fetch(user_agent, url) diff --git a/tests/general/test_robot_parser.py b/tests/general/test_robot_parser.py index 91ce90581..1a120cf2c 100644 --- a/tests/general/test_robot_parser.py +++ b/tests/general/test_robot_parser.py @@ -182,6 +182,8 @@ async def test_disallow_query_wildcard_only_blocks_query_urls(): assert await parser.can_fetch("https://shop.example/?s=search", "*") is False # Empty-but-present query still counts as carrying a query string assert await parser.can_fetch("https://shop.example/page?", "*") is False + # page?#frag has a query marker before the fragment and must stay blocked + assert await parser.can_fetch("https://shop.example/page?#frag", "*") is False print("✓ Disallow: /*? blocks query URLs only (issue #2225)") finally: shutil.rmtree(temp_dir)