Skip to content

Commit 572960c

Browse files
committed
feat(tools): add web_fetch builtin tool (HTTP fetch + readable extraction)
A plain-HTTP web fetch tool at veadk/tools/builtin_tools/web_fetch.py, modeled on OpenClaw's web_fetch: - HTTP GET (no JavaScript), Chrome-like headers, http(s) only. - SSRF guard: rejects private / loopback / link-local / reserved / multicast addresses and re-validates every redirect, including <meta refresh> hops (bounded by a 3-hop budget). - Extraction: HTML -> markdown or plain text (coarse, dependency-free) and PDF -> text via pypdf (by content-type or %PDF magic bytes). - Limits: 2MB download cap (10MB for PDFs), 30s timeout, max_chars truncation, 15-minute in-process TTL cache. - Returns {"url", "title", "content", "truncated"}, or {"error": ...} on failure.
1 parent f58924a commit 572960c

1 file changed

Lines changed: 378 additions & 0 deletions

File tree

Lines changed: 378 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,378 @@
1+
# Copyright (c) 2025 Beijing Volcano Engine Technology Co., Ltd. and/or its affiliates.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""A plain-HTTP web fetch tool that extracts readable content.
16+
17+
Does a plain HTTP GET and converts the HTML into bounded markdown/text. It does
18+
NOT execute JavaScript — for JS-heavy / login-protected pages a headless browser
19+
is needed instead. The design mirrors OpenClaw's `web_fetch`: Chrome-like
20+
headers, SSRF protection (block private/internal addresses, re-validate every
21+
redirect), a download-size cap, a coarse HTML→markdown extractor, and a short
22+
in-memory cache.
23+
"""
24+
25+
import html as _html
26+
import ipaddress
27+
import re
28+
import socket
29+
import time
30+
from urllib.parse import urljoin, urlparse
31+
32+
import requests
33+
34+
from google.adk.tools import ToolContext
35+
36+
from veadk.utils.logger import get_logger
37+
38+
logger = get_logger(__name__)
39+
40+
# ---- limits / defaults (mirror OpenClaw's tools.web.fetch.* config) ----------
41+
_USER_AGENT = (
42+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
43+
"(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
44+
)
45+
_ACCEPT_LANGUAGE = "en-US,en;q=0.9"
46+
_TIMEOUT_SECONDS = 30
47+
_MAX_REDIRECTS = 3
48+
_MAX_RESPONSE_BYTES = 2_000_000 # cap the download before truncation (HTML)
49+
_MAX_PDF_BYTES = 10_000_000 # higher cap for PDFs (truncation corrupts parsing)
50+
_MAX_CHARS_CAP = 200_000 # hard ceiling for the max_chars parameter
51+
_DEFAULT_MAX_CHARS = 50_000
52+
_CACHE_TTL_SECONDS = 15 * 60
53+
_CACHE_MAX_ENTRIES = 128
54+
55+
# Tiny in-process TTL cache: {(url, mode, max_chars): (expires_at, result)}.
56+
_CACHE: dict[tuple[str, str, int], tuple[float, dict]] = {}
57+
58+
59+
class _WebFetchError(Exception):
60+
"""Internal: surfaced to the model as {"error": ...}."""
61+
62+
63+
# ---------------------------------------------------------------- SSRF guard --
64+
def _assert_public_host(host: str) -> None:
65+
"""Resolve `host` and reject private/internal/loopback/link-local targets.
66+
67+
Note: this validates the resolved addresses but does not pin the connection
68+
to them, so a determined attacker controlling DNS could still race the
69+
re-resolution (TOCTOU). It blocks the common SSRF cases; pinning the socket
70+
to the validated IP would be the hardening follow-up.
71+
"""
72+
if not host:
73+
raise _WebFetchError("missing host")
74+
try:
75+
infos = socket.getaddrinfo(host, None)
76+
except socket.gaierror as e:
77+
raise _WebFetchError(f"DNS resolution failed for {host!r}: {e}")
78+
for info in infos:
79+
addr = info[4][0]
80+
try:
81+
ip = ipaddress.ip_address(addr)
82+
except ValueError:
83+
continue
84+
if (
85+
ip.is_private
86+
or ip.is_loopback
87+
or ip.is_link_local
88+
or ip.is_reserved
89+
or ip.is_multicast
90+
or ip.is_unspecified
91+
):
92+
raise _WebFetchError(f"Blocked non-public address for host {host!r}: {ip}")
93+
94+
95+
def _check_url(url: str) -> str:
96+
parsed = urlparse(url)
97+
if parsed.scheme not in ("http", "https"):
98+
raise _WebFetchError("only http(s) URLs are supported")
99+
if not parsed.hostname:
100+
raise _WebFetchError("URL has no host")
101+
_assert_public_host(parsed.hostname)
102+
return parsed.hostname
103+
104+
105+
# ---------------------------------------------- coarse HTML -> markdown/text --
106+
def _normalize_whitespace(value: str) -> str:
107+
value = value.replace("\r", "")
108+
value = re.sub(r"[ \t]+\n", "\n", value)
109+
value = re.sub(r"\n{3,}", "\n\n", value)
110+
value = re.sub(r"[ \t]{2,}", " ", value)
111+
return value.strip()
112+
113+
114+
def _strip_tags(value: str) -> str:
115+
return _html.unescape(re.sub(r"<[^>]+>", "", value))
116+
117+
118+
def _html_to_markdown(html_text: str) -> tuple[str, str | None]:
119+
"""Coarse HTML→markdown (ported from OpenClaw's `htmlToMarkdown`)."""
120+
title_match = re.search(
121+
r"<title[^>]*>([\s\S]*?)</title>", html_text, flags=re.IGNORECASE
122+
)
123+
title = (
124+
_normalize_whitespace(_strip_tags(title_match.group(1)))
125+
if title_match
126+
else None
127+
)
128+
129+
text = re.sub(r"<script[\s\S]*?</script>", "", html_text, flags=re.IGNORECASE)
130+
text = re.sub(r"<style[\s\S]*?</style>", "", text, flags=re.IGNORECASE)
131+
text = re.sub(r"<noscript[\s\S]*?</noscript>", "", text, flags=re.IGNORECASE)
132+
133+
# Preserve link targets so fetched pages stay source-auditable.
134+
def _link(m: re.Match) -> str:
135+
href, body = m.group(1), m.group(2)
136+
label = _normalize_whitespace(_strip_tags(body))
137+
return f"[{label}]({href})" if label else href
138+
139+
text = re.sub(
140+
r"<a\s+[^>]*href=[\"']([^\"']+)[\"'][^>]*>([\s\S]*?)</a>",
141+
_link,
142+
text,
143+
flags=re.IGNORECASE,
144+
)
145+
146+
def _heading(m: re.Match) -> str:
147+
level = max(1, min(6, int(m.group(1))))
148+
label = _normalize_whitespace(_strip_tags(m.group(2)))
149+
return f"\n{'#' * level} {label}\n"
150+
151+
text = re.sub(
152+
r"<h([1-6])[^>]*>([\s\S]*?)</h\1>", _heading, text, flags=re.IGNORECASE
153+
)
154+
155+
def _li(m: re.Match) -> str:
156+
label = _normalize_whitespace(_strip_tags(m.group(1)))
157+
return f"\n- {label}" if label else ""
158+
159+
text = re.sub(r"<li[^>]*>([\s\S]*?)</li>", _li, text, flags=re.IGNORECASE)
160+
text = re.sub(r"<(br|hr)\s*/?>", "\n", text, flags=re.IGNORECASE)
161+
text = re.sub(
162+
r"</(p|div|section|article|header|footer|table|tr|ul|ol)>",
163+
"\n",
164+
text,
165+
flags=re.IGNORECASE,
166+
)
167+
text = _strip_tags(text)
168+
return _normalize_whitespace(text), title
169+
170+
171+
def _meta_refresh_url(html_text: str) -> str | None:
172+
"""Return the target of a `<meta http-equiv="refresh" content="0;url=...">`
173+
tag, if present (the JS-free redirect used by shell pages like sina.com)."""
174+
m = re.search(
175+
r"<meta[^>]+http-equiv=[\"']?refresh[\"']?[^>]*content=[\"']?[^\"'>]*?url=([^\"'>\s]+)",
176+
html_text[:4096],
177+
flags=re.IGNORECASE,
178+
)
179+
return _html.unescape(m.group(1)) if m else None
180+
181+
182+
def _extract_pdf_text(raw: bytes) -> tuple[str, str | None] | None:
183+
"""Extract text from PDF bytes via pypdf. Returns (text, title), or None if
184+
pypdf isn't installed."""
185+
try:
186+
import io
187+
188+
from pypdf import PdfReader
189+
except ImportError:
190+
return None
191+
reader = PdfReader(io.BytesIO(raw))
192+
parts: list[str] = []
193+
for page in reader.pages:
194+
try:
195+
parts.append(page.extract_text() or "")
196+
except Exception: # noqa: BLE001 - skip unparseable pages
197+
continue
198+
title = None
199+
try:
200+
if reader.metadata and reader.metadata.title:
201+
title = str(reader.metadata.title)
202+
except Exception: # noqa: BLE001
203+
title = None
204+
return _normalize_whitespace("\n\n".join(parts)), title
205+
206+
207+
def _markdown_to_text(markdown: str) -> str:
208+
text = re.sub(r"!\[[^\]]*]\([^)]+\)", "", markdown) # images
209+
text = re.sub(r"\[([^\]]+)]\([^)]+\)", r"\1", text) # links -> label
210+
text = re.sub(r"`([^`]+)`", r"\1", text) # inline code
211+
text = re.sub(r"^#{1,6}\s+", "", text, flags=re.MULTILINE) # headings
212+
text = re.sub(r"^\s*[-*+]\s+", "", text, flags=re.MULTILINE) # bullets
213+
text = re.sub(r"^\s*\d+\.\s+", "", text, flags=re.MULTILINE) # numbered
214+
return _normalize_whitespace(text)
215+
216+
217+
# ----------------------------------------------------------------- the tool --
218+
def web_fetch(
219+
url: str,
220+
extract_mode: str = "markdown",
221+
max_chars: int = _DEFAULT_MAX_CHARS,
222+
tool_context: ToolContext | None = None,
223+
) -> dict:
224+
"""Fetch a web page over HTTP(S) and return its readable main content.
225+
226+
Performs a plain HTTP GET (no JavaScript execution) and extracts the page's
227+
readable text. Handles HTML pages (converted to markdown/text) and **PDF**
228+
URLs (text extracted via pypdf). Follows HTTP and `<meta refresh>` redirects.
229+
Use it to read articles, docs, or any public URL the user references. For
230+
pages that require login or render entirely via JavaScript, the content may
231+
be incomplete.
232+
233+
Args:
234+
url: The http(s) URL to fetch.
235+
extract_mode: "markdown" (default, keeps headings/links/lists) or "text"
236+
(plain text with markdown decoration removed).
237+
max_chars: Truncate the extracted content to at most this many characters.
238+
239+
Returns:
240+
A dict with keys: "url" (final URL after redirects), "title", "content",
241+
and "truncated" (bool). On failure, a dict with an "error" key.
242+
"""
243+
mode = extract_mode if extract_mode in ("markdown", "text") else "markdown"
244+
try:
245+
max_chars = int(max_chars)
246+
except (TypeError, ValueError):
247+
max_chars = _DEFAULT_MAX_CHARS
248+
max_chars = max(1, min(max_chars, _MAX_CHARS_CAP))
249+
250+
cache_key = (url, mode, max_chars)
251+
cached = _CACHE.get(cache_key)
252+
if cached and cached[0] > time.monotonic():
253+
return cached[1]
254+
255+
try:
256+
result = _fetch_and_extract(url, mode, max_chars)
257+
except _WebFetchError as e:
258+
logger.warning(f"web_fetch failed for {url!r}: {e}")
259+
return {"error": str(e)}
260+
except requests.RequestException as e:
261+
logger.warning(f"web_fetch request error for {url!r}: {e}")
262+
return {"error": f"request failed: {e}"}
263+
264+
# Cache + bound the cache size.
265+
if len(_CACHE) >= _CACHE_MAX_ENTRIES:
266+
_CACHE.clear()
267+
_CACHE[cache_key] = (time.monotonic() + _CACHE_TTL_SECONDS, result)
268+
return result
269+
270+
271+
def _read_capped(response: requests.Response, cap: int) -> bytes:
272+
chunks: list[bytes] = []
273+
total = 0
274+
for chunk in response.iter_content(chunk_size=16_384):
275+
if not chunk:
276+
continue
277+
chunks.append(chunk)
278+
total += len(chunk)
279+
if total >= cap:
280+
break
281+
return b"".join(chunks)
282+
283+
284+
def _result(url: str, title: str | None, content: str, max_chars: int) -> dict:
285+
return {
286+
"url": url,
287+
"title": title,
288+
"content": content[:max_chars],
289+
"truncated": len(content) > max_chars,
290+
}
291+
292+
293+
def _fetch_and_extract(url: str, mode: str, max_chars: int) -> dict:
294+
session = requests.Session()
295+
headers = {
296+
"User-Agent": _USER_AGENT,
297+
"Accept-Language": _ACCEPT_LANGUAGE,
298+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
299+
}
300+
301+
current = url
302+
hops = 0
303+
# Follow HTTP 3xx AND <meta refresh> redirects, both bounded by the same hop
304+
# budget, SSRF-checking every hop.
305+
while True:
306+
_check_url(current)
307+
response = session.get(
308+
current,
309+
headers=headers,
310+
timeout=_TIMEOUT_SECONDS,
311+
allow_redirects=False,
312+
stream=True,
313+
)
314+
315+
if response.is_redirect and response.headers.get("location"):
316+
response.close()
317+
hops += 1
318+
if hops > _MAX_REDIRECTS:
319+
raise _WebFetchError(f"too many redirects (>{_MAX_REDIRECTS})")
320+
current = urljoin(current, response.headers["location"])
321+
continue
322+
323+
if not response.ok:
324+
status = response.status_code
325+
response.close()
326+
raise _WebFetchError(f"HTTP {status}")
327+
328+
content_type = response.headers.get("content-type", "")
329+
is_pdf = "pdf" in content_type.lower()
330+
raw = _read_capped(response, _MAX_PDF_BYTES if is_pdf else _MAX_RESPONSE_BYTES)
331+
response.close()
332+
333+
# PDF (by content-type or magic bytes) -> extract text via pypdf.
334+
if is_pdf or raw[:5] == b"%PDF-":
335+
extracted = _extract_pdf_text(raw)
336+
if extracted is None:
337+
return {
338+
"url": current,
339+
"title": None,
340+
"content": "[PDF detected — text extraction requires `pypdf`]",
341+
"truncated": False,
342+
}
343+
text, title = extracted
344+
return _result(
345+
current, title, text or "[PDF had no extractable text]", max_chars
346+
)
347+
348+
# Decode: honor charset from the header, else fall back to utf-8.
349+
charset = "utf-8"
350+
m = re.search(r"charset=([\w-]+)", content_type, flags=re.IGNORECASE)
351+
if m:
352+
charset = m.group(1)
353+
try:
354+
body = raw.decode(charset, errors="replace")
355+
except LookupError:
356+
body = raw.decode("utf-8", errors="replace")
357+
358+
is_html = "html" in content_type.lower() or "xml" in content_type.lower()
359+
360+
# Honor <meta refresh> redirects on shell pages (e.g. sina.com).
361+
if is_html and hops < _MAX_REDIRECTS:
362+
meta = _meta_refresh_url(body)
363+
if meta:
364+
nxt = urljoin(current, meta)
365+
if nxt != current:
366+
hops += 1
367+
current = nxt
368+
continue
369+
370+
if not is_html:
371+
# Other text-ish content: return as-is (truncated).
372+
return _result(current, None, _normalize_whitespace(body), max_chars)
373+
374+
markdown, title = _html_to_markdown(body)
375+
content = markdown if mode == "markdown" else _markdown_to_text(markdown)
376+
if not content:
377+
content = _normalize_whitespace(_strip_tags(body))
378+
return _result(current, title, content, max_chars)

0 commit comments

Comments
 (0)