From 1ad9c8e3b74711ca7a42a75a6746054ac040f569 Mon Sep 17 00:00:00 2001 From: JuampiHernandez <102912458+JuampiHernandez@users.noreply.github.com> Date: Sun, 25 Oct 2026 22:45:16 +0000 Subject: [PATCH] feat(search): add Firecrawl as an opt-in keyed fallback search engine Adds a firecrawl adapter to the keyed fallback registry, after serply in the default SEARCH_FALLBACK_ORDER. POST https://api.firecrawl.dev/v2/search with the query in the JSON body and the key in an Authorization: Bearer header (neither in the URL), limit capped at 100 (the API's per-request max), highlights off so description is the plain result snippet, data.web[] title/url/description mapped to the shared title/href/body shape, non-http links skipped. A new _http_post_json helper mirrors _http_get_json (same timeout, returns None on any failure). Inert unless FIRECRAWL_API_KEY is set. Signed-off-by: JuampiHernandez <102912458+JuampiHernandez@users.noreply.github.com> --- .env.example | 1 + apps/api/services/leadgen/config.py | 1 + apps/api/services/leadgen/search_engines.py | 58 +++++++++++++++++++-- tests/test_search_cache.py | 4 +- tests/test_search_multi_engine.py | 51 ++++++++++++++++-- 5 files changed, 107 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 23b2d1291..7729d26dd 100644 --- a/.env.example +++ b/.env.example @@ -63,6 +63,7 @@ # GOOGLE_CSE_ID= # Google Custom Search Engine ID # SERPINGAPI_API_KEY= # Serping API key (Google SERP; keyed search fallback) — https://serpingapi.com # SERPLY_API_KEY= # Serply API key (Google SERP; keyed search fallback) — https://serply.io +# FIRECRAWL_API_KEY= # Firecrawl API key (web search; keyed search fallback) — https://www.firecrawl.dev/?utm_source=opengtm&utm_medium=integration # === Email Outreach (optional, also configurable in Settings) === # SMTP_HOST=smtp.gmail.com diff --git a/apps/api/services/leadgen/config.py b/apps/api/services/leadgen/config.py index bdd1b62a1..e4dd5570b 100644 --- a/apps/api/services/leadgen/config.py +++ b/apps/api/services/leadgen/config.py @@ -127,6 +127,7 @@ BRAVE_SEARCH_KEY = os.getenv("BRAVE_SEARCH_KEY", "") SERPINGAPI_API_KEY = os.getenv("SERPINGAPI_API_KEY", "") SERPLY_API_KEY = os.getenv("SERPLY_API_KEY", "") +FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY", "") # ── DDG result cache + adaptive backoff (see services/leadgen/search_cache.py) # In-process only (no DB). Both behaviours are behind flags, default ON; set diff --git a/apps/api/services/leadgen/search_engines.py b/apps/api/services/leadgen/search_engines.py index 4fb9ef56d..aaebba445 100644 --- a/apps/api/services/leadgen/search_engines.py +++ b/apps/api/services/leadgen/search_engines.py @@ -16,10 +16,10 @@ behavior is identical to today (DDG only). * Adapters never raise to the caller — a failing engine returns `[]` and the chain moves on. (Exceptions are logged at debug level.) - * Network access is wrapped in a tiny helper so tests can mock one place. + * Network access is wrapped in tiny GET/POST helpers so tests can mock them. Engine order (first configured one is tried first) is controlled by -SEARCH_FALLBACK_ORDER, default "serpapi,bing,google_cse,brave,serpingapi,serply". +SEARCH_FALLBACK_ORDER, default "serpapi,bing,google_cse,brave,serpingapi,serply,firecrawl". """ from __future__ import annotations @@ -33,7 +33,7 @@ _ENGINE_TIMEOUT = float(os.getenv("SEARCH_ENGINE_TIMEOUT", "10")) # Default attempt order for keyed fallback engines. -_DEFAULT_ORDER = "serpapi,bing,google_cse,brave,serpingapi,serply" +_DEFAULT_ORDER = "serpapi,bing,google_cse,brave,serpingapi,serply,firecrawl" # ── Config (all optional) ──────────────────────────────────────────────── @@ -65,6 +65,10 @@ def _serply_key() -> str: return _key("SERPLY_API_KEY") +def _firecrawl_key() -> str: + return _key("FIRECRAWL_API_KEY") + + # ── HTTP helper (single mockable seam) ─────────────────────────────────── def _http_get_json(url: str, params: Dict, headers: Optional[Dict] = None) -> Optional[Dict]: """GET `url` and return parsed JSON, or None on any failure. @@ -86,6 +90,26 @@ def _http_get_json(url: str, params: Dict, headers: Optional[Dict] = None) -> Op return None +def _http_post_json(url: str, body: Dict, headers: Optional[Dict] = None) -> Optional[Dict]: + """POST `body` as JSON to `url` and return parsed JSON, or None on any failure. + + Same contract as `_http_get_json`, for engines that take the query in the + request body instead of the URL. + """ + try: + import httpx + + with httpx.Client(timeout=_ENGINE_TIMEOUT) as client: + resp = client.post(url, json=body, headers=headers or {}) + if resp.status_code != 200: + logger.debug("search engine %s returned HTTP %s", url, resp.status_code) + return None + return resp.json() + except Exception as e: # noqa: BLE001 — adapters must never raise to caller + logger.debug("search engine POST %s failed: %s", url, e) + return None + + # ── Adapters ───────────────────────────────────────────────────────────── def _serpapi(query: str, max_results: int) -> List[Dict]: key = _serpapi_key() @@ -244,6 +268,33 @@ def _serply(query: str, max_results: int) -> List[Dict]: return out +def _firecrawl(query: str, max_results: int) -> List[Dict]: + key = _firecrawl_key() + if not key: + return [] + data = _http_post_json( + "https://api.firecrawl.dev/v2/search", + # Firecrawl caps `limit` at 100 per request. + {"query": query, "limit": min(max_results, 100), "highlights": False, "origin": "opengtm"}, + headers={"Authorization": f"Bearer {key}", "User-Agent": "OpenGTM"}, + ) + if not data: + return [] + out: List[Dict] = [] + for r in ((data.get("data") or {}).get("web") or []): + href = r.get("url") or "" + if not href.startswith(("http://", "https://")): + continue + out.append({ + "title": r.get("title", "") or "", + "href": href, + "body": r.get("description", "") or "", + }) + if len(out) >= max_results: + break + return out + + # name → (adapter, is_configured) registry _ADAPTERS: Dict[str, tuple[Callable[[str, int], List[Dict]], Callable[[], bool]]] = { "serpapi": (_serpapi, lambda: bool(_serpapi_key())), @@ -252,6 +303,7 @@ def _serply(query: str, max_results: int) -> List[Dict]: "brave": (_brave, lambda: bool(_brave_key())), "serpingapi": (_serpingapi, lambda: bool(_serpingapi_key())), "serply": (_serply, lambda: bool(_serply_key())), + "firecrawl": (_firecrawl, lambda: bool(_firecrawl_key())), } diff --git a/tests/test_search_cache.py b/tests/test_search_cache.py index df5aa5f00..17683846b 100644 --- a/tests/test_search_cache.py +++ b/tests/test_search_cache.py @@ -53,8 +53,8 @@ def _clean(monkeypatch): """Fresh cache/backoff + keyless, flag-default slate for every test.""" for var in ( "SERPAPI_KEY", "BING_SEARCH_KEY", "GOOGLE_CSE_ID", "GOOGLE_CSE_KEY", - "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY", "SERPLY_API_KEY", "SEARCH_FALLBACK_ORDER", - "SEARCH_ATTEMPT_ORDER", + "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY", "SERPLY_API_KEY", "FIRECRAWL_API_KEY", + "SEARCH_FALLBACK_ORDER", "SEARCH_ATTEMPT_ORDER", "SEARCH_BACKENDS", "SEARCH_CACHE_ENABLED", "SEARCH_BACKOFF_ENABLED", "SEARCH_CACHE_TTL", "SEARCH_CACHE_EMPTY_TTL", "SEARCH_CACHE_MAX_ENTRIES", "SEARCH_BACKOFF_BASE", "SEARCH_BACKOFF_CAP", diff --git a/tests/test_search_multi_engine.py b/tests/test_search_multi_engine.py index 0d26f87cc..b6a9d5d83 100644 --- a/tests/test_search_multi_engine.py +++ b/tests/test_search_multi_engine.py @@ -58,8 +58,8 @@ def _clear_env(monkeypatch): for var in ( "SERPAPI_KEY", "BING_SEARCH_KEY", "GOOGLE_CSE_ID", "GOOGLE_CSE_KEY", "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY", - "SERPLY_API_KEY", "SEARCH_FALLBACK_ORDER", "SEARCH_ATTEMPT_ORDER", "SEARCH_BACKENDS", - "SEARCH_CACHE_ENABLED", "SEARCH_BACKOFF_ENABLED", + "SERPLY_API_KEY", "FIRECRAWL_API_KEY", "SEARCH_FALLBACK_ORDER", "SEARCH_ATTEMPT_ORDER", + "SEARCH_BACKENDS", "SEARCH_CACHE_ENABLED", "SEARCH_BACKOFF_ENABLED", ): monkeypatch.delenv(var, raising=False) _FakeDDGS.behavior = "empty" @@ -118,15 +118,22 @@ def test_shape_parity_across_all_adapters(monkeypatch): {"SERPLY_API_KEY": "k"}, {"results": [{"title": "F", "link": "https://f.example", "description": "sf", "position": 1}]}, ), + "firecrawl": ( + {"FIRECRAWL_API_KEY": "k"}, + {"success": True, "data": {"web": [ + {"title": "G", "url": "https://g.example", "description": "sg", "position": 1}, + ]}}, + ), } for engine, (env, json_body) in payloads.items(): for var in ("SERPAPI_KEY", "BING_SEARCH_KEY", "GOOGLE_CSE_ID", "GOOGLE_CSE_KEY", "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY", - "SERPLY_API_KEY"): + "SERPLY_API_KEY", "FIRECRAWL_API_KEY"): monkeypatch.delenv(var, raising=False) for k, v in env.items(): monkeypatch.setenv(k, v) monkeypatch.setattr(se, "_http_get_json", lambda *a, **k: json_body) + monkeypatch.setattr(se, "_http_post_json", lambda *a, **k: json_body) assert se.configured_engines() == [engine] results = se.search_fallback("q", max_results=5) assert len(results) == 1, engine @@ -174,6 +181,44 @@ def fake_get(url, params, headers=None): assert seen["params"]["num"] == 10 +def test_firecrawl_posts_query_with_bearer_key_and_caps_limit(monkeypatch): + """Firecrawl gets the query in a POST body and the key in the Authorization header.""" + monkeypatch.setenv("FIRECRAWL_API_KEY", "fc-test") + seen = {} + + def fake_post(url, body, headers=None): + seen.update(url=url, body=body, headers=headers or {}) + return {"success": True, "data": {"web": [ + {"title": "G", "url": "https://g.example", "description": "sg", "position": 1}, + {"title": "Not a URL", "url": "javascript:void(0)", "description": "x"}, + {"title": "H", "url": "https://h.example", "description": "sh", "position": 2}, + ]}} + + def no_get(*a, **k): + raise AssertionError("firecrawl must not send the query in a GET URL") + + monkeypatch.setattr(se, "_http_post_json", fake_post) + monkeypatch.setattr(se, "_http_get_json", no_get) + assert se.configured_engines() == ["firecrawl"] + results = se.search_fallback("site:example.com acme", max_results=500) + assert results == [ + {"title": "G", "href": "https://g.example", "body": "sg"}, + {"title": "H", "href": "https://h.example", "body": "sh"}, + ] + assert seen["url"] == "https://api.firecrawl.dev/v2/search" + assert seen["headers"]["Authorization"] == "Bearer fc-test" + assert seen["body"]["query"] == "site:example.com acme" + assert seen["body"]["limit"] == 100 + assert "fc-test" not in seen["url"] and "fc-test" not in str(seen["body"]) + assert "scrapeOptions" not in seen["body"] + assert seen["body"]["highlights"] is False + + assert se.search_fallback("q", max_results=1) == [ + {"title": "G", "href": "https://g.example", "body": "sg"}, + ] + assert seen["body"]["limit"] == 1 + + def test_engine_order_respected(monkeypatch): monkeypatch.setenv("SERPAPI_KEY", "k") monkeypatch.setenv("BRAVE_SEARCH_KEY", "k")