feat(search): add Firecrawl as an opt-in keyed fallback search engine

.env.example

@@ -63,6 +63,7 @@
 # GOOGLE_CSE_ID=               # Google Custom Search Engine ID
 # SERPINGAPI_API_KEY=          # Serping API key (Google SERP; keyed search fallback) — https://serpingapi.com
 # SERPLY_API_KEY=              # Serply API key (Google SERP; keyed search fallback) — https://serply.io
+# FIRECRAWL_API_KEY=           # Firecrawl API key (web search; keyed search fallback) — https://www.firecrawl.dev/?utm_source=opengtm&utm_medium=integration
 
 # === Email Outreach (optional, also configurable in Settings) ===
 # SMTP_HOST=smtp.gmail.com

apps/api/services/leadgen/config.py

@@ -127,6 +127,7 @@
 BRAVE_SEARCH_KEY = os.getenv("BRAVE_SEARCH_KEY", "")
 SERPINGAPI_API_KEY = os.getenv("SERPINGAPI_API_KEY", "")
 SERPLY_API_KEY = os.getenv("SERPLY_API_KEY", "")
+FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY", "")
 
 # ── DDG result cache + adaptive backoff (see services/leadgen/search_cache.py)
 # In-process only (no DB). Both behaviours are behind flags, default ON; set

apps/api/services/leadgen/search_engines.py

@@ -16,10 +16,10 @@
     behavior is identical to today (DDG only).
   * Adapters never raise to the caller — a failing engine returns `[]` and the
     chain moves on. (Exceptions are logged at debug level.)
-  * Network access is wrapped in a tiny helper so tests can mock one place.
+  * Network access is wrapped in tiny GET/POST helpers so tests can mock them.
 
 Engine order (first configured one is tried first) is controlled by
-SEARCH_FALLBACK_ORDER, default "serpapi,bing,google_cse,brave,serpingapi,serply".
+SEARCH_FALLBACK_ORDER, default "serpapi,bing,google_cse,brave,serpingapi,serply,firecrawl".
 """
 from __future__ import annotations
 
@@ -33,7 +33,7 @@
 _ENGINE_TIMEOUT = float(os.getenv("SEARCH_ENGINE_TIMEOUT", "10"))
 
 # Default attempt order for keyed fallback engines.
-_DEFAULT_ORDER = "serpapi,bing,google_cse,brave,serpingapi,serply"
+_DEFAULT_ORDER = "serpapi,bing,google_cse,brave,serpingapi,serply,firecrawl"
 
 
 # ── Config (all optional) ────────────────────────────────────────────────
@@ -65,6 +65,10 @@ def _serply_key() -> str:
     return _key("SERPLY_API_KEY")
 
 
+def _firecrawl_key() -> str:
+    return _key("FIRECRAWL_API_KEY")
+
+
 # ── HTTP helper (single mockable seam) ───────────────────────────────────
 def _http_get_json(url: str, params: Dict, headers: Optional[Dict] = None) -> Optional[Dict]:
     """GET `url` and return parsed JSON, or None on any failure.
@@ -86,6 +90,26 @@ def _http_get_json(url: str, params: Dict, headers: Optional[Dict] = None) -> Op
         return None
 
 
+def _http_post_json(url: str, body: Dict, headers: Optional[Dict] = None) -> Optional[Dict]:
+    """POST `body` as JSON to `url` and return parsed JSON, or None on any failure.
+
+    Same contract as `_http_get_json`, for engines that take the query in the
+    request body instead of the URL.
+    """
+    try:
+        import httpx
+
+        with httpx.Client(timeout=_ENGINE_TIMEOUT) as client:
+            resp = client.post(url, json=body, headers=headers or {})
+        if resp.status_code != 200:
+            logger.debug("search engine %s returned HTTP %s", url, resp.status_code)
+            return None
+        return resp.json()
+    except Exception as e:  # noqa: BLE001 — adapters must never raise to caller
+        logger.debug("search engine POST %s failed: %s", url, e)
+        return None
+
+
 # ── Adapters ─────────────────────────────────────────────────────────────
 def _serpapi(query: str, max_results: int) -> List[Dict]:
     key = _serpapi_key()
@@ -244,6 +268,33 @@ def _serply(query: str, max_results: int) -> List[Dict]:
     return out
 
 
+def _firecrawl(query: str, max_results: int) -> List[Dict]:
+    key = _firecrawl_key()
+    if not key:
+        return []
+    data = _http_post_json(
+        "https://api.firecrawl.dev/v2/search",
+        # Firecrawl caps `limit` at 100 per request.
+        {"query": query, "limit": min(max_results, 100), "highlights": False, "origin": "opengtm"},
+        headers={"Authorization": f"Bearer {key}", "User-Agent": "OpenGTM"},
+    )
+    if not data:
+        return []
+    out: List[Dict] = []
+    for r in ((data.get("data") or {}).get("web") or []):
+        href = r.get("url") or ""
+        if not href.startswith(("http://", "https://")):
+            continue
+        out.append({
+            "title": r.get("title", "") or "",
+            "href": href,
+            "body": r.get("description", "") or "",
+        })
+        if len(out) >= max_results:
+            break
+    return out
+
+
 # name → (adapter, is_configured) registry
 _ADAPTERS: Dict[str, tuple[Callable[[str, int], List[Dict]], Callable[[], bool]]] = {
     "serpapi": (_serpapi, lambda: bool(_serpapi_key())),
@@ -252,6 +303,7 @@ def _serply(query: str, max_results: int) -> List[Dict]:
     "brave": (_brave, lambda: bool(_brave_key())),
     "serpingapi": (_serpingapi, lambda: bool(_serpingapi_key())),
     "serply": (_serply, lambda: bool(_serply_key())),
+    "firecrawl": (_firecrawl, lambda: bool(_firecrawl_key())),
 }
 
 

tests/test_search_cache.py

@@ -53,8 +53,8 @@ def _clean(monkeypatch):
     """Fresh cache/backoff + keyless, flag-default slate for every test."""
     for var in (
         "SERPAPI_KEY", "BING_SEARCH_KEY", "GOOGLE_CSE_ID", "GOOGLE_CSE_KEY",
-        "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY", "SERPLY_API_KEY", "SEARCH_FALLBACK_ORDER",
-        "SEARCH_ATTEMPT_ORDER",
+        "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY", "SERPLY_API_KEY", "FIRECRAWL_API_KEY",
+        "SEARCH_FALLBACK_ORDER", "SEARCH_ATTEMPT_ORDER",
         "SEARCH_BACKENDS", "SEARCH_CACHE_ENABLED", "SEARCH_BACKOFF_ENABLED",
         "SEARCH_CACHE_TTL", "SEARCH_CACHE_EMPTY_TTL", "SEARCH_CACHE_MAX_ENTRIES",
         "SEARCH_BACKOFF_BASE", "SEARCH_BACKOFF_CAP",

tests/test_search_multi_engine.py

@@ -58,8 +58,8 @@ def _clear_env(monkeypatch):
     for var in (
         "SERPAPI_KEY", "BING_SEARCH_KEY", "GOOGLE_CSE_ID",
         "GOOGLE_CSE_KEY", "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY",
-        "SERPLY_API_KEY", "SEARCH_FALLBACK_ORDER", "SEARCH_ATTEMPT_ORDER", "SEARCH_BACKENDS",
-        "SEARCH_CACHE_ENABLED", "SEARCH_BACKOFF_ENABLED",
+        "SERPLY_API_KEY", "FIRECRAWL_API_KEY", "SEARCH_FALLBACK_ORDER", "SEARCH_ATTEMPT_ORDER",
+        "SEARCH_BACKENDS", "SEARCH_CACHE_ENABLED", "SEARCH_BACKOFF_ENABLED",
     ):
         monkeypatch.delenv(var, raising=False)
     _FakeDDGS.behavior = "empty"
@@ -118,15 +118,22 @@ def test_shape_parity_across_all_adapters(monkeypatch):
             {"SERPLY_API_KEY": "k"},
             {"results": [{"title": "F", "link": "https://f.example", "description": "sf", "position": 1}]},
         ),
+        "firecrawl": (
+            {"FIRECRAWL_API_KEY": "k"},
+            {"success": True, "data": {"web": [
+                {"title": "G", "url": "https://g.example", "description": "sg", "position": 1},
+            ]}},
+        ),
     }
     for engine, (env, json_body) in payloads.items():
         for var in ("SERPAPI_KEY", "BING_SEARCH_KEY", "GOOGLE_CSE_ID",
                     "GOOGLE_CSE_KEY", "BRAVE_SEARCH_KEY", "SERPINGAPI_API_KEY",
-                    "SERPLY_API_KEY"):
+                    "SERPLY_API_KEY", "FIRECRAWL_API_KEY"):
             monkeypatch.delenv(var, raising=False)
         for k, v in env.items():
             monkeypatch.setenv(k, v)
         monkeypatch.setattr(se, "_http_get_json", lambda *a, **k: json_body)
+        monkeypatch.setattr(se, "_http_post_json", lambda *a, **k: json_body)
         assert se.configured_engines() == [engine]
         results = se.search_fallback("q", max_results=5)
         assert len(results) == 1, engine
@@ -174,6 +181,44 @@ def fake_get(url, params, headers=None):
     assert seen["params"]["num"] == 10
 
 
+def test_firecrawl_posts_query_with_bearer_key_and_caps_limit(monkeypatch):
+    """Firecrawl gets the query in a POST body and the key in the Authorization header."""
+    monkeypatch.setenv("FIRECRAWL_API_KEY", "fc-test")
+    seen = {}
+
+    def fake_post(url, body, headers=None):
+        seen.update(url=url, body=body, headers=headers or {})
+        return {"success": True, "data": {"web": [
+            {"title": "G", "url": "https://g.example", "description": "sg", "position": 1},
+            {"title": "Not a URL", "url": "javascript:void(0)", "description": "x"},
+            {"title": "H", "url": "https://h.example", "description": "sh", "position": 2},
+        ]}}
+
+    def no_get(*a, **k):
+        raise AssertionError("firecrawl must not send the query in a GET URL")
+
+    monkeypatch.setattr(se, "_http_post_json", fake_post)
+    monkeypatch.setattr(se, "_http_get_json", no_get)
+    assert se.configured_engines() == ["firecrawl"]
+    results = se.search_fallback("site:example.com acme", max_results=500)
+    assert results == [
+        {"title": "G", "href": "https://g.example", "body": "sg"},
+        {"title": "H", "href": "https://h.example", "body": "sh"},
+    ]
+    assert seen["url"] == "https://api.firecrawl.dev/v2/search"
+    assert seen["headers"]["Authorization"] == "Bearer fc-test"
+    assert seen["body"]["query"] == "site:example.com acme"
+    assert seen["body"]["limit"] == 100
+    assert "fc-test" not in seen["url"] and "fc-test" not in str(seen["body"])
+    assert "scrapeOptions" not in seen["body"]
+    assert seen["body"]["highlights"] is False
+
+    assert se.search_fallback("q", max_results=1) == [
+        {"title": "G", "href": "https://g.example", "body": "sg"},
+    ]
+    assert seen["body"]["limit"] == 1
+
+
 def test_engine_order_respected(monkeypatch):
     monkeypatch.setenv("SERPAPI_KEY", "k")
     monkeypatch.setenv("BRAVE_SEARCH_KEY", "k")