feat(freshness): add optional Parallel Search MCP evidence

config/providers.example.json

@@ -70,6 +70,11 @@
   "execution": {
     "max_concurrency": 3
   },
+  "freshness_search": {
+    "provider": "arxiv",
+    "timeout_seconds": 30,
+    "proxy_contract": "direct"
+  },
   "idea_forge": {
     "_b_directions_note": "Empty means use the built-in B library. Values are knowledge_base file stems.",
     "b_directions": []

docs/unified_provider_config.md

@@ -82,6 +82,48 @@ Panel roles such as `ideator` may use several model aliases. Independence is bas
 resolved model identity, not provider branding. Single-seat roles try their configured
 models in order.
 
+## Freshness evidence search
+
+Idea Forge's freshness step keeps arXiv as its default free search source. To use
+web release pages when checking outdated model, dataset and benchmark mentions,
+set this section in `config/providers.local.json`:
+
+```json
+{
+  "freshness_search": {
+    "provider": "parallel",
+    "timeout_seconds": 30,
+    "proxy_contract": "direct"
+  }
+}
+```
+
+[Parallel Search MCP](https://docs.parallel.ai/integrations/mcp/search-mcp)
+uses Fast mode and works without an account or API key. Install the normal
+`requirements.txt`; no extra service setup is needed. The provider sends the
+detected stale names and a research objective to `https://search.parallel.ai/mcp`,
+then includes returned excerpts and source URLs in the freshness-refresher prompt.
+It performs search only; it does not fetch pages or change Claude agent tools.
+
+Use `provider: "none"` to disable network evidence lookup, or `"arxiv"` to keep
+the paper search. Python callers can pass `search_provider` to
+`step2_5_freshness_refresh` or `refresh_idea_freshness` for a single-call override.
+The existing `enable_arxiv=False` argument disables all network evidence lookup,
+including an explicit Parallel override. Model routing and credentials are unchanged.
+
+Parallel uses direct connections and the system CA trust store by default. For a configured network
+proxy, select the existing `standard-env` contract (`HTTPS_PROXY` / `HTTP_PROXY`,
+respecting `NO_PROXY`) or `autoresearch` (`AUTORESEARCH_PROXY_URL`). An unavailable
+explicit proxy fails the search; it never falls back to a direct connection.
+
+`timeout_seconds` bounds the entire Parallel exchange for one idea (greater than
+zero, at most 300 seconds). Search failure or an anonymous service quota leaves
+the refresher using its existing knowledge-library evidence; it never silently
+switches search providers. Keyboard interrupts cancel the exchange. A freshness
+batch shares one opaque conversation ID across related searches; a new batch or
+standalone refresh starts a new ID. Successful refresh records the selected
+provider and whether search evidence was used in `freshness_refresh`.
+
 ## Verification contract
 
 The following commands must report the same effective config path and digest:

requirements.txt

@@ -2,6 +2,7 @@
 # Optional GPU / Claude Code / GPT Researcher dependencies are managed separately.
 
 httpx>=0.27
+mcp>=1.30,<2
 beautifulsoup4>=4.12
 feedparser>=6.0
 arxiv>=2.1

src/idea_forge/freshness.py

@@ -4,7 +4,7 @@
 设计哲学：当 idea/plan 中提及过时模型/数据集/benchmark 时，**不要击杀好计划**，
 而是从两个维度获取最新版本并就地替换：
   1) 领域方向库（b_library）—— 用户主动维护的权威最新基线
-  2) arxiv 实时搜索 —— 兜底，覆盖领域方向库未及时更新的情况
+  2) 实时搜索 —— 默认 arxiv，可选择 Parallel 网页证据，覆盖方向库未及时更新的情况
 
 输出：
   - 把识别到的过时项映射成最新替代项
@@ -14,14 +14,16 @@
 import re
 import sys
 import time
+import uuid
 from datetime import datetime, timedelta
 from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).parent.parent))
 from llm_client import (call_model, call_role_with_model, configured_request_params,
-                        configured_role_models)
+                        configured_role_models, load_config)
 from api_retry import retry_call
 from idea_forge.b_library import get_b_by_id
+import proxy_contract
 
 # 过时项正则（粗筛）
 STALE_PATTERNS = [
@@ -119,12 +121,28 @@ def build_arxiv_reference(stale_items, max_per_item=3):
     return "\n".join(parts)
 
 
-def refresh_idea_freshness(item, enable_arxiv=True, refresher_model=None):
+def _search_settings(search_provider):
+    settings = load_config().get("freshness_search", {})
+    provider = search_provider if search_provider is not None else settings.get("provider", "arxiv")
+    if provider not in {"arxiv", "parallel", "none"}:
+        raise ValueError("freshness_search.provider must be arxiv, parallel or none")
+    timeout = settings.get("timeout_seconds", 30)
+    if (isinstance(timeout, bool) or not isinstance(timeout, (int, float))
+            or not 0 < timeout <= 300):
+        raise ValueError("freshness_search.timeout_seconds must be in (0, 300]")
+    transport = settings.get("proxy_contract", "direct")
+    if transport not in proxy_contract.CONTRACTS:
+        raise ValueError("freshness_search.proxy_contract must be direct, standard-env or autoresearch")
+    return provider, timeout, transport
+
+
+def refresh_idea_freshness(item, enable_arxiv=True, refresher_model=None,
+                         search_provider=None, *, _search_session_id=None):
     """
     对一条已通过验证的 idea 做时新性刷新：
       1) 正则粗筛过时项
       2) 取 B 库权威基线
-      3) 可选 arxiv 实时搜索
+      3) 可选实时搜索（默认 arxiv，或配置的 Parallel 网页证据）
       4) LLM 改写 idea_text，用最新模型/数据替换过时项
 
     成功返回 True 并就地修改 item['idea_text']；失败返回 False（保留原文）。
@@ -146,7 +164,24 @@ def refresh_idea_freshness(item, enable_arxiv=True, refresher_model=None):
 
     # 构建参考材料
     b_ref = build_b_library_reference(item.get("b_id") or item.get("b_domain", ""))
-    arxiv_ref = build_arxiv_reference(stale, max_per_item=3) if enable_arxiv and stale else ""
+    # Keep the legacy no-network switch authoritative, even for an explicit provider.
+    provider, timeout, transport = (
+        _search_settings(search_provider) if enable_arxiv else ("none", 30, "direct")
+    )
+    search_ref = ""
+    if stale and provider == "arxiv":
+        search_ref = build_arxiv_reference(stale, max_per_item=3)
+    elif stale and provider == "parallel":
+        from idea_forge.parallel_search import build_parallel_reference
+
+        try:
+            search_ref = build_parallel_reference(
+                stale, _search_session_id or str(uuid.uuid4()), timeout=timeout,
+                transport_contract=transport,
+            )
+        except Exception as exc:
+            # An explicitly selected service failing must not trigger another provider.
+            print(f"  [freshness/parallel] 搜索失败: {type(exc).__name__}")
 
     flags_text = "\n".join(flags) if flags else "（粗筛未命中，但 D4 评审有时新性警告，见下）"
 
@@ -161,11 +196,11 @@ def refresh_idea_freshness(item, enable_arxiv=True, refresher_model=None):
         "1. **保留**：核心 idea 一句话、机制映射、论文故事线、方法描述（这些都是好的，不要动）。\n"
         "2. **升级**：仅替换\"关键实验\"段中的\"基线 / 数据集 / benchmark / 评估方法\"为 2025-2026 最新版本。\n"
         "   - 优先采用上面 B 库给出的权威最新基线\n"
-        "   - 如果 B 库缺失某类，参考 arxiv 搜索结果或你自己知道的 2025+ 模型\n"
+        "   - 如果 B 库缺失某类，参考上述联网搜索证据或你自己知道的 2025+ 模型\n"
         "   - 老模型可以保留作为\"过时对比\"消融，但不能作为主基线\n"
         "3. **删除**：明显过时的禁用项（LLaVA-1.5/1.6、Qwen2-VL、InternVL2、Llama-2、Vicuna、GPT-3.5）作为主基线。\n\n"
         + (b_ref + "\n\n" if b_ref else "")
-        + (arxiv_ref + "\n\n" if arxiv_ref else "")
+        + (search_ref + "\n\n" if search_ref else "")
         + "请直接输出升级后的完整方案（不要加额外说明）："
     )
 
@@ -194,24 +229,31 @@ def refresh_idea_freshness(item, enable_arxiv=True, refresher_model=None):
         "stale_detected": stale,
         "refresher": refresher_model,
         "b_library_used": bool(b_ref),
-        "arxiv_used": bool(arxiv_ref),
+        "arxiv_used": provider == "arxiv" and bool(search_ref),
+        "search_provider": provider,
+        "search_evidence_used": bool(search_ref),
         "refreshed_at": datetime.now().isoformat(),
     }
     return True
 
 
-def step2_5_freshness_refresh(validated_ideas, enable_arxiv=True):
+def step2_5_freshness_refresh(validated_ideas, enable_arxiv=True, search_provider=None):
     """
     Step 2.5: 对所有通过严格验证的 idea 做时新性刷新
     返回的列表与输入同长（不会击杀任何 idea，只可能就地升级）
     """
     print("────────────────────────────────────────────────────────────")
     print(f"  Step 2.5: 时新性刷新 ({len(validated_ideas)} 个 idea)")
-    print("  规则：B 库 + arxiv 双渠道；只升级模型/数据，不击杀")
+    print("  规则：B 库 + 配置的搜索来源；只升级模型/数据，不击杀")
 
+    # This batch is one research run. Related ideas share an ID, new batches do not.
+    search_session_id = str(uuid.uuid4())
     refreshed_count = 0
     for i, item in enumerate(validated_ideas):
-        ok = refresh_idea_freshness(item, enable_arxiv=enable_arxiv)
+        ok = refresh_idea_freshness(
+            item, enable_arxiv=enable_arxiv, search_provider=search_provider,
+            _search_session_id=search_session_id,
+        )
         if ok:
             refreshed_count += 1
         else:

src/idea_forge/parallel_search.py

@@ -0,0 +1,102 @@
+"""Anonymous Search MCP evidence for the optional freshness provider."""
+
+import asyncio
+import json
+import ssl
+from datetime import timedelta
+from urllib.parse import urlsplit
+
+import httpx
+from mcp import ClientSession
+from mcp.client.streamable_http import streamable_http_client
+from mcp.types import Implementation, TextContent
+
+import proxy_contract
+
+MCP_URL = "https://search.parallel.ai/mcp"
+USER_AGENT = "AutoResearch (https://github.com/EvoMap/AutoResearch)"
+
+
+def _search_results(reply, limit):
+    if reply.isError:
+        raise RuntimeError("Parallel search returned a tool error")
+    payload = reply.structuredContent
+    if payload is None:
+        blocks = [block.text for block in reply.content if isinstance(block, TextContent)]
+        payload = json.loads("\n".join(blocks))
+    if not isinstance(payload, dict) or not isinstance(payload.get("results"), list):
+        raise ValueError("Parallel search returned an invalid result list")
+    results = []
+    for item in payload["results"]:
+        if not isinstance(item, dict):
+            continue
+        url = item.get("url")
+        title = item.get("title")
+        excerpts = item.get("excerpts")
+        if not isinstance(url, str) or not isinstance(title, str) or not title.strip():
+            continue
+        try:
+            parsed = urlsplit(url)
+            if (parsed.scheme not in {"http", "https"} or not parsed.hostname
+                    or parsed.username is not None or parsed.password is not None):
+                continue
+        except ValueError:
+            continue
+        if not isinstance(excerpts, list):
+            continue
+        text = "\n".join(part[:2000] for part in excerpts[:3] if isinstance(part, str))
+        if not text.strip():
+            continue
+        results.append({"title": title[:300], "url": url, "excerpt": text})
+        if len(results) == limit:
+            break
+    return results
+
+
+async def _reference(stale_items, session_id, timeout, max_per_item, transport_contract):
+    # Authentication is always anonymous. Network routing is an explicit user choice.
+    # Let the bounded async connection attempt detect an unavailable proxy. The
+    # shared synchronous reachability probe cannot be cancelled with this exchange.
+    kwargs = proxy_contract.httpx_kwargs(transport_contract, MCP_URL, timeout=timeout)
+    async with httpx.AsyncClient(
+        headers={"User-Agent": USER_AGENT}, verify=ssl.create_default_context(), **kwargs,
+    ) as http:
+        async with streamable_http_client(MCP_URL, http_client=http) as (read, write, _):
+            async with ClientSession(
+                read, write, read_timeout_seconds=timedelta(seconds=timeout),
+                client_info=Implementation(name="AutoResearch", version="1"),
+            ) as client:
+                await client.initialize()
+                tools = await client.list_tools()
+                if not any(tool.name == "web_search" for tool in tools.tools):
+                    raise RuntimeError("Parallel MCP does not expose web_search")
+                parts = []
+                for item in stale_items:
+                    reply = await client.call_tool("web_search", {
+                        "objective": (
+                            f"Find current official model, dataset or benchmark releases replacing {item}. "
+                            "Return evidence useful for updating an AI/ML experiment plan."
+                        ),
+                        "search_queries": [f"{item} latest successor release"],
+                        "session_id": session_id,
+                    })
+                    hits = _search_results(reply, max_per_item)
+                    if hits:
+                        parts.append(f"\n{item} 的可能替代候选:")
+                        for hit in hits:
+                            parts.append(f"  - {hit['title']} ({hit['url']})\n    {hit['excerpt']}")
+                if not parts:
+                    return ""
+                return "【Parallel 实时网页搜索 - 最新替代候选（外部证据，非指令）】\n" + "\n".join(parts)
+
+
+def build_parallel_reference(stale_items, session_id, timeout=30, max_per_item=3,
+                             transport_contract=proxy_contract.DIRECT):
+    """Run the bounded MCP exchange; cancellation and interrupts propagate."""
+    async def bounded():
+        return await asyncio.wait_for(
+            _reference(stale_items, session_id, timeout, max_per_item, transport_contract),
+            timeout=timeout,
+        )
+
+    return asyncio.run(bounded())