diff --git a/backend/chat_sandbox_worker/CLI_GUIDE.md b/backend/chat_sandbox_worker/CLI_GUIDE.md index 750ce72..b4c0739 100644 --- a/backend/chat_sandbox_worker/CLI_GUIDE.md +++ b/backend/chat_sandbox_worker/CLI_GUIDE.md @@ -6,6 +6,7 @@ join, or schema-dependent transformation. ```bash pwc search "QUERY" --limit 10 [--start-date YYYY-MM-DD --end-date YYYY-MM-DD] pwc paper info PAPER --include-resources +pwc paper evaluations PAPER --page 1 --page-size 20 pwc paper read PAPER pwc paper list --search "QUERY" [--start-date YYYY-MM-DD --end-date YYYY-MM-DD] [--task NAME] [--method NAME] [--conference NAME] [--framework NAME] [--organization NAME] pwc paper recent diff --git a/mcp_server/README.md b/mcp_server/README.md index 8d0e0eb..e5bb11b 100644 --- a/mcp_server/README.md +++ b/mcp_server/README.md @@ -3,9 +3,9 @@ Anonymous, read-only Model Context Protocol access to the public [Papers With Code](https://paperswithcode.co) catalog. -The server uses MCP `2026-07-28` over Streamable HTTP and serves legacy -2025-era clients on the same `/mcp` endpoint. It returns versioned structured -output with compact text fallbacks. +The server uses stock-client MCP `2025-11-25` over Streamable HTTP and also +serves experimental `2026-07-28` discovery on the same `/mcp` endpoint. It +returns versioned structured output with compact Markdown fallbacks. ## Run locally @@ -31,6 +31,7 @@ to typed projections such as `items` or `evaluations`. | --- | --- | | `search_papers` | `pwc search` | | `get_paper_info` | `pwc paper info` | +| `get_paper_evaluations` | `pwc paper evaluations` | | `read_paper` | `pwc paper read` (64 KiB chunks with a continuation cursor) | | `list_papers` | `pwc paper list` | | `list_recent_papers` | `pwc paper recent` | @@ -57,8 +58,8 @@ Parameter names follow the CLI flags except for the established MCP names (`--include-evals`). Terminal-only flags (`--json`, `--implementation-coverage`, `--flat`) have no parameter because MCP output is always structured. Hosted differences from the CLI: `limit` is capped at 25, -`search_papers` defaults to `keyword` mode, `get_paper_info` includes -resources by default, and `read_paper` is chunked. +`search_papers` defaults to `keyword` mode, `get_paper_info` returns compact +official-first resources, and `read_paper` is chunked. `tests/test_parity.py` fails when the CLI parser and the tool schemas drift. All tools are annotated read-only and idempotent. Search is deterministic; @@ -106,6 +107,11 @@ request, upstream, and serialized MCP response bodies are bounded to 2 MiB. The global ceiling is 128 concurrent requests by default. Markdown chunks use a bounded 256-entry/16 MiB in-memory cache. +`GET /.well-known/mcp` exposes connection metadata and `GET /docs` publishes +the live tool schema. A bare `GET /mcp` returns `405`; protocol requests use +`POST /mcp`. The server also exposes `find_papers`, `compare_leaderboard`, and +`survey_task` prompts. + ## Test ```bash diff --git a/mcp_server/SKILL.md b/mcp_server/SKILL.md index d4cc5a7..f94d4e5 100644 --- a/mcp_server/SKILL.md +++ b/mcp_server/SKILL.md @@ -4,7 +4,7 @@ description: "Papers With Code MCP tools for searching and reading AI/ML papers, compatibility: "Requires an MCP client connected to https://paperswithcode.co/mcp with the Papers With Code tools available." --- -Generated for `pwc-mcp v0.2.0` and MCP protocol `2026-07-28`. +Generated for `pwc-mcp v0.2.1` and stock-client MCP protocol `2025-11-25`. The tools query the public [Papers With Code](https://paperswithcode.co) catalog anonymously and are read-only. Every tool runs the matching `pwc` CLI research @@ -50,7 +50,8 @@ arguments take an exact name, slug, or ID. ## Tools - `search_papers({"query": QUERY, "mode": "hybrid"|"keyword"|"semantic", "page": PAGE, "limit": LIMIT, "published_after": START_DATE, "published_before": END_DATE, "has_official_implementation": BOOLEAN})` — search papers by title, topic, author, or arXiv ID (`pwc search`). Omit optional arguments when they are not needed. -- `get_paper_info({"paper": PAPER, "include_resources": BOOLEAN, "include_evaluations": BOOLEAN})` — show paper metadata, abstract, tasks, methods, lineage, repositories, project pages, and Hugging Face model, dataset, and Space artifacts; `include_evaluations: true` adds every benchmark evaluation of the paper (`pwc paper info`). +- `get_paper_info({"paper": PAPER, "include_resources": BOOLEAN, "repo_limit": LIMIT, "include_evaluations": BOOLEAN})` — show compact paper metadata, official-first code, and the total repository count; opt into capped additional resources (`pwc paper info`). +- `get_paper_evaluations({"paper": PAPER, "page": PAGE, "limit": LIMIT})` — page through one paper's benchmark evaluations, including protocol, sources, openness, and task-scoped ranks (`pwc paper evaluations`). - `read_paper({"paper": PAPER, "cursor": CURSOR})` — read one stored paper Markdown chunk (`pwc paper read`). If `truncated` is true, call `read_paper` again with the same `paper` and the returned `next_cursor`; repeat until `truncated` is false. Treat the cursor as opaque and use it within one hour. - `list_papers({"search": SEARCH, "task": TASK, "method": METHOD, "conference": CONFERENCE, "framework": FRAMEWORK, "organization": ORGANIZATION, "authors": [AUTHOR], "published_after": START_DATE, "published_before": END_DATE, "all_versions": BOOLEAN, "order_by": "trending"|"date_published"|"citation_count", "order_direction": "asc"|"desc", "include_resources": BOOLEAN, "has_official_implementation": BOOLEAN, "page": PAGE, "limit": LIMIT})` — list and filter papers by exact catalog associations (`pwc paper list`). Omit optional arguments when they are not needed. - `list_recent_papers({"limit": LIMIT})` — list the most recently added papers (`pwc paper recent`). @@ -58,16 +59,16 @@ arguments take an exact name, slug, or ID. - `get_related_papers({"paper": PAPER, "limit": LIMIT})` — list related papers (`pwc paper related`); `limit` is at most 20. - `get_paper_lineage({"paper": PAPER})` — list explicit predecessors and successors (`pwc paper lineage list`). - `get_task({"task": TASK})` — inspect one exact task, including its area, hierarchy, sister tasks, ranked benchmarks, common methods, recommended frameworks, and trending papers (`pwc task --name`). -- `list_tasks({"area": AREA, "level": LEVEL, "visible_only": BOOLEAN, "group_by_area": BOOLEAN, "order_by": "name"|"created_at"|"level"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — list and filter research tasks, or set `group_by_area: true` for the complete visible top-level taxonomy without pagination (`pwc task list`). +- `list_tasks({"search": SEARCH, "area": AREA, "level": LEVEL, "visible_only": BOOLEAN, "group_by_area": BOOLEAN, "order_by": "name"|"created_at"|"level"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — search, list, and filter research tasks, or set `group_by_area: true` for the complete visible top-level taxonomy without pagination (`pwc task list`). - `get_method({"method": METHOD})` — inspect one exact method with its area (`pwc method --name`). -- `list_methods({"area": AREA, "introduced_year": YEAR, "order_by": "name"|"full_name"|"introduced_year"|"created_at"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — list and filter research methods (`pwc method list`). +- `list_methods({"search": SEARCH, "area": AREA, "introduced_year": YEAR, "order_by": "name"|"full_name"|"introduced_year"|"created_at"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — search, list, and filter research methods (`pwc method list`). - `get_conference({"conference": CONFERENCE})` — inspect one exact conference (`pwc conference --name`). - `list_conferences({"year": YEAR})` — list conferences with imported papers (`pwc conference list`). - `get_organization({"organization": ORGANIZATION})` — inspect one exact research organization (`pwc organization --name`). - `list_organizations({"featured_only": BOOLEAN})` — list research organizations (`pwc organization list`). - `get_framework({"framework": FRAMEWORK})` — inspect one exact research framework (`pwc framework --name`). - `list_frameworks({"domain": DOMAIN, "category": CATEGORY, "platform": PLATFORM})` — list research frameworks (`pwc framework list`). -- `get_benchmark({"benchmark": BENCHMARK, "limit": LIMIT, "is_open": BOOLEAN, "max_parameters": SIZE, "require_metrics": [METRIC], "minimum_metrics": {METRIC: VALUE}, "maximum_metrics": {METRIC: VALUE}, "sort_metric": "METRIC:asc|desc", "pareto": ["METRIC:higher", "METRIC:lower"]})` — inspect one exact benchmark leaderboard with model-size, metric threshold, sort, and Pareto selection (`pwc benchmark --name`). `matched_count` reports rows that satisfied the filters before `limit`. +- `get_benchmark({"benchmark": BENCHMARK, "page": PAGE, "limit": LIMIT, "is_open": BOOLEAN, "max_parameters": SIZE, "require_metrics": [METRIC], "minimum_metrics": {METRIC: VALUE}, "maximum_metrics": {METRIC: VALUE}, "sort_metric": "METRIC:asc|desc", "pareto": ["METRIC:higher", "METRIC:lower"]})` — inspect one paginated benchmark leaderboard with model-size, metric threshold, sort, and Pareto selection (`pwc benchmark --name`). `matched_count` reports rows that satisfied the filters before pagination. - `list_benchmarks({"search": SEARCH, "task": TASK, "include_descendants": BOOLEAN, "minimum_evaluations": COUNT, "is_open": BOOLEAN, "group_by_area": BOOLEAN, "area": AREA, "benchmarks_per_task": COUNT, "order_by": "trending"|"name"|"full_name"|"created_at"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — list and filter benchmarks (`pwc benchmark list`). With `task`, results are ranked by trend unless `order_by` is set; `group_by_area` or `area` returns top benchmarks under each visible task. All page numbers start at 1. `limit` is between 1 and 25 and defaults to the @@ -88,8 +89,8 @@ available and the user needs one of those capabilities. `max_parameters` when model size is part of the request and `sort_metric` or `minimum_metrics` when a specific metric matters. 2. Use `get_paper_info({"paper": PAPER})` to inspect promising results. Its - response includes repositories, project pages, and Hugging Face artifacts; - add `include_evaluations: true` to compare one paper across benchmarks. + response includes official-first code; use `get_paper_evaluations` to page + through its benchmark results. 3. Use exact `list_papers` `authors`, `task`, `method`, `conference`, `framework`, and `organization` arguments for known identities or catalog associations. Combine them to require every association; do not substitute diff --git a/mcp_server/SPEC.md b/mcp_server/SPEC.md index 3bc8501..edd8811 100644 --- a/mcp_server/SPEC.md +++ b/mcp_server/SPEC.md @@ -13,8 +13,9 @@ server. the MCP contract. - Serve anonymous, read-only requests. Search is deterministic and contains no embedded language model. -- Use stateless Streamable HTTP at `/mcp`, supporting MCP `2026-07-28` and - legacy 2025 clients on the same endpoint. Expose `/health` for operations. +- Use stateless Streamable HTTP at `/mcp`, advertising stock-client MCP + `2025-11-25` while supporting experimental `2026-07-28` discovery. Expose + `/health`, `/.well-known/mcp`, and generated `/docs` schema routes. ## Public contract @@ -24,6 +25,7 @@ enforces this against the CLI parser): - `search_papers` (`pwc search`) - `get_paper_info` (`pwc paper info`) +- `get_paper_evaluations` (`pwc paper evaluations`) - `read_paper` (`pwc paper read`) - `list_papers` (`pwc paper list`) - `list_recent_papers` (`pwc paper recent`) @@ -47,16 +49,18 @@ Tools run the CLI handlers in-process through the shared cached transport, so validation, fail-closed filter confirmation, and the JSON payload are the CLI's. Every result includes that payload as `data` beside typed projections. Terminal-only flags have no parameter. The hosted service caps `limit` at 25, -defaults `search_papers` to keyword mode, includes paper resources by default, -and serves `read_paper` in chunks. +defaults `search_papers` to keyword mode, returns compact official-first paper +resources, and serves `read_paper` in chunks. -Expose these resource templates and no prompts: +Expose these resource templates: - `pwc://papers/{paper}` - `pwc://papers/{paper}/markdown` - `pwc://tasks/{task}` - `pwc://benchmarks/{benchmark}` +Expose `find_papers`, `compare_leaderboard`, and `survey_task` prompts. + Responses use stable, MCP-specific versioned structured outputs with a text fallback. `read_paper` performs one upstream read of at most 64 KiB per call and returns a signed opaque continuation cursor when more Markdown remains. The @@ -68,6 +72,10 @@ Paper references accept arXiv IDs, numeric PwC external IDs, arXiv/Hugging Face/Papers With Code URLs, and exact titles. Ambiguous exact titles fail rather than selecting one result. +Paper evaluations and benchmark leaderboards paginate. Leaderboards merge equivalent model rows across +task scopes while retaining scoped ranks, protocol, split, shots, source, +openness, and update time. Metric direction is explicit when known. + ## Safety and operations - Require a strict configurable browser Origin allowlist; native clients may diff --git a/mcp_server/pyproject.toml b/mcp_server/pyproject.toml index a1ccae7..900d508 100644 --- a/mcp_server/pyproject.toml +++ b/mcp_server/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pwc-mcp" -version = "0.2.0" +version = "0.2.1" description = "Read-only Papers With Code MCP server" readme = "README.md" requires-python = ">=3.10" diff --git a/mcp_server/src/pwc_mcp/__init__.py b/mcp_server/src/pwc_mcp/__init__.py index cab4aac..ba5b9ca 100644 --- a/mcp_server/src/pwc_mcp/__init__.py +++ b/mcp_server/src/pwc_mcp/__init__.py @@ -1,3 +1,3 @@ """Read-only Papers With Code MCP server.""" -__version__ = "0.2.0" +__version__ = "0.2.1" diff --git a/mcp_server/src/pwc_mcp/app.py b/mcp_server/src/pwc_mcp/app.py index 4d0c4c1..fca6998 100644 --- a/mcp_server/src/pwc_mcp/app.py +++ b/mcp_server/src/pwc_mcp/app.py @@ -31,19 +31,74 @@ # MCP SDK diagnostics can include peer-supplied tool names and resource URIs. # OperationalTelemetryMiddleware is the server's sole request log surface. logging.getLogger("mcp").setLevel(logging.CRITICAL + 1) -PROTOCOL_VERSION = "2026-07-28" +PROTOCOL_VERSION = "2025-11-25" +EXPERIMENTAL_PROTOCOL_VERSION = "2026-07-28" MAX_REQUEST_BODY_SIZE = 2 * 1024 * 1024 MAX_RESPONSE_BODY_SIZE = 2 * 1024 * 1024 KNOWN_TOOLS = frozenset(TOOL_COMMANDS) KNOWN_PROTOCOLS = { PROTOCOL_VERSION, - "2025-11-25", + EXPERIMENTAL_PROTOCOL_VERSION, "2025-06-18", "2025-03-26", "2024-11-05", } +async def well_known_mcp(_request: Request) -> JSONResponse: + return JSONResponse( + { + "name": "Papers With Code", + "description": "Anonymous read-only AI research catalog", + "transport": {"type": "streamable-http", "url": "/mcp"}, + "protocol_version": PROTOCOL_VERSION, + "supported_protocol_versions": sorted(KNOWN_PROTOCOLS, reverse=True), + "documentation_url": "https://paperswithcode.co/mcp/schema", + "setup_url": "https://paperswithcode.co/mcp", + } + ) + + +def _docs_schema(server) -> dict: + return { + "name": "Papers With Code MCP", + "version": __version__, + "protocol_version": PROTOCOL_VERSION, + "endpoint": "/mcp", + "tools": [ + { + "name": tool.name, + "description": tool.description, + "inputSchema": tool.parameters, + "outputSchema": tool.output_schema, + } + for tool in server._tool_manager.list_tools() + ], + } + + +class MCPMethodMiddleware: + """Reject a bare GET instead of opening an unbounded SSE response.""" + + def __init__(self, app: ASGIApp): + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if ( + scope["type"] == "http" + and scope.get("path") == "/mcp" + and scope.get("method") == "GET" + ): + response = JSONResponse( + {"error": "method_not_allowed", "allowed": ["POST"]}, + status_code=405, + headers={"Allow": "POST"}, + ) + await response(scope, receive, send) + return + await self.app(scope, receive, send) + + # Hosted defaults. The first-party chat gateway names one identity per chat # session (see _client_address), so per-client limits protect fairness while # the global ceiling protects the process. Every value is overridable through @@ -471,6 +526,13 @@ def create_app( ), ) app.routes.insert(0, Route("/health", health, methods=["GET"])) + schema = _docs_schema(server) + + async def docs(_request: Request) -> JSONResponse: + return JSONResponse(schema) + + app.routes.insert(1, Route("/docs", docs, methods=["GET"])) + app.routes.insert(2, Route("/.well-known/mcp", well_known_mcp, methods=["GET"])) app.state.pwc_catalog_readiness = CatalogReadiness( catalog_client, initially_ready=catalog is not None, @@ -483,6 +545,7 @@ def create_app( global_concurrency_limit=global_concurrency_limit, trust_proxy_headers=trust_proxy_headers, ) + wrapped = MCPMethodMiddleware(wrapped) wrapped = ResponseSizeLimitMiddleware(wrapped) wrapped = OperationalTelemetryMiddleware(wrapped) return CORSMiddleware( diff --git a/mcp_server/src/pwc_mcp/models.py b/mcp_server/src/pwc_mcp/models.py index 5fdf0af..33e137e 100644 --- a/mcp_server/src/pwc_mcp/models.py +++ b/mcp_server/src/pwc_mcp/models.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field @@ -9,6 +10,13 @@ class OutputModel(BaseModel): model_config = ConfigDict(extra="forbid") +def _absolute_url(value: object) -> str | None: + if not value: + return None + url = str(value) + return f"https://paperswithcode.co{url}" if url.startswith("/") else url + + class QueryResult(OutputModel): """The exact ``data`` document that ``pwc --json`` prints.""" @@ -27,7 +35,12 @@ class PaperSummary(OutputModel): published: str | None = None citation_count: int | None = None url: str | None = None - has_official_implementation: bool + has_official_implementation: bool = Field( + description=( + "True when the catalog marks at least one linked repository as the " + "paper's official implementation; this is not an independent code audit." + ) + ) code_repository_count: int @@ -57,6 +70,7 @@ class PaperDetail(OutputModel): citation_count: int | None = None url: str | None = None pdf_url: str | None = None + code_repository_count: int tasks: list[CatalogReference] methods: list[CatalogReference] repositories: list[RepositoryReference] @@ -104,6 +118,11 @@ class PaperLineageResult(QueryResult): paper: PaperReference predecessors: list[PaperReference] successors: list[PaperReference] + coverage: Literal["explicit_catalog_links_only"] = "explicit_catalog_links_only" + coverage_note: str = ( + "Only explicit catalog relationships are returned; an empty list does not " + "prove that no predecessor or successor exists." + ) class AreaReference(OutputModel): @@ -117,6 +136,7 @@ class BenchmarkSummary(OutputModel): slug: str | None = None full_name: str | None = None description: str | None = None + split: str | None = None hf_url: str | None = None paper_count: int @@ -159,6 +179,13 @@ class BenchmarkPage(QueryResult): next_page: int | None = None +class EvaluationRankScope(OutputModel): + task_id: str | None = None + task_name: str | None = None + task_slug: str | None = None + rank: int | None = None + + class Evaluation(OutputModel): id: str model_name: str @@ -171,8 +198,22 @@ class Evaluation(OutputModel): paper_title: str | None = None paper_arxiv_id: str | None = None paper_published: str | None = None - is_open: bool + rank_scopes: list[EvaluationRankScope] = Field(default_factory=list) + is_open: bool | None = Field( + description=( + "Catalog openness flag for the evaluated implementation: true=open, " + "false=closed, null=not recorded." + ) + ) num_parameters: int | None = None + split: str | None = None + shots: int | None = None + evaluation_protocol: str | None = None + source_url: str | None = None + code_url: str | None = None + hf_model_url: str | None = None + updated_at: str | None = None + uses_additional_data: bool | None = None class BenchmarkResult(QueryResult): @@ -180,6 +221,21 @@ class BenchmarkResult(QueryResult): evaluation_count: int matched_count: int | None = None evaluations: list[Evaluation] + metric_directions: dict[str, Literal["higher", "lower", "unknown"]] + page: int + next_page: int | None = None + ranking_note: str = ( + "Ranks are scoped by task and are not necessarily comparable across " + "rank_scopes." + ) + + +class EvaluationPage(QueryResult): + paper: str + evaluation_count: int + page: int + next_page: int | None = None + evaluations: list[Evaluation] def _text(value: Any) -> str | None: @@ -226,16 +282,20 @@ def catalog_reference(item: dict[str, Any]) -> CatalogReference: ) -def paper_detail(item: dict[str, Any]) -> PaperDetail: +def paper_detail( + item: dict[str, Any], *, include_resources: bool = False, repo_limit: int = 5 +) -> PaperDetail: repositories = [] for repository in item.get("repositories") or []: if isinstance(repository, dict) and repository.get("url"): - repositories.append( - RepositoryReference( - url=str(repository["url"]), - is_official=repository.get("is_official") is True, - ) + value = RepositoryReference( + url=str(repository["url"]), + is_official=repository.get("is_official") is True, ) + if include_resources or value.is_official: + repositories.append(value) + repositories.sort(key=lambda repository: not repository.is_official) + repositories = repositories[:repo_limit] return PaperDetail( id=str(item.get("id") or ""), arxiv_id=_text(item.get("arxiv_id")), @@ -246,6 +306,7 @@ def paper_detail(item: dict[str, Any]) -> PaperDetail: citation_count=_int(item.get("citation_count")), url=_text(item.get("url_abs") or item.get("source_url")), pdf_url=_text(item.get("url_pdf")), + code_repository_count=int(item.get("code_repository_count") or 0), tasks=[ catalog_reference(task) for task in item.get("tasks") or [] @@ -257,10 +318,18 @@ def paper_detail(item: dict[str, Any]) -> PaperDetail: if isinstance(method, dict) ], repositories=repositories, - project_pages=_urls(item.get("project_pages")), - hf_models=_urls(item.get("hf_models")), - hf_datasets=_urls(item.get("hf_datasets")), - hf_spaces=_urls(item.get("hf_spaces")), + project_pages=_urls(item.get("project_pages"))[:repo_limit] + if include_resources + else [], + hf_models=_urls(item.get("hf_models"))[:repo_limit] + if include_resources + else [], + hf_datasets=_urls(item.get("hf_datasets"))[:repo_limit] + if include_resources + else [], + hf_spaces=_urls(item.get("hf_spaces"))[:repo_limit] + if include_resources + else [], ) @@ -308,6 +377,7 @@ def benchmark_summary(item: dict[str, Any]) -> BenchmarkSummary: slug=_text(item.get("slug")), full_name=_text(item.get("full_name")), description=_text(item.get("description")), + split=_text(item.get("split_name") or item.get("split")), hf_url=_text(item.get("hf_url")), paper_count=_int(count) or 0, ) @@ -318,7 +388,7 @@ def evaluation(item: dict[str, Any]) -> Evaluation: id=str(item.get("id") or ""), model_name=str(item.get("model_name") or "Unknown model"), harness=_text(item.get("harness")), - metrics=_metrics(item), + metrics={name: _metric_value(value) for name, value in _metrics(item).items()}, best_metric=_text(item.get("best_metric")), best_rank=_int(item.get("best_rank")), task=_text(item.get("task_name")), @@ -326,6 +396,105 @@ def evaluation(item: dict[str, Any]) -> Evaluation: paper_title=_text(item.get("paper_title")), paper_arxiv_id=_text(item.get("paper_arxiv_id")), paper_published=_text(item.get("paper_published_date")), - is_open=item.get("is_open") is not False, + rank_scopes=_rank_scopes(item), + is_open=( + item.get("is_open") if isinstance(item.get("is_open"), bool) else None + ), num_parameters=_int(item.get("num_parameters")), + split=_text( + item.get("split") or item.get("split_name") or item.get("evaluated_on") + ), + shots=_shot_count(item.get("methodology")), + evaluation_protocol=_text(item.get("methodology")), + source_url=_absolute_url( + item.get("result_url") + or item.get("source_url") + or item.get("external_source_url") + ), + code_url=_absolute_url(item.get("code_url")), + hf_model_url=_absolute_url(item.get("hf_model_url")), + updated_at=_text(item.get("updated_at")), + uses_additional_data=( + item.get("uses_additional_data") + if isinstance(item.get("uses_additional_data"), bool) + else None + ), ) + + +def _rank_scopes(item: dict[str, Any]) -> list[EvaluationRankScope]: + existing = item.get("rank_scopes") + if isinstance(existing, list): + return [ + EvaluationRankScope.model_validate(scope) + for scope in existing + if isinstance(scope, dict) + ] + if not any( + item.get(field) is not None for field in ("task_id", "task_name", "best_rank") + ): + return [] + return [ + EvaluationRankScope( + task_id=_text(item.get("task_id")), + task_name=_text(item.get("task_name")), + task_slug=_text(item.get("task_slug")), + rank=_int(item.get("best_rank")), + ) + ] + + +def _shot_count(methodology: object) -> int | None: + match = re.search(r"\b(\d+)\s*[- ]?shot\b", str(methodology), re.IGNORECASE) + return int(match.group(1)) if match else None + + +def _metric_value(value: Any) -> float | int | str | None: + if value is None or isinstance(value, (int, float)) and not isinstance(value, bool): + return value + if isinstance(value, str): + try: + return float(value) + except ValueError: + return value + return str(value) + + +def metric_directions( + evaluations: list[Evaluation], +) -> dict[str, Literal["higher", "lower", "unknown"]]: + lower = { + "error", + "loss", + "perplexity", + "latency", + "runtime", + "wer", + "cer", + "eer", + "fid", + "mae", + "rmse", + } + higher = { + "accuracy", + "precision", + "recall", + "f1", + "bleu", + "rouge", + "map", + "auc", + "score", + "em", + } + result: dict[str, Literal["higher", "lower", "unknown"]] = {} + for name in sorted({key for row in evaluations for key in row.metrics}): + words = set(re.sub(r"[^a-z0-9]+", " ", name.casefold()).split()) + if words & lower: + result[name] = "lower" + elif words & higher: + result[name] = "higher" + else: + result[name] = "unknown" + return result diff --git a/mcp_server/src/pwc_mcp/server.py b/mcp_server/src/pwc_mcp/server.py index f2d9d7e..32e2f74 100644 --- a/mcp_server/src/pwc_mcp/server.py +++ b/mcp_server/src/pwc_mcp/server.py @@ -14,16 +14,16 @@ from collections.abc import Mapping from datetime import date from functools import cache -from typing import Annotated, Any, Literal, Protocol +from typing import Annotated, Any, Literal, Protocol, TypeVar from mcp.server.caching import CacheHint from mcp.server.mcpserver import MCPServer from mcp.server.mcpserver.exceptions import ToolError -from mcp.types import ToolAnnotations +from mcp.types import CallToolResult, TextContent, ToolAnnotations from pwc_cli import queries from pwc_cli.cli import UsageError -from pwc_cli.transport import ResponseError, TransportError -from pydantic import Field +from pwc_cli.transport import HTTPStatusError, ResponseError, TransportError +from pydantic import BaseModel, Field from pwc_mcp import __version__ from pwc_mcp.catalog import PaperMarkdownChunk, PaperVersionMismatchError @@ -37,6 +37,7 @@ AreaReference, BenchmarkPage, BenchmarkResult, + EvaluationPage, MethodDetail, MethodResult, PaperInfoResult, @@ -49,6 +50,7 @@ benchmark_summary, catalog_reference, evaluation, + metric_directions, paper_detail, paper_evaluation, paper_reference, @@ -84,17 +86,23 @@ def catalog_error_message(error: Exception) -> str: """Return the CLI's own lookup message when it is actionable, else a generic one.""" + if isinstance(error, HTTPStatusError) and error.status == 404: + return "not_found: the requested catalog record does not exist" + if isinstance(error, TransportError) and "timeout" in str(error).casefold(): + return "upstream_timeout: the Papers With Code catalog timed out" if isinstance(error, ResponseError) and not isinstance(error, TransportError): message = str(error) if message.startswith(CLIENT_FACING_ERRORS): - return message - return GENERIC_ERROR + code = "ambiguous" if "ambiguous" in message.casefold() else "not_found" + return f"{code}: {message}" + return f"upstream_error: {GENERIC_ERROR}" # One read-only CLI command per tool. TOOL_COMMANDS: dict[str, tuple[str, ...]] = { "search_papers": ("search",), "get_paper_info": ("paper", "info"), + "get_paper_evaluations": ("paper", "evaluations"), "read_paper": ("paper", "read"), "list_papers": ("paper", "list"), "list_recent_papers": ("paper", "recent"), @@ -134,7 +142,11 @@ def catalog_error_message(error: Exception) -> str: "get_benchmark": "benchmark", } # Parameters with no CLI flag; ``read_paper`` continues with a signed cursor. -MCP_ONLY_PARAMETERS: dict[str, frozenset[str]] = {"read_paper": frozenset({"cursor"})} +MCP_ONLY_PARAMETERS: dict[str, frozenset[str]] = { + "read_paper": frozenset({"cursor"}), + "get_paper_info": frozenset({"repo_limit"}), + "get_task": frozenset({"benchmark_limit"}), +} _CLI_DESTINATIONS = {mcp: cli for cli, mcp in PARAMETER_NAMES.items()} @@ -189,6 +201,23 @@ def cli_options(tool: str, **parameters: Any) -> dict[str, Any]: description="Exact author names, numeric IDs, or @HF_USERNAME; every author must match.", ), ] +ResourceLimit = Annotated[int, Field(ge=1, le=10)] +Output = TypeVar("Output", bound=BaseModel) + + +def _tool_result(value: Output, markdown: str) -> CallToolResult: + return CallToolResult( + content=[TextContent(type="text", text=markdown)], + structured_content=value.model_dump(mode="json"), + ) + + +def _structured_model(result: Any, model: type[Output]) -> Output: + if isinstance(result, CallToolResult): + return model.model_validate(result.structured_content) + return result + + Area = Annotated[str, Field(description="Case-insensitive exact area name or area ID.")] Direction = Literal["asc", "desc"] ParameterSize = Annotated[ @@ -299,6 +328,14 @@ def build_server( "Read-only access to papers, tasks, methods, conferences, organizations, " "frameworks, and benchmarks; every pwc CLI research command and flag." ), + instructions=( + "Use search_papers for relevance and list tools for deterministic filters. " + "Pass slugs or numeric IDs returned by list tools to exact lookup tools. " + "Dates use YYYY-MM-DD. Structured data is in structuredContent; text is a " + "short Markdown summary. Official implementation means catalog-designated " + "official code, not an independent audit. Evaluation is_open describes the " + "cataloged implementation and may be null when unknown." + ), version=__version__, website_url="https://paperswithcode.co", cache_hints={ @@ -330,7 +367,7 @@ def search_papers( ) -> PaperPage: """Search papers by title, topic, author, or arXiv ID (`pwc search`). Broad discovery only: for best, top, or state-of-the-art model questions start with get_task, list_benchmarks, and get_benchmark, which return leaderboard evidence that search cannot.""" _validate_date_range(published_after, published_before) - return _paper_page( + result = _paper_page( run( "search_papers", query=query, @@ -342,25 +379,35 @@ def search_papers( has_official_implementation=has_official_implementation, ) ) + return _tool_result( + result, + f"Found {len(result.items)} papers." + + (f" Next page: {result.next_page}." if result.next_page else ""), + ) @server.tool(annotations=READ_ONLY, structured_output=True) def get_paper_info( paper: Reference, - include_resources: bool = True, + include_resources: bool = False, + repo_limit: ResourceLimit = 5, include_evaluations: bool = False, ) -> PaperInfoResult: - """Get paper metadata, abstract, tasks, methods, lineage, repositories, Hugging Face artifacts, and optionally every evaluation (`pwc paper info`).""" + """Get compact paper metadata and official-first code. Set include_resources for additional repositories/Hugging Face artifacts, repo_limit to cap each list, and include_evaluations only for bounded legacy compatibility (`pwc paper info`).""" data = run( "get_paper_info", paper=paper, - include_resources=include_resources, + # Fetch once so the compact default can retain official code while + # omitting hundreds of community repositories from the response. + include_resources=True, include_evaluations=include_evaluations, ) if not isinstance(data, dict): raise TypeError("paper response did not contain a paper") evaluations = data.get("evaluations") - return PaperInfoResult( - paper=paper_detail(data), + result = PaperInfoResult( + paper=paper_detail( + data, include_resources=include_resources, repo_limit=repo_limit + ), evaluation_count=( int(evaluations.get("count") or 0) if isinstance(evaluations, dict) @@ -371,7 +418,33 @@ def get_paper_info( if isinstance(evaluations, dict) else None ), - data=data, + data=None, + ) + return _tool_result( + result, + f"## {result.paper.title}\n\n{result.paper.code_repository_count} code repositories; returned {len(result.paper.repositories)}.", + ) + + @server.tool(annotations=READ_ONLY, structured_output=True) + def get_paper_evaluations( + paper: Reference, page: Page = 1, limit: Limit = 20 + ) -> EvaluationPage: + """Get one page of benchmark evaluations for a paper, including protocol, source, openness, and task-scoped ranks (`pwc paper evaluations`).""" + data = run("get_paper_evaluations", paper=paper, page=page, limit=limit) + rows = _dicts(data.get("results")) if isinstance(data, dict) else [] + values = [evaluation(row) for row in rows] + result = EvaluationPage( + paper=paper, + evaluation_count=int(data.get("count") or len(values)), + page=page, + next_page=_next_page(data), + evaluations=values, + data=None, + ) + return _tool_result( + result, + f"Found {result.evaluation_count} evaluation rows; returned {len(values)} on page {page}." + + (f" Next page: {result.next_page}." if result.next_page else ""), ) @server.tool(annotations=READ_ONLY, structured_output=True) @@ -411,6 +484,12 @@ def read_paper(paper: Reference, cursor: str | None = None) -> PaperReadResult: raise ToolError( "paper changed; restart reading from the beginning" ) from error + except HTTPStatusError as error: + if error.status == 404: + raise ToolError( + "no_markdown: no stored Markdown is available for this paper" + ) from error + raise ToolError(catalog_error_message(error)) from error except (ResponseError, TransportError) as error: raise ToolError(catalog_error_message(error)) from error if chunk.paper != canonical or chunk.source != source: @@ -525,14 +604,14 @@ def get_paper_lineage(paper: Reference) -> PaperLineageResult: ) @server.tool(annotations=READ_ONLY, structured_output=True) - def get_task(task: Entity) -> TaskResult: + def get_task(task: Entity, benchmark_limit: ResourceLimit = 10) -> TaskResult: """Get one exact task by name, slug, or ID with its hierarchy, ranked benchmarks, common methods, and trending papers (`pwc task --name`). Start here for any question about the best or state-of-the-art models for a task, then inspect a leaderboard with get_benchmark.""" data = run("get_task", task=task) item = data.get("task") if isinstance(data, dict) else None if not isinstance(item, dict): raise TypeError("task response did not contain a task") area_item = data.get("area") - return TaskResult( + result = TaskResult( task=TaskDetail( id=str(item.get("id") or ""), name=str(item.get("name") or "Unknown task"), @@ -555,13 +634,18 @@ def get_task(task: Entity) -> TaskResult: children=[catalog_reference(v) for v in _dicts(data.get("children"))], benchmarks=[ benchmark_summary(v) for v in _dicts(data.get("benchmarks")) - ], + ][:benchmark_limit], ), - data=data, + data=None, + ) + return _tool_result( + result, + f"## {result.task.name}\n\n{result.task.paper_count} papers; returned {len(result.task.benchmarks)} benchmarks.", ) @server.tool(annotations=READ_ONLY, structured_output=True) def list_tasks( + search: str | None = None, area: Area | None = None, level: int | None = None, visible_only: bool = False, @@ -575,6 +659,7 @@ def list_tasks( return QueryResult( data=run( "list_tasks", + search=search, area=area, level=level, visible_only=visible_only, @@ -593,7 +678,7 @@ def get_method(method: Entity) -> MethodResult: item = data.get("method") if isinstance(data, dict) else None if not isinstance(item, dict): raise TypeError("method response did not contain a method") - return MethodResult( + result = MethodResult( method=MethodDetail( id=str(item.get("id") or ""), name=str(item.get("name") or "Unknown method"), @@ -612,17 +697,27 @@ def get_method(method: Entity) -> MethodResult: if item.get("source_paper_id") else None ), - source_url=str(item["source_url"]) if item.get("source_url") else None, + source_url=( + f"https://paperswithcode.co{item['source_url']}" + if str(item.get("source_url") or "").startswith("/") + else str(item["source_url"]) + if item.get("source_url") + else None + ), source_title=( str(item["source_title"]) if item.get("source_title") else None ), paper_count=int(item.get("paper_count") or 0), ), - data=data, + data=None, + ) + return _tool_result( + result, f"## {result.method.name}\n\n{result.method.paper_count} papers." ) @server.tool(annotations=READ_ONLY, structured_output=True) def list_methods( + search: str | None = None, area: Area | None = None, introduced_year: int | None = None, order_by: Literal[ @@ -636,6 +731,7 @@ def list_methods( return QueryResult( data=run( "list_methods", + search=search, area=area, introduced_year=introduced_year, order_by=order_by, @@ -686,6 +782,7 @@ def list_frameworks( @server.tool(annotations=READ_ONLY, structured_output=True) def get_benchmark( benchmark: Entity, + page: Page = 1, limit: Limit = 20, is_open: bool | None = None, max_parameters: ParameterSize | None = None, @@ -699,6 +796,7 @@ def get_benchmark( data = run( "get_benchmark", benchmark=benchmark, + page=page, limit=limit, is_open=is_open, max_parameters=max_parameters, @@ -712,12 +810,21 @@ def get_benchmark( if not isinstance(item, dict): raise TypeError("benchmark response did not contain a benchmark") matched = data.get("matched_count") - return BenchmarkResult( + evaluations = [evaluation(value) for value in _dicts(data.get("results"))] + result = BenchmarkResult( benchmark=benchmark_summary(item), evaluation_count=int(data.get("count") or 0), matched_count=int(matched) if matched is not None else None, - evaluations=[evaluation(value) for value in _dicts(data.get("results"))], - data=data, + evaluations=evaluations, + metric_directions=metric_directions(evaluations), + page=page, + next_page=_next_page(data), + data=None, + ) + return _tool_result( + result, + f"## {result.benchmark.name}\n\n{result.evaluation_count} evaluation rows; returned {len(result.evaluations)} models on page {page}. Ranks are task-scoped." + + (f" Next page: {result.next_page}." if result.next_page else ""), ) @server.tool(annotations=READ_ONLY, structured_output=True) @@ -764,6 +871,27 @@ def list_benchmarks( data=data, ) + @server.prompt(name="find_papers", title="Find papers") + def find_papers_prompt(topic: str) -> str: + return ( + f"Find papers about {topic!r}. Use semantic search for concepts, then " + "inspect the most relevant papers and cite their canonical URLs." + ) + + @server.prompt(name="compare_leaderboard", title="Compare a leaderboard") + def compare_leaderboard_prompt(benchmark: str) -> str: + return ( + f"Inspect the {benchmark!r} benchmark. Explain metric direction and " + "evaluation settings, and do not compare ranks from different task scopes." + ) + + @server.prompt(name="survey_task", title="Survey a research task") + def survey_task_prompt(task: str) -> str: + return ( + f"Survey the {task!r} task. Resolve its slug, inspect high-coverage " + "benchmarks, then summarize representative papers and methods." + ) + @server.resource( "pwc://papers/{paper}", name="paper-info", @@ -772,7 +900,9 @@ def list_benchmarks( mime_type="application/json", ) def paper_info_resource(paper: str) -> str: - return get_paper_info(paper).model_dump_json() + return _structured_model( + get_paper_info(paper), PaperInfoResult + ).model_dump_json() @server.resource( "pwc://papers/{paper}/markdown", @@ -797,7 +927,7 @@ def paper_markdown_resource(paper: str) -> str: mime_type="application/json", ) def task_resource(task: str) -> str: - return get_task(task).model_dump_json() + return _structured_model(get_task(task), TaskResult).model_dump_json() @server.resource( "pwc://benchmarks/{benchmark}", @@ -807,6 +937,8 @@ def task_resource(task: str) -> str: mime_type="application/json", ) def benchmark_resource(benchmark: str) -> str: - return get_benchmark(benchmark).model_dump_json() + return _structured_model( + get_benchmark(benchmark), BenchmarkResult + ).model_dump_json() return server diff --git a/mcp_server/tests/test_app.py b/mcp_server/tests/test_app.py index 846e03e..a2a06a9 100644 --- a/mcp_server/tests/test_app.py +++ b/mcp_server/tests/test_app.py @@ -43,14 +43,30 @@ def test_health_and_browser_origin_policy_are_explicit(): assert health.json() == { "status": "ok", "service": "pwc-mcp", - "version": "0.2.0", - "protocol": "2026-07-28", + "version": "0.2.1", + "protocol": "2025-11-25", } assert rejected.status_code == 403 assert preflight.status_code == 200 assert preflight.headers["access-control-allow-origin"] == "https://chatgpt.com" +def test_discovery_schema_and_bare_get_are_bounded(): + app = create_app(StubCatalog(), allowed_hosts=["testserver"]) + + with TestClient(app) as client: + discovery = client.get("/.well-known/mcp") + schema = client.get("/docs") + bare_get = client.get("/mcp") + + assert discovery.status_code == 200 + assert discovery.json()["protocol_version"] == "2025-11-25" + assert discovery.json()["documentation_url"].endswith("/mcp/schema") + assert len(schema.json()["tools"]) == 21 + assert bare_get.status_code == 405 + assert bare_get.headers["allow"] == "POST" + + def test_wildcard_browser_origin_is_rejected_at_startup(): try: create_app(StubCatalog(), allowed_origins=["*"]) diff --git a/mcp_server/tests/test_parity.py b/mcp_server/tests/test_parity.py index c25541b..0a0f36e 100644 --- a/mcp_server/tests/test_parity.py +++ b/mcp_server/tests/test_parity.py @@ -18,7 +18,7 @@ from test_server import StubCatalog # Deliberate default divergences from the CLI, documented in README.md. -DEFAULT_EXCEPTIONS = {("get_paper_info", "include_resources"): True} +DEFAULT_EXCEPTIONS = {} def _tools(): @@ -87,6 +87,7 @@ def test_paper_references_and_entity_names_are_required_everywhere(): assert tools[tool].input_schema["required"] == [entity] for tool in ( "get_paper_info", + "get_paper_evaluations", "read_paper", "get_related_papers", "get_paper_lineage", diff --git a/mcp_server/tests/test_server.py b/mcp_server/tests/test_server.py index 5f12a98..9ea738a 100644 --- a/mcp_server/tests/test_server.py +++ b/mcp_server/tests/test_server.py @@ -121,6 +121,12 @@ ("paper", "recent"): [PAPER_ROW], ("paper", "trending"): [PAPER_ROW], ("paper", "related"): {"results": [PAPER_ROW]}, + ("paper", "evaluations"): { + "count": 1, + "page": 1, + "next_page": 2, + "results": [EVALUATION_ROW], + }, ("paper", "lineage", "list"): { "paper": { "id": 755, @@ -153,6 +159,8 @@ "benchmark": {"id": "72", "name": "ImageNet-1k", "slug": "imagenet-1k"}, "count": 1, "matched_count": 1, + "page": 1, + "next_page": 2, "results": [EVALUATION_ROW], }, ("benchmark", "list"): {"next_page": None, "results": [BENCHMARK]}, @@ -322,7 +330,7 @@ def query(self, command, options): assert result.is_error is True assert result.content[0].text == ( "Error executing tool search_papers: " - "the Papers With Code catalog request failed" + "upstream_error: the Papers With Code catalog request failed" ) assert secret_query not in caplog.text @@ -348,10 +356,10 @@ def query(self, command, options): assert missing.content[0].text == ( "Error executing tool get_task: " - "Task not found: language-modelling; closest results: Language Modeling" + "not_found: Task not found: language-modelling; closest results: Language Modeling" ) assert transport.content[0].text == ( - "Error executing tool get_task: the Papers With Code catalog request failed" + "Error executing tool get_task: not_found: the requested catalog record does not exist" ) assert "language-modelling" not in caplog.text @@ -405,9 +413,10 @@ async def exercise(): {"id": "6", "name": "Machine Translation", "slug": "machine-translation"} ] assert paper["repositories"][0]["is_official"] is True - assert paper["hf_models"] == ["https://huggingface.co/google-t5/t5-base"] + assert paper["hf_models"] == [] + assert paper["code_repository_count"] == 595 assert info.structured_content["evaluations"] is None - assert info.structured_content["data"]["abstract"] == "A transformer architecture." + assert info.structured_content["data"] is None assert catalog.queries[1][1]["include_evals"] is True assert evaluated.structured_content["evaluation_count"] == 1 @@ -441,6 +450,26 @@ async def exercise(): assert catalog.read_calls == [(0, None, 5), (5, "a" * 64, 5)] +def test_paper_evaluations_are_paginated_and_compact(): + catalog = StubCatalog() + (result,) = _call( + catalog, + [("get_paper_evaluations", {"paper": "1706.03762", "limit": 5})], + ) + + assert catalog.options(("paper", "evaluations")) == { + "paper": "1706.03762", + "page": 1, + "page_size": 5, + } + assert result.structured_content["next_page"] == 2 + assert result.structured_content["data"] is None + assert ( + result.structured_content["evaluations"][0]["rank_scopes"][0]["task_name"] + == "Image Classification" + ) + + def test_read_paper_rejects_invalid_continuation_as_an_expected_error(): (result,) = _call( StubCatalog(), @@ -538,7 +567,7 @@ async def exercise(): tool_names, task, method, benchmarks, benchmark = asyncio.run(exercise()) assert tool_names == set(TOOL_COMMANDS) - assert len(tool_names) == 20 + assert len(tool_names) == 21 assert catalog.options(("task",)) == {"name": "image-classification"} assert catalog.options(("method",)) == {"name": "transformer"} assert catalog.options(("benchmark", "list")) == { @@ -557,6 +586,7 @@ async def exercise(): } assert catalog.options(("benchmark",)) == { "name": "imagenet-1k", + "page": 1, "limit": 5, "is_open": True, "max_parameters": "4B", @@ -568,27 +598,23 @@ async def exercise(): } assert task.structured_content["task"]["area"] == {"id": "1", "name": "Vision"} assert task.structured_content["task"]["benchmarks"][0]["slug"] == "imagenet-1k" - assert task.structured_content["data"]["common_methods"][0]["name"] == "Transformer" + assert task.structured_content["data"] is None assert method.structured_content["method"]["introduced_year"] == 2017 assert benchmarks.structured_content["items"][0]["slug"] == "imagenet-1k" assert benchmark.structured_content["matched_count"] == 1 - assert benchmark.structured_content["evaluations"] == [ + assert benchmark.structured_content["next_page"] == 2 + evaluation = benchmark.structured_content["evaluations"][0] + assert evaluation["model_name"] == "ExampleNet" + assert evaluation["metrics"] == {"Accuracy": 90.1} + assert evaluation["rank_scopes"] == [ { - "id": "10", - "model_name": "ExampleNet", - "harness": "timm", - "metrics": {"Accuracy": 90.1}, - "best_metric": "Accuracy", - "best_rank": 1, - "task": "Image Classification", - "paper_id": "755", - "paper_title": "Attention Is All You Need", - "paper_arxiv_id": "1706.03762", - "paper_published": "2017-06-12", - "is_open": True, - "num_parameters": 1000, + "task_id": None, + "task_name": "Image Classification", + "task_slug": None, + "rank": 1, } ] + assert benchmark.structured_content["metric_directions"] == {"Accuracy": "higher"} def test_grouped_listings_omit_pagination_and_flatten_benchmarks(): @@ -605,6 +631,7 @@ def test_grouped_listings_omit_pagination_and_flatten_benchmarks(): assert catalog.queries[0] == ( ("task", "list"), { + "search": None, "area": "Vision", "level": 1, "visible_only": False, @@ -631,6 +658,7 @@ def test_grouped_listings_omit_pagination_and_flatten_benchmarks(): "slug": "imagenet-1k", "full_name": None, "description": None, + "split": None, "hf_url": None, "paper_count": 124, } @@ -658,6 +686,7 @@ def test_new_catalog_tools_return_the_cli_json_document(): "data": PAYLOADS[TOOL_COMMANDS[name]], } assert catalog.options(("method", "list")) == { + "search": None, "area": None, "introduced_year": 2017, "order_by": "paper_count", diff --git a/mcp_server/uv.lock b/mcp_server/uv.lock index a620cde..129c972 100644 --- a/mcp_server/uv.lock +++ b/mcp_server/uv.lock @@ -443,7 +443,7 @@ source = { editable = "../standalone_cli" } [[package]] name = "pwc-mcp" -version = "0.2.0" +version = "0.2.1" source = { editable = "." } dependencies = [ { name = "mcp" }, diff --git a/standalone_cli/SKILL.md b/standalone_cli/SKILL.md index f1ebe33..187307a 100644 --- a/standalone_cli/SKILL.md +++ b/standalone_cli/SKILL.md @@ -43,6 +43,7 @@ case-insensitive but exact; ambiguous titles fail with their matching IDs. - `pwc search QUERY [--limit LIMIT] [--page PAGE] [--mode hybrid|keyword|semantic] [--start-date START_DATE] [--end-date END_DATE] [--has-official-implementation] [--implementation-coverage] [--json]` — search papers. - `pwc paper info PAPER [--include-resources] [--include-evals] [--json]` — show paper metadata including abstract. +- `pwc paper evaluations PAPER [--page PAGE] [--page-size N] [--json]` — page through one paper's benchmark evaluations. - `pwc paper read PAPER [--json]` — print stored paper Markdown. - `pwc paper list [--page PAGE] [--page-size PAGE_SIZE] [--search SEARCH] [--start-date START_DATE] [--end-date END_DATE] [--task TASK] [--method METHOD] [--conference CONFERENCE] [--framework FRAMEWORK] [--organization ORGANIZATION] [--author AUTHOR] [--all-versions] [--order-by trending|date_published|citation_count] [--order-dir asc|desc] [--include-resources] [--has-official-implementation] [--implementation-coverage] [--json]` — list and filter papers. - `pwc paper recent [--limit LIMIT] [--implementation-coverage] [--json]` — list recent papers. diff --git a/standalone_cli/scripts/check_contract.py b/standalone_cli/scripts/check_contract.py index 0d9ddc9..51c2153 100644 --- a/standalone_cli/scripts/check_contract.py +++ b/standalone_cli/scripts/check_contract.py @@ -16,6 +16,7 @@ COMMANDS = ( "search", "paper info", + "paper evaluations", "paper read", "paper list", "paper recent", diff --git a/standalone_cli/src/pwc_cli/cli.py b/standalone_cli/src/pwc_cli/cli.py index 9e9961f..45069e2 100644 --- a/standalone_cli/src/pwc_cli/cli.py +++ b/standalone_cli/src/pwc_cli/cli.py @@ -504,6 +504,36 @@ def paper_info(args: argparse.Namespace, client: Client) -> int: return 0 +def paper_evaluations(args: argparse.Namespace, client: Client) -> int: + paper = _resolve_paper(args.paper, client) + detail = client.get( + f"papers/{quote(paper, safe='.')}", {"include_resources": False} + ).json() + paper_id = detail.get("id") if isinstance(detail, dict) else None + if not paper_id: + raise ResponseError("Paper response did not contain an ID") + payload = client.get( + "evaluations/", + { + "page": args.page, + "page_size": args.page_size, + "paper_id": paper_id, + "ordering": "-benchmark_popularity", + }, + ).json() + items, count = _rows(payload) + data = { + "count": count if count is not None else len(items), + "page": args.page, + "next_page": payload.get("next_page") if isinstance(payload, dict) else None, + "results": _merged_evaluations(items), + } + if args.json: + return _emit_json(args, data) + _render_paper_evaluations(data["results"]) + return 0 + + def _paper_info_lineage_markdown(item: dict[str, Any]) -> str: reference = item.get("route_identifier") title = ( @@ -839,6 +869,7 @@ def task_list(args: argparse.Namespace, client: Client) -> int: and not args.flat and args.page == 1 and args.page_size == 50 + and args.search is None and args.level is None ) if args.group_by_area or automatic_grouping: @@ -850,6 +881,7 @@ def task_list(args: argparse.Namespace, client: Client) -> int: { "page": args.page, "page_size": args.page_size, + **({"q": args.search} if args.search else {}), "area_id": area_id, "level": args.level, "visible_only": args.visible_only, @@ -1252,6 +1284,7 @@ def method_list(args: argparse.Namespace, client: Client) -> int: { "page": args.page, "page_size": args.page_size, + **({"q": args.search} if args.search else {}), "area_id": area_id, "introduced_year": args.introduced_year, "ordering": _ordering(args.order_by, args.order_dir), @@ -1769,18 +1802,37 @@ def _benchmark_match(name: str, items: list[dict[str, Any]]) -> dict[str, Any] | def _merged_evaluations(items: list[dict[str, Any]]) -> list[dict[str, Any]]: - merged: dict[tuple[str, ...], dict[str, Any]] = {} - parameter_counts: dict[tuple[str, ...], set[int | None]] = {} + """Merge equivalent metric rows while retaining every task-scoped rank.""" + merged: dict[tuple[str, ...], list[dict[str, Any]]] = {} for item in items: - key = tuple( - str(item.get(field) or "") - for field in ( - "paper_id", - "task_id", - "dataset_id", - "model_name", - "harness", - ) + methodology = str(item.get("methodology") or "") + shot_match = re.search(r"\b(\d+)\s*[- ]?shot\b", methodology, re.IGNORECASE) + key = ( + *( + str(item.get(field) or "") + for field in ("paper_id", "dataset_id", "model_name", "harness") + ), + str( + item.get("split") + or item.get("split_name") + or item.get("evaluated_on") + or "" + ), + shot_match.group(1) if shot_match else "", + ) + metrics = dict(item.get("metrics") or {}) + candidates = merged.setdefault(key, []) + existing = next( + ( + candidate + for candidate in candidates + if all( + name not in candidate["metrics"] + or candidate["metrics"][name] == value + for name, value in metrics.items() + ) + ), + None, ) count = item.get("num_parameters") valid_count = ( @@ -1788,13 +1840,43 @@ def _merged_evaluations(items: list[dict[str, Any]]) -> list[dict[str, Any]]: if isinstance(count, int) and not isinstance(count, bool) and count > 0 else None ) - parameter_counts.setdefault(key, set()).add(valid_count) - existing = merged.get(key) + scope = { + "task_id": ( + str(item["task_id"]) if item.get("task_id") is not None else None + ), + "task_name": item.get("task_name"), + "task_slug": item.get("task_slug"), + "rank": item.get("best_rank"), + } if existing is None: - existing = {**item, "metrics": dict(item.get("metrics") or {})} - merged[key] = existing + candidates.append( + { + **item, + "metrics": metrics, + "rank_scopes": [scope], + "_parameter_counts": {valid_count}, + } + ) continue - existing["metrics"].update(item.get("metrics") or {}) + existing["metrics"].update(metrics) + known_scope = next( + ( + value + for value in existing["rank_scopes"] + if value.get("task_id") == scope["task_id"] + ), + None, + ) + if known_scope is None: + existing["rank_scopes"].append(scope) + else: + scope_ranks = [ + rank + for rank in (known_scope.get("rank"), scope.get("rank")) + if isinstance(rank, int) + ] + known_scope["rank"] = min(scope_ranks) if scope_ranks else None + existing["_parameter_counts"].add(valid_count) ranks = [ rank for rank in (existing.get("best_rank"), item.get("best_rank")) @@ -1803,11 +1885,13 @@ def _merged_evaluations(items: list[dict[str, Any]]) -> list[dict[str, Any]]: existing["best_rank"] = min(ranks) if ranks else None if not existing.get("best_metric") and item.get("best_metric"): existing["best_metric"] = item["best_metric"] - for key, counts in parameter_counts.items(): + rows = [row for candidates in merged.values() for row in candidates] + for row in rows: + counts = row.pop("_parameter_counts") parameter_count = next(iter(counts)) if len(counts) == 1 else None - merged[key]["num_parameters"] = parameter_count + row["num_parameters"] = parameter_count return sorted( - merged.values(), + rows, key=lambda item: ( item.get("best_rank") if isinstance(item.get("best_rank"), int) @@ -2052,24 +2136,26 @@ def _benchmark_evaluations( is_open: str | None, max_parameters: int | None, scan_all: bool, -) -> tuple[list[dict[str, Any]], int]: + page: int, + page_size: int, +) -> tuple[list[dict[str, Any]], int, int | None]: evaluations = [] - page = 1 + upstream_page = 1 if scan_all else page total = 0 while True: if max_parameters is None: path = f"datasets/{quote(str(benchmark_id), safe='')}/evaluations/" params = { - "page": page, - "page_size": 100, + "page": upstream_page, + "page_size": 100 if scan_all else page_size, "ordering": "best_rank", "is_open": is_open, } else: path = "evaluations/" params = { - "page": page, - "page_size": 100, + "page": upstream_page, + "page_size": 100 if scan_all else page_size, "dataset_id": str(benchmark_id), "ordering": "best_rank", "is_open": is_open, @@ -2093,18 +2179,18 @@ def _benchmark_evaluations( or (max_parameters is None and len(evaluations) >= total) or (max_parameters is not None and next_page is None) ): - return evaluations, total + return evaluations, total, next_page if isinstance(next_page, int) else None if len(evaluations) >= MAX_METRIC_SCAN_ROWS: raise ResponseError( f"metric selection requires scanning {total} evaluation rows; " f"safety limit is {MAX_METRIC_SCAN_ROWS}" ) if max_parameters is not None: - if not isinstance(next_page, int) or next_page <= page: + if not isinstance(next_page, int) or next_page <= upstream_page: raise ResponseError("API returned invalid evaluation pagination") - page = next_page + upstream_page = next_page else: - page += 1 + upstream_page += 1 def _paper_markdown(item: dict[str, Any]) -> str: @@ -2152,12 +2238,14 @@ def benchmark_detail(args: argparse.Namespace, client: Client) -> int: raise ResponseError(f"Benchmark not found: {args.name}{suffix}") scan_all = bool(_metric_requests(args)) - evaluations, total = _benchmark_evaluations( + evaluations, total, next_page = _benchmark_evaluations( client, benchmark["id"], is_open=args.is_open, max_parameters=args.max_parameters, scan_all=scan_all, + page=args.page, + page_size=args.limit, ) merged = _merged_evaluations(evaluations) if args.max_parameters is not None and any( @@ -2170,11 +2258,18 @@ def benchmark_detail(args: argparse.Namespace, client: Client) -> int: "API returned a model outside the requested parameter limit" ) selected = _select_metric_rows(merged, args) - rows = selected[: args.limit] + if scan_all: + start = (args.page - 1) * args.limit + rows = selected[start : start + args.limit] + next_page = args.page + 1 if start + args.limit < len(selected) else None + else: + rows = selected[: args.limit] data = { "benchmark": benchmark, "count": total, "matched_count": len(selected), + "page": args.page, + "next_page": next_page, "results": rows, } if args.json: @@ -2359,6 +2454,16 @@ def build_parser( ) _json(info) info.set_defaults(handler=paper_info) + evaluations = paper_commands.add_parser( + "evaluations", help="list one paper's benchmark evaluations" + ) + evaluations.add_argument( + "paper", help="ArXiv ID, external-paper numeric ID, or exact paper title" + ) + evaluations.add_argument("--page", type=_page, default=1) + evaluations.add_argument("--page-size", type=_page_size, default=20) + _json(evaluations) + evaluations.set_defaults(handler=paper_evaluations) read = paper_commands.add_parser("read", help="print stored paper Markdown") read.add_argument("paper", help="modern ArXiv ID or exact paper title") _json(read) @@ -2439,6 +2544,7 @@ def build_parser( tasks = task_commands.add_parser("list", help="list and filter research tasks") tasks.add_argument("--page", type=_page, default=1) tasks.add_argument("--page-size", type=_page_size, default=50) + tasks.add_argument("--search", help="search task names and slugs") task_display = tasks.add_mutually_exclusive_group() task_display.add_argument( "--group-by-area", @@ -2475,6 +2581,7 @@ def build_parser( ) methods.add_argument("--page", type=_page, default=1) methods.add_argument("--page-size", type=_method_page_size, default=50) + methods.add_argument("--search", help="search method names, full names, and slugs") methods.add_argument( "--area", help="case-insensitive exact area name (for example Audio) or area ID", @@ -2543,6 +2650,7 @@ def build_parser( default=20, help="maximum leaderboard rows, 1-100 (default: 20)", ) + benchmark.add_argument("--page", type=_page, default=1) benchmark.add_argument("--is-open", choices=("true", "false")) benchmark.add_argument( "--max-parameters", diff --git a/standalone_cli/src/pwc_cli/queries.py b/standalone_cli/src/pwc_cli/queries.py index 5fc9306..d5adfec 100644 --- a/standalone_cli/src/pwc_cli/queries.py +++ b/standalone_cli/src/pwc_cli/queries.py @@ -25,6 +25,7 @@ READ_ONLY_COMMANDS: tuple[Command, ...] = ( ("search",), ("paper", "info"), + ("paper", "evaluations"), ("paper", "read"), ("paper", "list"), ("paper", "recent"), diff --git a/standalone_cli/tests/test_cli.py b/standalone_cli/tests/test_cli.py index 617d46e..54ad4ca 100644 --- a/standalone_cli/tests/test_cli.py +++ b/standalone_cli/tests/test_cli.py @@ -918,6 +918,8 @@ def get(self, path, params=None): "list", "--area", "vision", + "--search", + "image", "--page-size", "1", "--order-by", @@ -936,6 +938,7 @@ def get(self, path, params=None): { "page": 1, "page_size": 1, + "q": "image", "area_id": "1", "level": None, "visible_only": False, @@ -1304,6 +1307,8 @@ def get(self, path, params=None): "list", "--area", "6", + "--search", + "wav2vec", "--introduced-year", "2020", ] @@ -1316,6 +1321,7 @@ def get(self, path, params=None): { "page": 1, "page_size": 50, + "q": "wav2vec", "area_id": "6", "introduced_year": 2020, "ordering": "name", @@ -1639,7 +1645,7 @@ def test_benchmark_detail_renders_merged_markdown_leaderboard(monkeypatch): ], } evaluations_payload = { - "count": 2, + "count": 3, "results": [ { "id": "1", @@ -1673,6 +1679,23 @@ def test_benchmark_detail_renders_merged_markdown_leaderboard(monkeypatch): "task_name": "Coding Agents", "is_open": False, }, + { + "id": "3", + "paper_id": "9", + "task_id": "8", + "dataset_id": "42", + "model_name": "Agent | One", + "metrics": {"Resolved": 55.5}, + "best_metric": "Resolved", + "best_rank": 4, + "num_parameters": 10_000_000_000, + "paper_title": "An Agent Paper", + "paper_arxiv_id": "2601.12345", + "paper_published_date": "2026-01-20", + "task_name": "Software Engineering", + "task_slug": "software-engineering", + "is_open": False, + }, ], } @@ -1701,11 +1724,30 @@ def get(self, path, params): "datasets/42/evaluations/", { "page": 1, - "page_size": 100, + "page_size": 20, "ordering": "best_rank", "is_open": None, }, ) + json_output = io.StringIO() + with redirect_stdout(json_output): + assert main(["benchmark", "--name", "SWE-Bench Pro", "--json"]) == 0 + rows = json.loads(json_output.getvalue())["data"]["results"] + assert len(rows) == 1 + assert rows[0]["rank_scopes"] == [ + { + "task_id": "3", + "task_name": "Coding Agents", + "task_slug": None, + "rank": 1, + }, + { + "task_id": "8", + "task_name": "Software Engineering", + "task_slug": "software-engineering", + "rank": 4, + }, + ] def test_benchmark_detail_renders_aligned_table_in_terminal(monkeypatch): @@ -1833,7 +1875,7 @@ def test_benchmark_detail_filters_by_inclusive_max_parameters(monkeypatch, capsy "evaluations/", { "page": 1, - "page_size": 100, + "page_size": 20, "dataset_id": "42", "ordering": "best_rank", "is_open": "true",