From ca740dea93245a8ba16a470a2dda55c8ec184106 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Thu, 17 Sep 2026 14:04:31 +0000 Subject: [PATCH 1/3] Improve MCP agent workflows and evaluation metadata --- mcp_server/README.md | 16 ++- mcp_server/SKILL.md | 13 +- mcp_server/SPEC.md | 18 ++- mcp_server/pyproject.toml | 2 +- mcp_server/src/pwc_mcp/__init__.py | 2 +- mcp_server/src/pwc_mcp/app.py | 67 ++++++++- mcp_server/src/pwc_mcp/models.py | 195 ++++++++++++++++++++++++-- mcp_server/src/pwc_mcp/server.py | 181 ++++++++++++++++++++---- mcp_server/tests/test_app.py | 20 ++- mcp_server/tests/test_parity.py | 3 +- mcp_server/tests/test_server.py | 67 ++++++--- mcp_server/uv.lock | 2 +- standalone_cli/src/pwc_cli/cli.py | 134 +++++++++++++++--- standalone_cli/src/pwc_cli/queries.py | 1 + standalone_cli/tests/test_cli.py | 44 +++++- 15 files changed, 659 insertions(+), 106 deletions(-) diff --git a/mcp_server/README.md b/mcp_server/README.md index 8d0e0eb..e5bb11b 100644 --- a/mcp_server/README.md +++ b/mcp_server/README.md @@ -3,9 +3,9 @@ Anonymous, read-only Model Context Protocol access to the public [Papers With Code](https://paperswithcode.co) catalog. -The server uses MCP `2026-07-28` over Streamable HTTP and serves legacy -2025-era clients on the same `/mcp` endpoint. It returns versioned structured -output with compact text fallbacks. +The server uses stock-client MCP `2025-11-25` over Streamable HTTP and also +serves experimental `2026-07-28` discovery on the same `/mcp` endpoint. It +returns versioned structured output with compact Markdown fallbacks. ## Run locally @@ -31,6 +31,7 @@ to typed projections such as `items` or `evaluations`. | --- | --- | | `search_papers` | `pwc search` | | `get_paper_info` | `pwc paper info` | +| `get_paper_evaluations` | `pwc paper evaluations` | | `read_paper` | `pwc paper read` (64 KiB chunks with a continuation cursor) | | `list_papers` | `pwc paper list` | | `list_recent_papers` | `pwc paper recent` | @@ -57,8 +58,8 @@ Parameter names follow the CLI flags except for the established MCP names (`--include-evals`). Terminal-only flags (`--json`, `--implementation-coverage`, `--flat`) have no parameter because MCP output is always structured. Hosted differences from the CLI: `limit` is capped at 25, -`search_papers` defaults to `keyword` mode, `get_paper_info` includes -resources by default, and `read_paper` is chunked. +`search_papers` defaults to `keyword` mode, `get_paper_info` returns compact +official-first resources, and `read_paper` is chunked. `tests/test_parity.py` fails when the CLI parser and the tool schemas drift. All tools are annotated read-only and idempotent. Search is deterministic; @@ -106,6 +107,11 @@ request, upstream, and serialized MCP response bodies are bounded to 2 MiB. The global ceiling is 128 concurrent requests by default. Markdown chunks use a bounded 256-entry/16 MiB in-memory cache. +`GET /.well-known/mcp` exposes connection metadata and `GET /docs` publishes +the live tool schema. A bare `GET /mcp` returns `405`; protocol requests use +`POST /mcp`. The server also exposes `find_papers`, `compare_leaderboard`, and +`survey_task` prompts. + ## Test ```bash diff --git a/mcp_server/SKILL.md b/mcp_server/SKILL.md index d4cc5a7..e2d3147 100644 --- a/mcp_server/SKILL.md +++ b/mcp_server/SKILL.md @@ -4,7 +4,7 @@ description: "Papers With Code MCP tools for searching and reading AI/ML papers, compatibility: "Requires an MCP client connected to https://paperswithcode.co/mcp with the Papers With Code tools available." --- -Generated for `pwc-mcp v0.2.0` and MCP protocol `2026-07-28`. +Generated for `pwc-mcp v0.2.1` and stock-client MCP protocol `2025-11-25`. The tools query the public [Papers With Code](https://paperswithcode.co) catalog anonymously and are read-only. Every tool runs the matching `pwc` CLI research @@ -50,7 +50,8 @@ arguments take an exact name, slug, or ID. ## Tools - `search_papers({"query": QUERY, "mode": "hybrid"|"keyword"|"semantic", "page": PAGE, "limit": LIMIT, "published_after": START_DATE, "published_before": END_DATE, "has_official_implementation": BOOLEAN})` — search papers by title, topic, author, or arXiv ID (`pwc search`). Omit optional arguments when they are not needed. -- `get_paper_info({"paper": PAPER, "include_resources": BOOLEAN, "include_evaluations": BOOLEAN})` — show paper metadata, abstract, tasks, methods, lineage, repositories, project pages, and Hugging Face model, dataset, and Space artifacts; `include_evaluations: true` adds every benchmark evaluation of the paper (`pwc paper info`). +- `get_paper_info({"paper": PAPER, "include_resources": BOOLEAN, "repo_limit": LIMIT, "include_evaluations": BOOLEAN})` — show compact paper metadata, official-first code, and the total repository count; opt into capped additional resources (`pwc paper info`). +- `get_paper_evaluations({"paper": PAPER, "page": PAGE, "limit": LIMIT})` — page through one paper's benchmark evaluations, including protocol, sources, openness, and task-scoped ranks (`pwc paper evaluations`). - `read_paper({"paper": PAPER, "cursor": CURSOR})` — read one stored paper Markdown chunk (`pwc paper read`). If `truncated` is true, call `read_paper` again with the same `paper` and the returned `next_cursor`; repeat until `truncated` is false. Treat the cursor as opaque and use it within one hour. - `list_papers({"search": SEARCH, "task": TASK, "method": METHOD, "conference": CONFERENCE, "framework": FRAMEWORK, "organization": ORGANIZATION, "authors": [AUTHOR], "published_after": START_DATE, "published_before": END_DATE, "all_versions": BOOLEAN, "order_by": "trending"|"date_published"|"citation_count", "order_direction": "asc"|"desc", "include_resources": BOOLEAN, "has_official_implementation": BOOLEAN, "page": PAGE, "limit": LIMIT})` — list and filter papers by exact catalog associations (`pwc paper list`). Omit optional arguments when they are not needed. - `list_recent_papers({"limit": LIMIT})` — list the most recently added papers (`pwc paper recent`). @@ -58,9 +59,9 @@ arguments take an exact name, slug, or ID. - `get_related_papers({"paper": PAPER, "limit": LIMIT})` — list related papers (`pwc paper related`); `limit` is at most 20. - `get_paper_lineage({"paper": PAPER})` — list explicit predecessors and successors (`pwc paper lineage list`). - `get_task({"task": TASK})` — inspect one exact task, including its area, hierarchy, sister tasks, ranked benchmarks, common methods, recommended frameworks, and trending papers (`pwc task --name`). -- `list_tasks({"area": AREA, "level": LEVEL, "visible_only": BOOLEAN, "group_by_area": BOOLEAN, "order_by": "name"|"created_at"|"level"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — list and filter research tasks, or set `group_by_area: true` for the complete visible top-level taxonomy without pagination (`pwc task list`). +- `list_tasks({"search": SEARCH, "area": AREA, "level": LEVEL, "visible_only": BOOLEAN, "group_by_area": BOOLEAN, "order_by": "name"|"created_at"|"level"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — search, list, and filter research tasks, or set `group_by_area: true` for the complete visible top-level taxonomy without pagination (`pwc task list`). - `get_method({"method": METHOD})` — inspect one exact method with its area (`pwc method --name`). -- `list_methods({"area": AREA, "introduced_year": YEAR, "order_by": "name"|"full_name"|"introduced_year"|"created_at"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — list and filter research methods (`pwc method list`). +- `list_methods({"search": SEARCH, "area": AREA, "introduced_year": YEAR, "order_by": "name"|"full_name"|"introduced_year"|"created_at"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — search, list, and filter research methods (`pwc method list`). - `get_conference({"conference": CONFERENCE})` — inspect one exact conference (`pwc conference --name`). - `list_conferences({"year": YEAR})` — list conferences with imported papers (`pwc conference list`). - `get_organization({"organization": ORGANIZATION})` — inspect one exact research organization (`pwc organization --name`). @@ -88,8 +89,8 @@ available and the user needs one of those capabilities. `max_parameters` when model size is part of the request and `sort_metric` or `minimum_metrics` when a specific metric matters. 2. Use `get_paper_info({"paper": PAPER})` to inspect promising results. Its - response includes repositories, project pages, and Hugging Face artifacts; - add `include_evaluations: true` to compare one paper across benchmarks. + response includes official-first code; use `get_paper_evaluations` to page + through its benchmark results. 3. Use exact `list_papers` `authors`, `task`, `method`, `conference`, `framework`, and `organization` arguments for known identities or catalog associations. Combine them to require every association; do not substitute diff --git a/mcp_server/SPEC.md b/mcp_server/SPEC.md index 3bc8501..c080c2c 100644 --- a/mcp_server/SPEC.md +++ b/mcp_server/SPEC.md @@ -13,8 +13,9 @@ server. the MCP contract. - Serve anonymous, read-only requests. Search is deterministic and contains no embedded language model. -- Use stateless Streamable HTTP at `/mcp`, supporting MCP `2026-07-28` and - legacy 2025 clients on the same endpoint. Expose `/health` for operations. +- Use stateless Streamable HTTP at `/mcp`, advertising stock-client MCP + `2025-11-25` while supporting experimental `2026-07-28` discovery. Expose + `/health`, `/.well-known/mcp`, and generated `/docs` schema routes. ## Public contract @@ -24,6 +25,7 @@ enforces this against the CLI parser): - `search_papers` (`pwc search`) - `get_paper_info` (`pwc paper info`) +- `get_paper_evaluations` (`pwc paper evaluations`) - `read_paper` (`pwc paper read`) - `list_papers` (`pwc paper list`) - `list_recent_papers` (`pwc paper recent`) @@ -47,16 +49,18 @@ Tools run the CLI handlers in-process through the shared cached transport, so validation, fail-closed filter confirmation, and the JSON payload are the CLI's. Every result includes that payload as `data` beside typed projections. Terminal-only flags have no parameter. The hosted service caps `limit` at 25, -defaults `search_papers` to keyword mode, includes paper resources by default, -and serves `read_paper` in chunks. +defaults `search_papers` to keyword mode, returns compact official-first paper +resources, and serves `read_paper` in chunks. -Expose these resource templates and no prompts: +Expose these resource templates: - `pwc://papers/{paper}` - `pwc://papers/{paper}/markdown` - `pwc://tasks/{task}` - `pwc://benchmarks/{benchmark}` +Expose `find_papers`, `compare_leaderboard`, and `survey_task` prompts. + Responses use stable, MCP-specific versioned structured outputs with a text fallback. `read_paper` performs one upstream read of at most 64 KiB per call and returns a signed opaque continuation cursor when more Markdown remains. The @@ -68,6 +72,10 @@ Paper references accept arXiv IDs, numeric PwC external IDs, arXiv/Hugging Face/Papers With Code URLs, and exact titles. Ambiguous exact titles fail rather than selecting one result. +Paper evaluations paginate. Leaderboards merge equivalent model rows across +task scopes while retaining scoped ranks, protocol, split, shots, source, +openness, and update time. Metric direction is explicit when known. + ## Safety and operations - Require a strict configurable browser Origin allowlist; native clients may diff --git a/mcp_server/pyproject.toml b/mcp_server/pyproject.toml index a1ccae7..900d508 100644 --- a/mcp_server/pyproject.toml +++ b/mcp_server/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pwc-mcp" -version = "0.2.0" +version = "0.2.1" description = "Read-only Papers With Code MCP server" readme = "README.md" requires-python = ">=3.10" diff --git a/mcp_server/src/pwc_mcp/__init__.py b/mcp_server/src/pwc_mcp/__init__.py index cab4aac..ba5b9ca 100644 --- a/mcp_server/src/pwc_mcp/__init__.py +++ b/mcp_server/src/pwc_mcp/__init__.py @@ -1,3 +1,3 @@ """Read-only Papers With Code MCP server.""" -__version__ = "0.2.0" +__version__ = "0.2.1" diff --git a/mcp_server/src/pwc_mcp/app.py b/mcp_server/src/pwc_mcp/app.py index 4d0c4c1..fca6998 100644 --- a/mcp_server/src/pwc_mcp/app.py +++ b/mcp_server/src/pwc_mcp/app.py @@ -31,19 +31,74 @@ # MCP SDK diagnostics can include peer-supplied tool names and resource URIs. # OperationalTelemetryMiddleware is the server's sole request log surface. logging.getLogger("mcp").setLevel(logging.CRITICAL + 1) -PROTOCOL_VERSION = "2026-07-28" +PROTOCOL_VERSION = "2025-11-25" +EXPERIMENTAL_PROTOCOL_VERSION = "2026-07-28" MAX_REQUEST_BODY_SIZE = 2 * 1024 * 1024 MAX_RESPONSE_BODY_SIZE = 2 * 1024 * 1024 KNOWN_TOOLS = frozenset(TOOL_COMMANDS) KNOWN_PROTOCOLS = { PROTOCOL_VERSION, - "2025-11-25", + EXPERIMENTAL_PROTOCOL_VERSION, "2025-06-18", "2025-03-26", "2024-11-05", } +async def well_known_mcp(_request: Request) -> JSONResponse: + return JSONResponse( + { + "name": "Papers With Code", + "description": "Anonymous read-only AI research catalog", + "transport": {"type": "streamable-http", "url": "/mcp"}, + "protocol_version": PROTOCOL_VERSION, + "supported_protocol_versions": sorted(KNOWN_PROTOCOLS, reverse=True), + "documentation_url": "https://paperswithcode.co/mcp/schema", + "setup_url": "https://paperswithcode.co/mcp", + } + ) + + +def _docs_schema(server) -> dict: + return { + "name": "Papers With Code MCP", + "version": __version__, + "protocol_version": PROTOCOL_VERSION, + "endpoint": "/mcp", + "tools": [ + { + "name": tool.name, + "description": tool.description, + "inputSchema": tool.parameters, + "outputSchema": tool.output_schema, + } + for tool in server._tool_manager.list_tools() + ], + } + + +class MCPMethodMiddleware: + """Reject a bare GET instead of opening an unbounded SSE response.""" + + def __init__(self, app: ASGIApp): + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if ( + scope["type"] == "http" + and scope.get("path") == "/mcp" + and scope.get("method") == "GET" + ): + response = JSONResponse( + {"error": "method_not_allowed", "allowed": ["POST"]}, + status_code=405, + headers={"Allow": "POST"}, + ) + await response(scope, receive, send) + return + await self.app(scope, receive, send) + + # Hosted defaults. The first-party chat gateway names one identity per chat # session (see _client_address), so per-client limits protect fairness while # the global ceiling protects the process. Every value is overridable through @@ -471,6 +526,13 @@ def create_app( ), ) app.routes.insert(0, Route("/health", health, methods=["GET"])) + schema = _docs_schema(server) + + async def docs(_request: Request) -> JSONResponse: + return JSONResponse(schema) + + app.routes.insert(1, Route("/docs", docs, methods=["GET"])) + app.routes.insert(2, Route("/.well-known/mcp", well_known_mcp, methods=["GET"])) app.state.pwc_catalog_readiness = CatalogReadiness( catalog_client, initially_ready=catalog is not None, @@ -483,6 +545,7 @@ def create_app( global_concurrency_limit=global_concurrency_limit, trust_proxy_headers=trust_proxy_headers, ) + wrapped = MCPMethodMiddleware(wrapped) wrapped = ResponseSizeLimitMiddleware(wrapped) wrapped = OperationalTelemetryMiddleware(wrapped) return CORSMiddleware( diff --git a/mcp_server/src/pwc_mcp/models.py b/mcp_server/src/pwc_mcp/models.py index 5fdf0af..57c1bda 100644 --- a/mcp_server/src/pwc_mcp/models.py +++ b/mcp_server/src/pwc_mcp/models.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Any, Literal from pydantic import BaseModel, ConfigDict, Field @@ -9,6 +10,13 @@ class OutputModel(BaseModel): model_config = ConfigDict(extra="forbid") +def _absolute_url(value: object) -> str | None: + if not value: + return None + url = str(value) + return f"https://paperswithcode.co{url}" if url.startswith("/") else url + + class QueryResult(OutputModel): """The exact ``data`` document that ``pwc --json`` prints.""" @@ -27,7 +35,12 @@ class PaperSummary(OutputModel): published: str | None = None citation_count: int | None = None url: str | None = None - has_official_implementation: bool + has_official_implementation: bool = Field( + description=( + "True when the catalog marks at least one linked repository as the " + "paper's official implementation; this is not an independent code audit." + ) + ) code_repository_count: int @@ -57,6 +70,7 @@ class PaperDetail(OutputModel): citation_count: int | None = None url: str | None = None pdf_url: str | None = None + code_repository_count: int tasks: list[CatalogReference] methods: list[CatalogReference] repositories: list[RepositoryReference] @@ -104,6 +118,11 @@ class PaperLineageResult(QueryResult): paper: PaperReference predecessors: list[PaperReference] successors: list[PaperReference] + coverage: Literal["explicit_catalog_links_only"] = "explicit_catalog_links_only" + coverage_note: str = ( + "Only explicit catalog relationships are returned; an empty list does not " + "prove that no predecessor or successor exists." + ) class AreaReference(OutputModel): @@ -117,6 +136,7 @@ class BenchmarkSummary(OutputModel): slug: str | None = None full_name: str | None = None description: str | None = None + split: str | None = None hf_url: str | None = None paper_count: int @@ -159,6 +179,13 @@ class BenchmarkPage(QueryResult): next_page: int | None = None +class EvaluationRankScope(OutputModel): + task_id: str | None = None + task_name: str | None = None + task_slug: str | None = None + rank: int | None = None + + class Evaluation(OutputModel): id: str model_name: str @@ -171,8 +198,22 @@ class Evaluation(OutputModel): paper_title: str | None = None paper_arxiv_id: str | None = None paper_published: str | None = None - is_open: bool + rank_scopes: list[EvaluationRankScope] = Field(default_factory=list) + is_open: bool | None = Field( + description=( + "Catalog openness flag for the evaluated implementation: true=open, " + "false=closed, null=not recorded." + ) + ) num_parameters: int | None = None + split: str | None = None + shots: int | None = None + evaluation_protocol: str | None = None + source_url: str | None = None + code_url: str | None = None + hf_model_url: str | None = None + updated_at: str | None = None + uses_additional_data: bool | None = None class BenchmarkResult(QueryResult): @@ -180,6 +221,19 @@ class BenchmarkResult(QueryResult): evaluation_count: int matched_count: int | None = None evaluations: list[Evaluation] + metric_directions: dict[str, Literal["higher", "lower", "unknown"]] + ranking_note: str = ( + "Ranks are scoped by task and are not necessarily comparable across " + "rank_scopes." + ) + + +class EvaluationPage(QueryResult): + paper: str + evaluation_count: int + page: int + next_page: int | None = None + evaluations: list[Evaluation] def _text(value: Any) -> str | None: @@ -226,16 +280,20 @@ def catalog_reference(item: dict[str, Any]) -> CatalogReference: ) -def paper_detail(item: dict[str, Any]) -> PaperDetail: +def paper_detail( + item: dict[str, Any], *, include_resources: bool = False, repo_limit: int = 5 +) -> PaperDetail: repositories = [] for repository in item.get("repositories") or []: if isinstance(repository, dict) and repository.get("url"): - repositories.append( - RepositoryReference( - url=str(repository["url"]), - is_official=repository.get("is_official") is True, - ) + value = RepositoryReference( + url=str(repository["url"]), + is_official=repository.get("is_official") is True, ) + if include_resources or value.is_official: + repositories.append(value) + repositories.sort(key=lambda repository: not repository.is_official) + repositories = repositories[:repo_limit] return PaperDetail( id=str(item.get("id") or ""), arxiv_id=_text(item.get("arxiv_id")), @@ -246,6 +304,7 @@ def paper_detail(item: dict[str, Any]) -> PaperDetail: citation_count=_int(item.get("citation_count")), url=_text(item.get("url_abs") or item.get("source_url")), pdf_url=_text(item.get("url_pdf")), + code_repository_count=int(item.get("code_repository_count") or 0), tasks=[ catalog_reference(task) for task in item.get("tasks") or [] @@ -257,10 +316,18 @@ def paper_detail(item: dict[str, Any]) -> PaperDetail: if isinstance(method, dict) ], repositories=repositories, - project_pages=_urls(item.get("project_pages")), - hf_models=_urls(item.get("hf_models")), - hf_datasets=_urls(item.get("hf_datasets")), - hf_spaces=_urls(item.get("hf_spaces")), + project_pages=_urls(item.get("project_pages"))[:repo_limit] + if include_resources + else [], + hf_models=_urls(item.get("hf_models"))[:repo_limit] + if include_resources + else [], + hf_datasets=_urls(item.get("hf_datasets"))[:repo_limit] + if include_resources + else [], + hf_spaces=_urls(item.get("hf_spaces"))[:repo_limit] + if include_resources + else [], ) @@ -308,6 +375,7 @@ def benchmark_summary(item: dict[str, Any]) -> BenchmarkSummary: slug=_text(item.get("slug")), full_name=_text(item.get("full_name")), description=_text(item.get("description")), + split=_text(item.get("split_name") or item.get("split")), hf_url=_text(item.get("hf_url")), paper_count=_int(count) or 0, ) @@ -318,7 +386,7 @@ def evaluation(item: dict[str, Any]) -> Evaluation: id=str(item.get("id") or ""), model_name=str(item.get("model_name") or "Unknown model"), harness=_text(item.get("harness")), - metrics=_metrics(item), + metrics={name: _metric_value(value) for name, value in _metrics(item).items()}, best_metric=_text(item.get("best_metric")), best_rank=_int(item.get("best_rank")), task=_text(item.get("task_name")), @@ -326,6 +394,105 @@ def evaluation(item: dict[str, Any]) -> Evaluation: paper_title=_text(item.get("paper_title")), paper_arxiv_id=_text(item.get("paper_arxiv_id")), paper_published=_text(item.get("paper_published_date")), - is_open=item.get("is_open") is not False, + rank_scopes=_rank_scopes(item), + is_open=( + item.get("is_open") if isinstance(item.get("is_open"), bool) else None + ), num_parameters=_int(item.get("num_parameters")), + split=_text( + item.get("split") or item.get("split_name") or item.get("evaluated_on") + ), + shots=_shot_count(item.get("methodology")), + evaluation_protocol=_text(item.get("methodology")), + source_url=_absolute_url( + item.get("result_url") + or item.get("source_url") + or item.get("external_source_url") + ), + code_url=_absolute_url(item.get("code_url")), + hf_model_url=_absolute_url(item.get("hf_model_url")), + updated_at=_text(item.get("updated_at")), + uses_additional_data=( + item.get("uses_additional_data") + if isinstance(item.get("uses_additional_data"), bool) + else None + ), ) + + +def _rank_scopes(item: dict[str, Any]) -> list[EvaluationRankScope]: + existing = item.get("rank_scopes") + if isinstance(existing, list): + return [ + EvaluationRankScope.model_validate(scope) + for scope in existing + if isinstance(scope, dict) + ] + if not any( + item.get(field) is not None for field in ("task_id", "task_name", "best_rank") + ): + return [] + return [ + EvaluationRankScope( + task_id=_text(item.get("task_id")), + task_name=_text(item.get("task_name")), + task_slug=_text(item.get("task_slug")), + rank=_int(item.get("best_rank")), + ) + ] + + +def _shot_count(methodology: object) -> int | None: + match = re.search(r"\b(\d+)\s*[- ]?shot\b", str(methodology), re.IGNORECASE) + return int(match.group(1)) if match else None + + +def _metric_value(value: Any) -> float | int | str | None: + if value is None or isinstance(value, (int, float)) and not isinstance(value, bool): + return value + if isinstance(value, str): + try: + return float(value) + except ValueError: + return value + return str(value) + + +def metric_directions( + evaluations: list[Evaluation], +) -> dict[str, Literal["higher", "lower", "unknown"]]: + lower = { + "error", + "loss", + "perplexity", + "latency", + "runtime", + "wer", + "cer", + "eer", + "fid", + "mae", + "rmse", + } + higher = { + "accuracy", + "precision", + "recall", + "f1", + "bleu", + "rouge", + "map", + "auc", + "score", + "em", + } + result: dict[str, Literal["higher", "lower", "unknown"]] = {} + for name in sorted({key for row in evaluations for key in row.metrics}): + words = set(re.sub(r"[^a-z0-9]+", " ", name.casefold()).split()) + if words & lower: + result[name] = "lower" + elif words & higher: + result[name] = "higher" + else: + result[name] = "unknown" + return result diff --git a/mcp_server/src/pwc_mcp/server.py b/mcp_server/src/pwc_mcp/server.py index f2d9d7e..b71a810 100644 --- a/mcp_server/src/pwc_mcp/server.py +++ b/mcp_server/src/pwc_mcp/server.py @@ -14,16 +14,16 @@ from collections.abc import Mapping from datetime import date from functools import cache -from typing import Annotated, Any, Literal, Protocol +from typing import Annotated, Any, Literal, Protocol, TypeVar from mcp.server.caching import CacheHint from mcp.server.mcpserver import MCPServer from mcp.server.mcpserver.exceptions import ToolError -from mcp.types import ToolAnnotations +from mcp.types import CallToolResult, TextContent, ToolAnnotations from pwc_cli import queries from pwc_cli.cli import UsageError -from pwc_cli.transport import ResponseError, TransportError -from pydantic import Field +from pwc_cli.transport import HTTPStatusError, ResponseError, TransportError +from pydantic import BaseModel, Field from pwc_mcp import __version__ from pwc_mcp.catalog import PaperMarkdownChunk, PaperVersionMismatchError @@ -37,6 +37,7 @@ AreaReference, BenchmarkPage, BenchmarkResult, + EvaluationPage, MethodDetail, MethodResult, PaperInfoResult, @@ -49,6 +50,7 @@ benchmark_summary, catalog_reference, evaluation, + metric_directions, paper_detail, paper_evaluation, paper_reference, @@ -84,17 +86,23 @@ def catalog_error_message(error: Exception) -> str: """Return the CLI's own lookup message when it is actionable, else a generic one.""" + if isinstance(error, HTTPStatusError) and error.status == 404: + return "not_found: the requested catalog record does not exist" + if isinstance(error, TransportError) and "timeout" in str(error).casefold(): + return "upstream_timeout: the Papers With Code catalog timed out" if isinstance(error, ResponseError) and not isinstance(error, TransportError): message = str(error) if message.startswith(CLIENT_FACING_ERRORS): - return message - return GENERIC_ERROR + code = "ambiguous" if "ambiguous" in message.casefold() else "not_found" + return f"{code}: {message}" + return f"upstream_error: {GENERIC_ERROR}" # One read-only CLI command per tool. TOOL_COMMANDS: dict[str, tuple[str, ...]] = { "search_papers": ("search",), "get_paper_info": ("paper", "info"), + "get_paper_evaluations": ("paper", "evaluations"), "read_paper": ("paper", "read"), "list_papers": ("paper", "list"), "list_recent_papers": ("paper", "recent"), @@ -134,7 +142,11 @@ def catalog_error_message(error: Exception) -> str: "get_benchmark": "benchmark", } # Parameters with no CLI flag; ``read_paper`` continues with a signed cursor. -MCP_ONLY_PARAMETERS: dict[str, frozenset[str]] = {"read_paper": frozenset({"cursor"})} +MCP_ONLY_PARAMETERS: dict[str, frozenset[str]] = { + "read_paper": frozenset({"cursor"}), + "get_paper_info": frozenset({"repo_limit"}), + "get_task": frozenset({"benchmark_limit"}), +} _CLI_DESTINATIONS = {mcp: cli for cli, mcp in PARAMETER_NAMES.items()} @@ -189,6 +201,23 @@ def cli_options(tool: str, **parameters: Any) -> dict[str, Any]: description="Exact author names, numeric IDs, or @HF_USERNAME; every author must match.", ), ] +ResourceLimit = Annotated[int, Field(ge=1, le=10)] +Output = TypeVar("Output", bound=BaseModel) + + +def _tool_result(value: Output, markdown: str) -> CallToolResult: + return CallToolResult( + content=[TextContent(type="text", text=markdown)], + structured_content=value.model_dump(mode="json"), + ) + + +def _structured_model(result: Any, model: type[Output]) -> Output: + if isinstance(result, CallToolResult): + return model.model_validate(result.structured_content) + return result + + Area = Annotated[str, Field(description="Case-insensitive exact area name or area ID.")] Direction = Literal["asc", "desc"] ParameterSize = Annotated[ @@ -299,6 +328,14 @@ def build_server( "Read-only access to papers, tasks, methods, conferences, organizations, " "frameworks, and benchmarks; every pwc CLI research command and flag." ), + instructions=( + "Use search_papers for relevance and list tools for deterministic filters. " + "Pass slugs or numeric IDs returned by list tools to exact lookup tools. " + "Dates use YYYY-MM-DD. Structured data is in structuredContent; text is a " + "short Markdown summary. Official implementation means catalog-designated " + "official code, not an independent audit. Evaluation is_open describes the " + "cataloged implementation and may be null when unknown." + ), version=__version__, website_url="https://paperswithcode.co", cache_hints={ @@ -330,7 +367,7 @@ def search_papers( ) -> PaperPage: """Search papers by title, topic, author, or arXiv ID (`pwc search`). Broad discovery only: for best, top, or state-of-the-art model questions start with get_task, list_benchmarks, and get_benchmark, which return leaderboard evidence that search cannot.""" _validate_date_range(published_after, published_before) - return _paper_page( + result = _paper_page( run( "search_papers", query=query, @@ -342,25 +379,35 @@ def search_papers( has_official_implementation=has_official_implementation, ) ) + return _tool_result( + result, + f"Found {len(result.items)} papers." + + (f" Next page: {result.next_page}." if result.next_page else ""), + ) @server.tool(annotations=READ_ONLY, structured_output=True) def get_paper_info( paper: Reference, - include_resources: bool = True, + include_resources: bool = False, + repo_limit: ResourceLimit = 5, include_evaluations: bool = False, ) -> PaperInfoResult: - """Get paper metadata, abstract, tasks, methods, lineage, repositories, Hugging Face artifacts, and optionally every evaluation (`pwc paper info`).""" + """Get compact paper metadata and official-first code. Set include_resources for additional repositories/Hugging Face artifacts, repo_limit to cap each list, and include_evaluations only for bounded legacy compatibility (`pwc paper info`).""" data = run( "get_paper_info", paper=paper, - include_resources=include_resources, + # Fetch once so the compact default can retain official code while + # omitting hundreds of community repositories from the response. + include_resources=True, include_evaluations=include_evaluations, ) if not isinstance(data, dict): raise TypeError("paper response did not contain a paper") evaluations = data.get("evaluations") - return PaperInfoResult( - paper=paper_detail(data), + result = PaperInfoResult( + paper=paper_detail( + data, include_resources=include_resources, repo_limit=repo_limit + ), evaluation_count=( int(evaluations.get("count") or 0) if isinstance(evaluations, dict) @@ -371,7 +418,33 @@ def get_paper_info( if isinstance(evaluations, dict) else None ), - data=data, + data=None, + ) + return _tool_result( + result, + f"## {result.paper.title}\n\n{result.paper.code_repository_count} code repositories; returned {len(result.paper.repositories)}.", + ) + + @server.tool(annotations=READ_ONLY, structured_output=True) + def get_paper_evaluations( + paper: Reference, page: Page = 1, limit: Limit = 20 + ) -> EvaluationPage: + """Get one page of benchmark evaluations for a paper, including protocol, source, openness, and task-scoped ranks (`pwc paper evaluations`).""" + data = run("get_paper_evaluations", paper=paper, page=page, limit=limit) + rows = _dicts(data.get("results")) if isinstance(data, dict) else [] + values = [evaluation(row) for row in rows] + result = EvaluationPage( + paper=paper, + evaluation_count=int(data.get("count") or len(values)), + page=page, + next_page=_next_page(data), + evaluations=values, + data=None, + ) + return _tool_result( + result, + f"Found {result.evaluation_count} evaluation rows; returned {len(values)} on page {page}." + + (f" Next page: {result.next_page}." if result.next_page else ""), ) @server.tool(annotations=READ_ONLY, structured_output=True) @@ -411,6 +484,12 @@ def read_paper(paper: Reference, cursor: str | None = None) -> PaperReadResult: raise ToolError( "paper changed; restart reading from the beginning" ) from error + except HTTPStatusError as error: + if error.status == 404: + raise ToolError( + "no_markdown: no stored Markdown is available for this paper" + ) from error + raise ToolError(catalog_error_message(error)) from error except (ResponseError, TransportError) as error: raise ToolError(catalog_error_message(error)) from error if chunk.paper != canonical or chunk.source != source: @@ -525,14 +604,14 @@ def get_paper_lineage(paper: Reference) -> PaperLineageResult: ) @server.tool(annotations=READ_ONLY, structured_output=True) - def get_task(task: Entity) -> TaskResult: + def get_task(task: Entity, benchmark_limit: ResourceLimit = 10) -> TaskResult: """Get one exact task by name, slug, or ID with its hierarchy, ranked benchmarks, common methods, and trending papers (`pwc task --name`). Start here for any question about the best or state-of-the-art models for a task, then inspect a leaderboard with get_benchmark.""" data = run("get_task", task=task) item = data.get("task") if isinstance(data, dict) else None if not isinstance(item, dict): raise TypeError("task response did not contain a task") area_item = data.get("area") - return TaskResult( + result = TaskResult( task=TaskDetail( id=str(item.get("id") or ""), name=str(item.get("name") or "Unknown task"), @@ -555,13 +634,18 @@ def get_task(task: Entity) -> TaskResult: children=[catalog_reference(v) for v in _dicts(data.get("children"))], benchmarks=[ benchmark_summary(v) for v in _dicts(data.get("benchmarks")) - ], + ][:benchmark_limit], ), - data=data, + data=None, + ) + return _tool_result( + result, + f"## {result.task.name}\n\n{result.task.paper_count} papers; returned {len(result.task.benchmarks)} benchmarks.", ) @server.tool(annotations=READ_ONLY, structured_output=True) def list_tasks( + search: str | None = None, area: Area | None = None, level: int | None = None, visible_only: bool = False, @@ -575,6 +659,7 @@ def list_tasks( return QueryResult( data=run( "list_tasks", + search=search, area=area, level=level, visible_only=visible_only, @@ -593,7 +678,7 @@ def get_method(method: Entity) -> MethodResult: item = data.get("method") if isinstance(data, dict) else None if not isinstance(item, dict): raise TypeError("method response did not contain a method") - return MethodResult( + result = MethodResult( method=MethodDetail( id=str(item.get("id") or ""), name=str(item.get("name") or "Unknown method"), @@ -612,17 +697,27 @@ def get_method(method: Entity) -> MethodResult: if item.get("source_paper_id") else None ), - source_url=str(item["source_url"]) if item.get("source_url") else None, + source_url=( + f"https://paperswithcode.co{item['source_url']}" + if str(item.get("source_url") or "").startswith("/") + else str(item["source_url"]) + if item.get("source_url") + else None + ), source_title=( str(item["source_title"]) if item.get("source_title") else None ), paper_count=int(item.get("paper_count") or 0), ), - data=data, + data=None, + ) + return _tool_result( + result, f"## {result.method.name}\n\n{result.method.paper_count} papers." ) @server.tool(annotations=READ_ONLY, structured_output=True) def list_methods( + search: str | None = None, area: Area | None = None, introduced_year: int | None = None, order_by: Literal[ @@ -636,6 +731,7 @@ def list_methods( return QueryResult( data=run( "list_methods", + search=search, area=area, introduced_year=introduced_year, order_by=order_by, @@ -712,12 +808,18 @@ def get_benchmark( if not isinstance(item, dict): raise TypeError("benchmark response did not contain a benchmark") matched = data.get("matched_count") - return BenchmarkResult( + evaluations = [evaluation(value) for value in _dicts(data.get("results"))] + result = BenchmarkResult( benchmark=benchmark_summary(item), evaluation_count=int(data.get("count") or 0), matched_count=int(matched) if matched is not None else None, - evaluations=[evaluation(value) for value in _dicts(data.get("results"))], - data=data, + evaluations=evaluations, + metric_directions=metric_directions(evaluations), + data=None, + ) + return _tool_result( + result, + f"## {result.benchmark.name}\n\n{result.evaluation_count} evaluation rows; returned {len(result.evaluations)} models. Ranks are task-scoped.", ) @server.tool(annotations=READ_ONLY, structured_output=True) @@ -764,6 +866,27 @@ def list_benchmarks( data=data, ) + @server.prompt(name="find_papers", title="Find papers") + def find_papers_prompt(topic: str) -> str: + return ( + f"Find papers about {topic!r}. Use semantic search for concepts, then " + "inspect the most relevant papers and cite their canonical URLs." + ) + + @server.prompt(name="compare_leaderboard", title="Compare a leaderboard") + def compare_leaderboard_prompt(benchmark: str) -> str: + return ( + f"Inspect the {benchmark!r} benchmark. Explain metric direction and " + "evaluation settings, and do not compare ranks from different task scopes." + ) + + @server.prompt(name="survey_task", title="Survey a research task") + def survey_task_prompt(task: str) -> str: + return ( + f"Survey the {task!r} task. Resolve its slug, inspect high-coverage " + "benchmarks, then summarize representative papers and methods." + ) + @server.resource( "pwc://papers/{paper}", name="paper-info", @@ -772,7 +895,9 @@ def list_benchmarks( mime_type="application/json", ) def paper_info_resource(paper: str) -> str: - return get_paper_info(paper).model_dump_json() + return _structured_model( + get_paper_info(paper), PaperInfoResult + ).model_dump_json() @server.resource( "pwc://papers/{paper}/markdown", @@ -797,7 +922,7 @@ def paper_markdown_resource(paper: str) -> str: mime_type="application/json", ) def task_resource(task: str) -> str: - return get_task(task).model_dump_json() + return _structured_model(get_task(task), TaskResult).model_dump_json() @server.resource( "pwc://benchmarks/{benchmark}", @@ -807,6 +932,8 @@ def task_resource(task: str) -> str: mime_type="application/json", ) def benchmark_resource(benchmark: str) -> str: - return get_benchmark(benchmark).model_dump_json() + return _structured_model( + get_benchmark(benchmark), BenchmarkResult + ).model_dump_json() return server diff --git a/mcp_server/tests/test_app.py b/mcp_server/tests/test_app.py index 846e03e..a2a06a9 100644 --- a/mcp_server/tests/test_app.py +++ b/mcp_server/tests/test_app.py @@ -43,14 +43,30 @@ def test_health_and_browser_origin_policy_are_explicit(): assert health.json() == { "status": "ok", "service": "pwc-mcp", - "version": "0.2.0", - "protocol": "2026-07-28", + "version": "0.2.1", + "protocol": "2025-11-25", } assert rejected.status_code == 403 assert preflight.status_code == 200 assert preflight.headers["access-control-allow-origin"] == "https://chatgpt.com" +def test_discovery_schema_and_bare_get_are_bounded(): + app = create_app(StubCatalog(), allowed_hosts=["testserver"]) + + with TestClient(app) as client: + discovery = client.get("/.well-known/mcp") + schema = client.get("/docs") + bare_get = client.get("/mcp") + + assert discovery.status_code == 200 + assert discovery.json()["protocol_version"] == "2025-11-25" + assert discovery.json()["documentation_url"].endswith("/mcp/schema") + assert len(schema.json()["tools"]) == 21 + assert bare_get.status_code == 405 + assert bare_get.headers["allow"] == "POST" + + def test_wildcard_browser_origin_is_rejected_at_startup(): try: create_app(StubCatalog(), allowed_origins=["*"]) diff --git a/mcp_server/tests/test_parity.py b/mcp_server/tests/test_parity.py index c25541b..0a0f36e 100644 --- a/mcp_server/tests/test_parity.py +++ b/mcp_server/tests/test_parity.py @@ -18,7 +18,7 @@ from test_server import StubCatalog # Deliberate default divergences from the CLI, documented in README.md. -DEFAULT_EXCEPTIONS = {("get_paper_info", "include_resources"): True} +DEFAULT_EXCEPTIONS = {} def _tools(): @@ -87,6 +87,7 @@ def test_paper_references_and_entity_names_are_required_everywhere(): assert tools[tool].input_schema["required"] == [entity] for tool in ( "get_paper_info", + "get_paper_evaluations", "read_paper", "get_related_papers", "get_paper_lineage", diff --git a/mcp_server/tests/test_server.py b/mcp_server/tests/test_server.py index 5f12a98..f1fb083 100644 --- a/mcp_server/tests/test_server.py +++ b/mcp_server/tests/test_server.py @@ -121,6 +121,12 @@ ("paper", "recent"): [PAPER_ROW], ("paper", "trending"): [PAPER_ROW], ("paper", "related"): {"results": [PAPER_ROW]}, + ("paper", "evaluations"): { + "count": 1, + "page": 1, + "next_page": 2, + "results": [EVALUATION_ROW], + }, ("paper", "lineage", "list"): { "paper": { "id": 755, @@ -322,7 +328,7 @@ def query(self, command, options): assert result.is_error is True assert result.content[0].text == ( "Error executing tool search_papers: " - "the Papers With Code catalog request failed" + "upstream_error: the Papers With Code catalog request failed" ) assert secret_query not in caplog.text @@ -348,10 +354,10 @@ def query(self, command, options): assert missing.content[0].text == ( "Error executing tool get_task: " - "Task not found: language-modelling; closest results: Language Modeling" + "not_found: Task not found: language-modelling; closest results: Language Modeling" ) assert transport.content[0].text == ( - "Error executing tool get_task: the Papers With Code catalog request failed" + "Error executing tool get_task: not_found: the requested catalog record does not exist" ) assert "language-modelling" not in caplog.text @@ -405,9 +411,10 @@ async def exercise(): {"id": "6", "name": "Machine Translation", "slug": "machine-translation"} ] assert paper["repositories"][0]["is_official"] is True - assert paper["hf_models"] == ["https://huggingface.co/google-t5/t5-base"] + assert paper["hf_models"] == [] + assert paper["code_repository_count"] == 595 assert info.structured_content["evaluations"] is None - assert info.structured_content["data"]["abstract"] == "A transformer architecture." + assert info.structured_content["data"] is None assert catalog.queries[1][1]["include_evals"] is True assert evaluated.structured_content["evaluation_count"] == 1 @@ -441,6 +448,26 @@ async def exercise(): assert catalog.read_calls == [(0, None, 5), (5, "a" * 64, 5)] +def test_paper_evaluations_are_paginated_and_compact(): + catalog = StubCatalog() + (result,) = _call( + catalog, + [("get_paper_evaluations", {"paper": "1706.03762", "limit": 5})], + ) + + assert catalog.options(("paper", "evaluations")) == { + "paper": "1706.03762", + "page": 1, + "page_size": 5, + } + assert result.structured_content["next_page"] == 2 + assert result.structured_content["data"] is None + assert ( + result.structured_content["evaluations"][0]["rank_scopes"][0]["task_name"] + == "Image Classification" + ) + + def test_read_paper_rejects_invalid_continuation_as_an_expected_error(): (result,) = _call( StubCatalog(), @@ -538,7 +565,7 @@ async def exercise(): tool_names, task, method, benchmarks, benchmark = asyncio.run(exercise()) assert tool_names == set(TOOL_COMMANDS) - assert len(tool_names) == 20 + assert len(tool_names) == 21 assert catalog.options(("task",)) == {"name": "image-classification"} assert catalog.options(("method",)) == {"name": "transformer"} assert catalog.options(("benchmark", "list")) == { @@ -568,27 +595,22 @@ async def exercise(): } assert task.structured_content["task"]["area"] == {"id": "1", "name": "Vision"} assert task.structured_content["task"]["benchmarks"][0]["slug"] == "imagenet-1k" - assert task.structured_content["data"]["common_methods"][0]["name"] == "Transformer" + assert task.structured_content["data"] is None assert method.structured_content["method"]["introduced_year"] == 2017 assert benchmarks.structured_content["items"][0]["slug"] == "imagenet-1k" assert benchmark.structured_content["matched_count"] == 1 - assert benchmark.structured_content["evaluations"] == [ + evaluation = benchmark.structured_content["evaluations"][0] + assert evaluation["model_name"] == "ExampleNet" + assert evaluation["metrics"] == {"Accuracy": 90.1} + assert evaluation["rank_scopes"] == [ { - "id": "10", - "model_name": "ExampleNet", - "harness": "timm", - "metrics": {"Accuracy": 90.1}, - "best_metric": "Accuracy", - "best_rank": 1, - "task": "Image Classification", - "paper_id": "755", - "paper_title": "Attention Is All You Need", - "paper_arxiv_id": "1706.03762", - "paper_published": "2017-06-12", - "is_open": True, - "num_parameters": 1000, + "task_id": None, + "task_name": "Image Classification", + "task_slug": None, + "rank": 1, } ] + assert benchmark.structured_content["metric_directions"] == {"Accuracy": "higher"} def test_grouped_listings_omit_pagination_and_flatten_benchmarks(): @@ -605,6 +627,7 @@ def test_grouped_listings_omit_pagination_and_flatten_benchmarks(): assert catalog.queries[0] == ( ("task", "list"), { + "search": None, "area": "Vision", "level": 1, "visible_only": False, @@ -631,6 +654,7 @@ def test_grouped_listings_omit_pagination_and_flatten_benchmarks(): "slug": "imagenet-1k", "full_name": None, "description": None, + "split": None, "hf_url": None, "paper_count": 124, } @@ -658,6 +682,7 @@ def test_new_catalog_tools_return_the_cli_json_document(): "data": PAYLOADS[TOOL_COMMANDS[name]], } assert catalog.options(("method", "list")) == { + "search": None, "area": None, "introduced_year": 2017, "order_by": "paper_count", diff --git a/mcp_server/uv.lock b/mcp_server/uv.lock index a620cde..129c972 100644 --- a/mcp_server/uv.lock +++ b/mcp_server/uv.lock @@ -443,7 +443,7 @@ source = { editable = "../standalone_cli" } [[package]] name = "pwc-mcp" -version = "0.2.0" +version = "0.2.1" source = { editable = "." } dependencies = [ { name = "mcp" }, diff --git a/standalone_cli/src/pwc_cli/cli.py b/standalone_cli/src/pwc_cli/cli.py index 9e9961f..c27c4a5 100644 --- a/standalone_cli/src/pwc_cli/cli.py +++ b/standalone_cli/src/pwc_cli/cli.py @@ -504,6 +504,36 @@ def paper_info(args: argparse.Namespace, client: Client) -> int: return 0 +def paper_evaluations(args: argparse.Namespace, client: Client) -> int: + paper = _resolve_paper(args.paper, client) + detail = client.get( + f"papers/{quote(paper, safe='.')}", {"include_resources": False} + ).json() + paper_id = detail.get("id") if isinstance(detail, dict) else None + if not paper_id: + raise ResponseError("Paper response did not contain an ID") + payload = client.get( + "evaluations/", + { + "page": args.page, + "page_size": args.page_size, + "paper_id": paper_id, + "ordering": "-benchmark_popularity", + }, + ).json() + items, count = _rows(payload) + data = { + "count": count if count is not None else len(items), + "page": args.page, + "next_page": payload.get("next_page") if isinstance(payload, dict) else None, + "results": _merged_evaluations(items), + } + if args.json: + return _emit_json(args, data) + _render_paper_evaluations(data["results"]) + return 0 + + def _paper_info_lineage_markdown(item: dict[str, Any]) -> str: reference = item.get("route_identifier") title = ( @@ -839,6 +869,7 @@ def task_list(args: argparse.Namespace, client: Client) -> int: and not args.flat and args.page == 1 and args.page_size == 50 + and args.search is None and args.level is None ) if args.group_by_area or automatic_grouping: @@ -850,6 +881,7 @@ def task_list(args: argparse.Namespace, client: Client) -> int: { "page": args.page, "page_size": args.page_size, + **({"q": args.search} if args.search else {}), "area_id": area_id, "level": args.level, "visible_only": args.visible_only, @@ -1252,6 +1284,7 @@ def method_list(args: argparse.Namespace, client: Client) -> int: { "page": args.page, "page_size": args.page_size, + **({"q": args.search} if args.search else {}), "area_id": area_id, "introduced_year": args.introduced_year, "ordering": _ordering(args.order_by, args.order_dir), @@ -1769,18 +1802,37 @@ def _benchmark_match(name: str, items: list[dict[str, Any]]) -> dict[str, Any] | def _merged_evaluations(items: list[dict[str, Any]]) -> list[dict[str, Any]]: - merged: dict[tuple[str, ...], dict[str, Any]] = {} - parameter_counts: dict[tuple[str, ...], set[int | None]] = {} + """Merge equivalent metric rows while retaining every task-scoped rank.""" + merged: dict[tuple[str, ...], list[dict[str, Any]]] = {} for item in items: - key = tuple( - str(item.get(field) or "") - for field in ( - "paper_id", - "task_id", - "dataset_id", - "model_name", - "harness", - ) + methodology = str(item.get("methodology") or "") + shot_match = re.search(r"\b(\d+)\s*[- ]?shot\b", methodology, re.IGNORECASE) + key = ( + *( + str(item.get(field) or "") + for field in ("paper_id", "dataset_id", "model_name", "harness") + ), + str( + item.get("split") + or item.get("split_name") + or item.get("evaluated_on") + or "" + ), + shot_match.group(1) if shot_match else "", + ) + metrics = dict(item.get("metrics") or {}) + candidates = merged.setdefault(key, []) + existing = next( + ( + candidate + for candidate in candidates + if all( + name not in candidate["metrics"] + or candidate["metrics"][name] == value + for name, value in metrics.items() + ) + ), + None, ) count = item.get("num_parameters") valid_count = ( @@ -1788,13 +1840,43 @@ def _merged_evaluations(items: list[dict[str, Any]]) -> list[dict[str, Any]]: if isinstance(count, int) and not isinstance(count, bool) and count > 0 else None ) - parameter_counts.setdefault(key, set()).add(valid_count) - existing = merged.get(key) + scope = { + "task_id": ( + str(item["task_id"]) if item.get("task_id") is not None else None + ), + "task_name": item.get("task_name"), + "task_slug": item.get("task_slug"), + "rank": item.get("best_rank"), + } if existing is None: - existing = {**item, "metrics": dict(item.get("metrics") or {})} - merged[key] = existing + candidates.append( + { + **item, + "metrics": metrics, + "rank_scopes": [scope], + "_parameter_counts": {valid_count}, + } + ) continue - existing["metrics"].update(item.get("metrics") or {}) + existing["metrics"].update(metrics) + known_scope = next( + ( + value + for value in existing["rank_scopes"] + if value.get("task_id") == scope["task_id"] + ), + None, + ) + if known_scope is None: + existing["rank_scopes"].append(scope) + else: + scope_ranks = [ + rank + for rank in (known_scope.get("rank"), scope.get("rank")) + if isinstance(rank, int) + ] + known_scope["rank"] = min(scope_ranks) if scope_ranks else None + existing["_parameter_counts"].add(valid_count) ranks = [ rank for rank in (existing.get("best_rank"), item.get("best_rank")) @@ -1803,11 +1885,13 @@ def _merged_evaluations(items: list[dict[str, Any]]) -> list[dict[str, Any]]: existing["best_rank"] = min(ranks) if ranks else None if not existing.get("best_metric") and item.get("best_metric"): existing["best_metric"] = item["best_metric"] - for key, counts in parameter_counts.items(): + rows = [row for candidates in merged.values() for row in candidates] + for row in rows: + counts = row.pop("_parameter_counts") parameter_count = next(iter(counts)) if len(counts) == 1 else None - merged[key]["num_parameters"] = parameter_count + row["num_parameters"] = parameter_count return sorted( - merged.values(), + rows, key=lambda item: ( item.get("best_rank") if isinstance(item.get("best_rank"), int) @@ -2359,6 +2443,16 @@ def build_parser( ) _json(info) info.set_defaults(handler=paper_info) + evaluations = paper_commands.add_parser( + "evaluations", help="list one paper's benchmark evaluations" + ) + evaluations.add_argument( + "paper", help="ArXiv ID, external-paper numeric ID, or exact paper title" + ) + evaluations.add_argument("--page", type=_page, default=1) + evaluations.add_argument("--page-size", type=_page_size, default=20) + _json(evaluations) + evaluations.set_defaults(handler=paper_evaluations) read = paper_commands.add_parser("read", help="print stored paper Markdown") read.add_argument("paper", help="modern ArXiv ID or exact paper title") _json(read) @@ -2439,6 +2533,7 @@ def build_parser( tasks = task_commands.add_parser("list", help="list and filter research tasks") tasks.add_argument("--page", type=_page, default=1) tasks.add_argument("--page-size", type=_page_size, default=50) + tasks.add_argument("--search", help="search task names and slugs") task_display = tasks.add_mutually_exclusive_group() task_display.add_argument( "--group-by-area", @@ -2475,6 +2570,7 @@ def build_parser( ) methods.add_argument("--page", type=_page, default=1) methods.add_argument("--page-size", type=_method_page_size, default=50) + methods.add_argument("--search", help="search method names, full names, and slugs") methods.add_argument( "--area", help="case-insensitive exact area name (for example Audio) or area ID", diff --git a/standalone_cli/src/pwc_cli/queries.py b/standalone_cli/src/pwc_cli/queries.py index 5fc9306..d5adfec 100644 --- a/standalone_cli/src/pwc_cli/queries.py +++ b/standalone_cli/src/pwc_cli/queries.py @@ -25,6 +25,7 @@ READ_ONLY_COMMANDS: tuple[Command, ...] = ( ("search",), ("paper", "info"), + ("paper", "evaluations"), ("paper", "read"), ("paper", "list"), ("paper", "recent"), diff --git a/standalone_cli/tests/test_cli.py b/standalone_cli/tests/test_cli.py index 617d46e..ea83472 100644 --- a/standalone_cli/tests/test_cli.py +++ b/standalone_cli/tests/test_cli.py @@ -918,6 +918,8 @@ def get(self, path, params=None): "list", "--area", "vision", + "--search", + "image", "--page-size", "1", "--order-by", @@ -936,6 +938,7 @@ def get(self, path, params=None): { "page": 1, "page_size": 1, + "q": "image", "area_id": "1", "level": None, "visible_only": False, @@ -1304,6 +1307,8 @@ def get(self, path, params=None): "list", "--area", "6", + "--search", + "wav2vec", "--introduced-year", "2020", ] @@ -1316,6 +1321,7 @@ def get(self, path, params=None): { "page": 1, "page_size": 50, + "q": "wav2vec", "area_id": "6", "introduced_year": 2020, "ordering": "name", @@ -1639,7 +1645,7 @@ def test_benchmark_detail_renders_merged_markdown_leaderboard(monkeypatch): ], } evaluations_payload = { - "count": 2, + "count": 3, "results": [ { "id": "1", @@ -1673,6 +1679,23 @@ def test_benchmark_detail_renders_merged_markdown_leaderboard(monkeypatch): "task_name": "Coding Agents", "is_open": False, }, + { + "id": "3", + "paper_id": "9", + "task_id": "8", + "dataset_id": "42", + "model_name": "Agent | One", + "metrics": {"Resolved": 55.5}, + "best_metric": "Resolved", + "best_rank": 4, + "num_parameters": 10_000_000_000, + "paper_title": "An Agent Paper", + "paper_arxiv_id": "2601.12345", + "paper_published_date": "2026-01-20", + "task_name": "Software Engineering", + "task_slug": "software-engineering", + "is_open": False, + }, ], } @@ -1706,6 +1729,25 @@ def get(self, path, params): "is_open": None, }, ) + json_output = io.StringIO() + with redirect_stdout(json_output): + assert main(["benchmark", "--name", "SWE-Bench Pro", "--json"]) == 0 + rows = json.loads(json_output.getvalue())["data"]["results"] + assert len(rows) == 1 + assert rows[0]["rank_scopes"] == [ + { + "task_id": "3", + "task_name": "Coding Agents", + "task_slug": None, + "rank": 1, + }, + { + "task_id": "8", + "task_name": "Software Engineering", + "task_slug": "software-engineering", + "rank": 4, + }, + ] def test_benchmark_detail_renders_aligned_table_in_terminal(monkeypatch): From d56f0d6b93442c762ecd5d399cc80001c209458a Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Thu, 17 Sep 2026 14:05:50 +0000 Subject: [PATCH 2/3] Document paginated paper evaluations --- backend/chat_sandbox_worker/CLI_GUIDE.md | 1 + standalone_cli/SKILL.md | 1 + standalone_cli/scripts/check_contract.py | 1 + 3 files changed, 3 insertions(+) diff --git a/backend/chat_sandbox_worker/CLI_GUIDE.md b/backend/chat_sandbox_worker/CLI_GUIDE.md index 750ce72..b4c0739 100644 --- a/backend/chat_sandbox_worker/CLI_GUIDE.md +++ b/backend/chat_sandbox_worker/CLI_GUIDE.md @@ -6,6 +6,7 @@ join, or schema-dependent transformation. ```bash pwc search "QUERY" --limit 10 [--start-date YYYY-MM-DD --end-date YYYY-MM-DD] pwc paper info PAPER --include-resources +pwc paper evaluations PAPER --page 1 --page-size 20 pwc paper read PAPER pwc paper list --search "QUERY" [--start-date YYYY-MM-DD --end-date YYYY-MM-DD] [--task NAME] [--method NAME] [--conference NAME] [--framework NAME] [--organization NAME] pwc paper recent diff --git a/standalone_cli/SKILL.md b/standalone_cli/SKILL.md index f1ebe33..187307a 100644 --- a/standalone_cli/SKILL.md +++ b/standalone_cli/SKILL.md @@ -43,6 +43,7 @@ case-insensitive but exact; ambiguous titles fail with their matching IDs. - `pwc search QUERY [--limit LIMIT] [--page PAGE] [--mode hybrid|keyword|semantic] [--start-date START_DATE] [--end-date END_DATE] [--has-official-implementation] [--implementation-coverage] [--json]` — search papers. - `pwc paper info PAPER [--include-resources] [--include-evals] [--json]` — show paper metadata including abstract. +- `pwc paper evaluations PAPER [--page PAGE] [--page-size N] [--json]` — page through one paper's benchmark evaluations. - `pwc paper read PAPER [--json]` — print stored paper Markdown. - `pwc paper list [--page PAGE] [--page-size PAGE_SIZE] [--search SEARCH] [--start-date START_DATE] [--end-date END_DATE] [--task TASK] [--method METHOD] [--conference CONFERENCE] [--framework FRAMEWORK] [--organization ORGANIZATION] [--author AUTHOR] [--all-versions] [--order-by trending|date_published|citation_count] [--order-dir asc|desc] [--include-resources] [--has-official-implementation] [--implementation-coverage] [--json]` — list and filter papers. - `pwc paper recent [--limit LIMIT] [--implementation-coverage] [--json]` — list recent papers. diff --git a/standalone_cli/scripts/check_contract.py b/standalone_cli/scripts/check_contract.py index 0d9ddc9..51c2153 100644 --- a/standalone_cli/scripts/check_contract.py +++ b/standalone_cli/scripts/check_contract.py @@ -16,6 +16,7 @@ COMMANDS = ( "search", "paper info", + "paper evaluations", "paper read", "paper list", "paper recent", From 08eddf8853d17bce70fd208c10b99a29aba152c9 Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Thu, 17 Sep 2026 14:09:39 +0000 Subject: [PATCH 3/3] Paginate benchmark leaderboards --- mcp_server/SKILL.md | 2 +- mcp_server/SPEC.md | 2 +- mcp_server/src/pwc_mcp/models.py | 2 ++ mcp_server/src/pwc_mcp/server.py | 7 +++++- mcp_server/tests/test_server.py | 4 ++++ standalone_cli/src/pwc_cli/cli.py | 36 ++++++++++++++++++++----------- standalone_cli/tests/test_cli.py | 4 ++-- 7 files changed, 40 insertions(+), 17 deletions(-) diff --git a/mcp_server/SKILL.md b/mcp_server/SKILL.md index e2d3147..f94d4e5 100644 --- a/mcp_server/SKILL.md +++ b/mcp_server/SKILL.md @@ -68,7 +68,7 @@ arguments take an exact name, slug, or ID. - `list_organizations({"featured_only": BOOLEAN})` — list research organizations (`pwc organization list`). - `get_framework({"framework": FRAMEWORK})` — inspect one exact research framework (`pwc framework --name`). - `list_frameworks({"domain": DOMAIN, "category": CATEGORY, "platform": PLATFORM})` — list research frameworks (`pwc framework list`). -- `get_benchmark({"benchmark": BENCHMARK, "limit": LIMIT, "is_open": BOOLEAN, "max_parameters": SIZE, "require_metrics": [METRIC], "minimum_metrics": {METRIC: VALUE}, "maximum_metrics": {METRIC: VALUE}, "sort_metric": "METRIC:asc|desc", "pareto": ["METRIC:higher", "METRIC:lower"]})` — inspect one exact benchmark leaderboard with model-size, metric threshold, sort, and Pareto selection (`pwc benchmark --name`). `matched_count` reports rows that satisfied the filters before `limit`. +- `get_benchmark({"benchmark": BENCHMARK, "page": PAGE, "limit": LIMIT, "is_open": BOOLEAN, "max_parameters": SIZE, "require_metrics": [METRIC], "minimum_metrics": {METRIC: VALUE}, "maximum_metrics": {METRIC: VALUE}, "sort_metric": "METRIC:asc|desc", "pareto": ["METRIC:higher", "METRIC:lower"]})` — inspect one paginated benchmark leaderboard with model-size, metric threshold, sort, and Pareto selection (`pwc benchmark --name`). `matched_count` reports rows that satisfied the filters before pagination. - `list_benchmarks({"search": SEARCH, "task": TASK, "include_descendants": BOOLEAN, "minimum_evaluations": COUNT, "is_open": BOOLEAN, "group_by_area": BOOLEAN, "area": AREA, "benchmarks_per_task": COUNT, "order_by": "trending"|"name"|"full_name"|"created_at"|"paper_count", "order_direction": "asc"|"desc", "page": PAGE, "limit": LIMIT})` — list and filter benchmarks (`pwc benchmark list`). With `task`, results are ranked by trend unless `order_by` is set; `group_by_area` or `area` returns top benchmarks under each visible task. All page numbers start at 1. `limit` is between 1 and 25 and defaults to the diff --git a/mcp_server/SPEC.md b/mcp_server/SPEC.md index c080c2c..edd8811 100644 --- a/mcp_server/SPEC.md +++ b/mcp_server/SPEC.md @@ -72,7 +72,7 @@ Paper references accept arXiv IDs, numeric PwC external IDs, arXiv/Hugging Face/Papers With Code URLs, and exact titles. Ambiguous exact titles fail rather than selecting one result. -Paper evaluations paginate. Leaderboards merge equivalent model rows across +Paper evaluations and benchmark leaderboards paginate. Leaderboards merge equivalent model rows across task scopes while retaining scoped ranks, protocol, split, shots, source, openness, and update time. Metric direction is explicit when known. diff --git a/mcp_server/src/pwc_mcp/models.py b/mcp_server/src/pwc_mcp/models.py index 57c1bda..33e137e 100644 --- a/mcp_server/src/pwc_mcp/models.py +++ b/mcp_server/src/pwc_mcp/models.py @@ -222,6 +222,8 @@ class BenchmarkResult(QueryResult): matched_count: int | None = None evaluations: list[Evaluation] metric_directions: dict[str, Literal["higher", "lower", "unknown"]] + page: int + next_page: int | None = None ranking_note: str = ( "Ranks are scoped by task and are not necessarily comparable across " "rank_scopes." diff --git a/mcp_server/src/pwc_mcp/server.py b/mcp_server/src/pwc_mcp/server.py index b71a810..32e2f74 100644 --- a/mcp_server/src/pwc_mcp/server.py +++ b/mcp_server/src/pwc_mcp/server.py @@ -782,6 +782,7 @@ def list_frameworks( @server.tool(annotations=READ_ONLY, structured_output=True) def get_benchmark( benchmark: Entity, + page: Page = 1, limit: Limit = 20, is_open: bool | None = None, max_parameters: ParameterSize | None = None, @@ -795,6 +796,7 @@ def get_benchmark( data = run( "get_benchmark", benchmark=benchmark, + page=page, limit=limit, is_open=is_open, max_parameters=max_parameters, @@ -815,11 +817,14 @@ def get_benchmark( matched_count=int(matched) if matched is not None else None, evaluations=evaluations, metric_directions=metric_directions(evaluations), + page=page, + next_page=_next_page(data), data=None, ) return _tool_result( result, - f"## {result.benchmark.name}\n\n{result.evaluation_count} evaluation rows; returned {len(result.evaluations)} models. Ranks are task-scoped.", + f"## {result.benchmark.name}\n\n{result.evaluation_count} evaluation rows; returned {len(result.evaluations)} models on page {page}. Ranks are task-scoped." + + (f" Next page: {result.next_page}." if result.next_page else ""), ) @server.tool(annotations=READ_ONLY, structured_output=True) diff --git a/mcp_server/tests/test_server.py b/mcp_server/tests/test_server.py index f1fb083..9ea738a 100644 --- a/mcp_server/tests/test_server.py +++ b/mcp_server/tests/test_server.py @@ -159,6 +159,8 @@ "benchmark": {"id": "72", "name": "ImageNet-1k", "slug": "imagenet-1k"}, "count": 1, "matched_count": 1, + "page": 1, + "next_page": 2, "results": [EVALUATION_ROW], }, ("benchmark", "list"): {"next_page": None, "results": [BENCHMARK]}, @@ -584,6 +586,7 @@ async def exercise(): } assert catalog.options(("benchmark",)) == { "name": "imagenet-1k", + "page": 1, "limit": 5, "is_open": True, "max_parameters": "4B", @@ -599,6 +602,7 @@ async def exercise(): assert method.structured_content["method"]["introduced_year"] == 2017 assert benchmarks.structured_content["items"][0]["slug"] == "imagenet-1k" assert benchmark.structured_content["matched_count"] == 1 + assert benchmark.structured_content["next_page"] == 2 evaluation = benchmark.structured_content["evaluations"][0] assert evaluation["model_name"] == "ExampleNet" assert evaluation["metrics"] == {"Accuracy": 90.1} diff --git a/standalone_cli/src/pwc_cli/cli.py b/standalone_cli/src/pwc_cli/cli.py index c27c4a5..45069e2 100644 --- a/standalone_cli/src/pwc_cli/cli.py +++ b/standalone_cli/src/pwc_cli/cli.py @@ -2136,24 +2136,26 @@ def _benchmark_evaluations( is_open: str | None, max_parameters: int | None, scan_all: bool, -) -> tuple[list[dict[str, Any]], int]: + page: int, + page_size: int, +) -> tuple[list[dict[str, Any]], int, int | None]: evaluations = [] - page = 1 + upstream_page = 1 if scan_all else page total = 0 while True: if max_parameters is None: path = f"datasets/{quote(str(benchmark_id), safe='')}/evaluations/" params = { - "page": page, - "page_size": 100, + "page": upstream_page, + "page_size": 100 if scan_all else page_size, "ordering": "best_rank", "is_open": is_open, } else: path = "evaluations/" params = { - "page": page, - "page_size": 100, + "page": upstream_page, + "page_size": 100 if scan_all else page_size, "dataset_id": str(benchmark_id), "ordering": "best_rank", "is_open": is_open, @@ -2177,18 +2179,18 @@ def _benchmark_evaluations( or (max_parameters is None and len(evaluations) >= total) or (max_parameters is not None and next_page is None) ): - return evaluations, total + return evaluations, total, next_page if isinstance(next_page, int) else None if len(evaluations) >= MAX_METRIC_SCAN_ROWS: raise ResponseError( f"metric selection requires scanning {total} evaluation rows; " f"safety limit is {MAX_METRIC_SCAN_ROWS}" ) if max_parameters is not None: - if not isinstance(next_page, int) or next_page <= page: + if not isinstance(next_page, int) or next_page <= upstream_page: raise ResponseError("API returned invalid evaluation pagination") - page = next_page + upstream_page = next_page else: - page += 1 + upstream_page += 1 def _paper_markdown(item: dict[str, Any]) -> str: @@ -2236,12 +2238,14 @@ def benchmark_detail(args: argparse.Namespace, client: Client) -> int: raise ResponseError(f"Benchmark not found: {args.name}{suffix}") scan_all = bool(_metric_requests(args)) - evaluations, total = _benchmark_evaluations( + evaluations, total, next_page = _benchmark_evaluations( client, benchmark["id"], is_open=args.is_open, max_parameters=args.max_parameters, scan_all=scan_all, + page=args.page, + page_size=args.limit, ) merged = _merged_evaluations(evaluations) if args.max_parameters is not None and any( @@ -2254,11 +2258,18 @@ def benchmark_detail(args: argparse.Namespace, client: Client) -> int: "API returned a model outside the requested parameter limit" ) selected = _select_metric_rows(merged, args) - rows = selected[: args.limit] + if scan_all: + start = (args.page - 1) * args.limit + rows = selected[start : start + args.limit] + next_page = args.page + 1 if start + args.limit < len(selected) else None + else: + rows = selected[: args.limit] data = { "benchmark": benchmark, "count": total, "matched_count": len(selected), + "page": args.page, + "next_page": next_page, "results": rows, } if args.json: @@ -2639,6 +2650,7 @@ def build_parser( default=20, help="maximum leaderboard rows, 1-100 (default: 20)", ) + benchmark.add_argument("--page", type=_page, default=1) benchmark.add_argument("--is-open", choices=("true", "false")) benchmark.add_argument( "--max-parameters", diff --git a/standalone_cli/tests/test_cli.py b/standalone_cli/tests/test_cli.py index ea83472..54ad4ca 100644 --- a/standalone_cli/tests/test_cli.py +++ b/standalone_cli/tests/test_cli.py @@ -1724,7 +1724,7 @@ def get(self, path, params): "datasets/42/evaluations/", { "page": 1, - "page_size": 100, + "page_size": 20, "ordering": "best_rank", "is_open": None, }, @@ -1875,7 +1875,7 @@ def test_benchmark_detail_filters_by_inclusive_max_parameters(monkeypatch, capsy "evaluations/", { "page": 1, - "page_size": 100, + "page_size": 20, "dataset_id": "42", "ordering": "best_rank", "is_open": "true",