diff --git a/mcp_server/SKILL.md b/mcp_server/SKILL.md index f94d4e5..d279512 100644 --- a/mcp_server/SKILL.md +++ b/mcp_server/SKILL.md @@ -4,7 +4,7 @@ description: "Papers With Code MCP tools for searching and reading AI/ML papers, compatibility: "Requires an MCP client connected to https://paperswithcode.co/mcp with the Papers With Code tools available." --- -Generated for `pwc-mcp v0.2.1` and stock-client MCP protocol `2025-11-25`. +Generated for `pwc-mcp v0.2.2` and stock-client MCP protocol `2025-11-25`. The tools query the public [Papers With Code](https://paperswithcode.co) catalog anonymously and are read-only. Every tool runs the matching `pwc` CLI research diff --git a/mcp_server/pyproject.toml b/mcp_server/pyproject.toml index 900d508..0ef7902 100644 --- a/mcp_server/pyproject.toml +++ b/mcp_server/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pwc-mcp" -version = "0.2.1" +version = "0.2.2" description = "Read-only Papers With Code MCP server" readme = "README.md" requires-python = ">=3.10" diff --git a/mcp_server/src/pwc_mcp/__init__.py b/mcp_server/src/pwc_mcp/__init__.py index ba5b9ca..0bfe2fb 100644 --- a/mcp_server/src/pwc_mcp/__init__.py +++ b/mcp_server/src/pwc_mcp/__init__.py @@ -1,3 +1,3 @@ """Read-only Papers With Code MCP server.""" -__version__ = "0.2.1" +__version__ = "0.2.2" diff --git a/mcp_server/src/pwc_mcp/catalog.py b/mcp_server/src/pwc_mcp/catalog.py index 02730b3..b39ae68 100644 --- a/mcp_server/src/pwc_mcp/catalog.py +++ b/mcp_server/src/pwc_mcp/catalog.py @@ -205,7 +205,10 @@ def _text(self, path: str, *, ttl: int) -> str: @staticmethod def _rows(payload: dict[str, Any]) -> list[dict[str, Any]]: - values = payload.get("results") or payload.get("items") + values = payload.get("results") + if values is None: + values = payload.get("items") + # An empty list is a valid (empty) result, not a missing list. if not isinstance(values, list): raise ResponseError("API response did not contain a result list") return [item for item in values if isinstance(item, dict)] @@ -255,8 +258,17 @@ def _resolve_paper(self, reference: str) -> str: "www.paperswithcode.co", }: slug_from_url = candidate.casefold() + elif urlparse(candidate).scheme in {"http", "https"}: + # A DOI, publisher, or venue URL is never an exact title; searching + # it would page through empty results before failing anyway. + raise ResponseError( + f"Paper URL not supported: {candidate}; only arXiv, Hugging Face, " + "and Papers With Code URLs resolve" + ) candidate = ARXIV_VERSION.sub("", candidate) if PAPER_ID.fullmatch(candidate): + if candidate.isdigit(): + return self._canonical_paper_id(candidate) return candidate query = candidate.replace("-", " ") if slug_from_url else candidate target = " ".join(candidate.split()).casefold() @@ -293,6 +305,30 @@ def _resolve_paper(self, reference: str) -> str: raise ResponseError(f"Paper title is ambiguous: {candidate}; {choices}") raise ResponseError(f"Paper title not found: {candidate}") + def _canonical_paper_id(self, catalog_id: str) -> str: + """Prefer the arXiv ID for a numeric catalog ID so every route accepts it. + + The Markdown read route stores arXiv papers under their arXiv ID and + only external papers under the numeric ID; list tools hand out numeric + IDs for both, so a bare numeric reference cannot tell them apart. + """ + path = f"papers/{quote(catalog_id, safe='')}" + record = self._json(path, ttl=cache_ttl(path)) + arxiv_id = record.get("arxiv_id") + if isinstance(arxiv_id, str) and arxiv_id.strip(): + return ARXIV_VERSION.sub("", arxiv_id.strip()) + return catalog_id + + def _paper_exists(self, reference: str) -> bool: + path = f"papers/{quote(reference, safe='.')}" + try: + self._json(path, ttl=cache_ttl(path)) + except HTTPStatusError as error: + if error.status == 404: + return False + raise + return True + def resolve_paper(self, paper: str) -> str: return self._resolve_paper(paper) @@ -326,6 +362,8 @@ def read_paper_chunk( raise PaperVersionMismatchError( "Paper Markdown changed; restart reading from the beginning" ) from error + if error.status == 404 and not self._paper_exists(reference): + raise ResponseError(f"Paper not found: {reference}") from error raise returned_version = response.headers.get("x-pwc-content-version", "") diff --git a/mcp_server/src/pwc_mcp/server.py b/mcp_server/src/pwc_mcp/server.py index 32e2f74..546911f 100644 --- a/mcp_server/src/pwc_mcp/server.py +++ b/mcp_server/src/pwc_mcp/server.py @@ -9,6 +9,7 @@ from __future__ import annotations +import logging import os import time from collections.abc import Mapping @@ -57,6 +58,7 @@ paper_summary, ) +logger = logging.getLogger(__name__) READ_ONLY = ToolAnnotations( read_only_hint=True, destructive_hint=False, @@ -77,24 +79,46 @@ "Area not found", "Paper title not found", "Paper title is ambiguous", + "Paper not found", + "Paper URL not supported", "Paper reference cannot be empty", "Too many results to resolve paper title", "Papers API did not confirm", ) GENERIC_ERROR = "the Papers With Code catalog request failed" +# Upstream validation failures the caller can correct; the detail is the +# public API's own message, bounded and stripped of control characters. +INVALID_ARGUMENT_STATUSES = frozenset({400, 422}) +MAX_UPSTREAM_DETAIL_CHARS = 200 + + +def _upstream_detail(error: HTTPStatusError) -> str: + detail = "".join( + ch if ch.isprintable() else " " for ch in str(error.detail or "") + ).strip() + return detail[:MAX_UPSTREAM_DETAIL_CHARS] or "the catalog rejected the request" def catalog_error_message(error: Exception) -> str: """Return the CLI's own lookup message when it is actionable, else a generic one.""" - if isinstance(error, HTTPStatusError) and error.status == 404: - return "not_found: the requested catalog record does not exist" - if isinstance(error, TransportError) and "timeout" in str(error).casefold(): - return "upstream_timeout: the Papers With Code catalog timed out" + if isinstance(error, HTTPStatusError): + if error.status == 404: + return "not_found: the requested catalog record does not exist" + if error.status in INVALID_ARGUMENT_STATUSES: + return f"invalid_argument: {_upstream_detail(error)}" + if error.status == 429: + return "rate_limited: the Papers With Code catalog is rate limiting; retry later" + if isinstance(error, TransportError): + message = str(error).casefold() + if "timeout" in message or "timed out" in message: + return "upstream_timeout: the Papers With Code catalog timed out" if isinstance(error, ResponseError) and not isinstance(error, TransportError): message = str(error) if message.startswith(CLIENT_FACING_ERRORS): code = "ambiguous" if "ambiguous" in message.casefold() else "not_found" return f"{code}: {message}" + # Only the exception class is logged: never the reference, query, or body. + logger.warning("pwc-mcp generic catalog error type=%s", type(error).__name__) return f"upstream_error: {GENERIC_ERROR}" @@ -493,7 +517,7 @@ def read_paper(paper: Reference, cursor: str | None = None) -> PaperReadResult: except (ResponseError, TransportError) as error: raise ToolError(catalog_error_message(error)) from error if chunk.paper != canonical or chunk.source != source: - raise ToolError("the Papers With Code catalog request failed") + raise ToolError("paper changed; restart reading from the beginning") next_cursor = None if chunk.next_offset is not None: next_cursor = codec.encode( @@ -792,7 +816,7 @@ def get_benchmark( sort_metric: SortMetric | None = None, pareto: ParetoObjectives | None = None, ) -> BenchmarkResult: - """Get one exact benchmark and its leaderboard (`pwc benchmark --name`). Use max_parameters (for example "4B") to keep models at or below a size, sort_metric to rank by a metric, and minimum_metrics, maximum_metrics, require_metrics, or pareto to select rows; matched_count reports how many rows passed before limit.""" + """Get one exact benchmark and its leaderboard (`pwc benchmark --name`). Use max_parameters (for example "4B") to keep models at or below a size, sort_metric to rank by a metric, and minimum_metrics, maximum_metrics, require_metrics, or pareto to select rows; matched_count reports how many rows passed before limit. Metric names are matched case-insensitively and through common aliases (AP/mAP, top1/Accuracy, AUROC/AUC); an unknown metric error lists the leaderboard's actual metric names.""" data = run( "get_benchmark", benchmark=benchmark, @@ -843,8 +867,24 @@ def list_benchmarks( page: Page = 1, limit: Limit | None = None, ) -> BenchmarkPage: - """List benchmarks for a task ranked by trend, filter them, or group them by area and task (`pwc benchmark list`). Follow with get_benchmark on the most relevant leaderboard.""" + """List benchmarks for a task ranked by trend, filter them, or group them by area and task (`pwc benchmark list`). order_by=trending needs task (without it the list is ordered by name); area groups a whole area and is dropped when combined with task, search, or ordering filters. Follow with get_benchmark on the most relevant leaderboard.""" + notes: list[str] = [] + if order_by == "trending" and not task: + order_by = None + notes.append("order_by=trending needs task; ordered by name instead") + filtered = bool( + task + or search + or include_descendants + or is_open is not None + or order_by is not None + or order_direction != "asc" + ) + if area is not None and filtered: + area = None + notes.append("area cannot be combined with filters; returned the filtered list") grouped = group_by_area or area is not None + # Grouped listings are not paginated; the CLI rejects page/limit there. data = run( "list_benchmarks", search=search, @@ -857,19 +897,25 @@ def list_benchmarks( benchmarks_per_task=benchmarks_per_task, order_by=order_by, order_direction=order_direction, - page=page, - limit=limit if limit is not None or grouped else MAX_ROWS, + page=1 if grouped else page, + limit=None if grouped else (limit if limit is not None else MAX_ROWS), ) if not isinstance(data, dict): raise TypeError("benchmark listing did not contain a result document") rows = _dicts(data.get("results")) if grouped: rows = _grouped_benchmarks(rows) - return BenchmarkPage( + result = BenchmarkPage( items=[benchmark_summary(item) for item in rows], next_page=_next_page(data), data=data, ) + summary = f"Found {len(result.items)} benchmarks." + if result.next_page: + summary += f" Next page: {result.next_page}." + for note in notes: + summary += f" Note: {note}." + return _tool_result(result, summary) @server.prompt(name="find_papers", title="Find papers") def find_papers_prompt(topic: str) -> str: diff --git a/mcp_server/tests/test_app.py b/mcp_server/tests/test_app.py index a2a06a9..3732c5d 100644 --- a/mcp_server/tests/test_app.py +++ b/mcp_server/tests/test_app.py @@ -43,7 +43,7 @@ def test_health_and_browser_origin_policy_are_explicit(): assert health.json() == { "status": "ok", "service": "pwc-mcp", - "version": "0.2.1", + "version": "0.2.2", "protocol": "2025-11-25", } assert rejected.status_code == 403 diff --git a/mcp_server/tests/test_catalog.py b/mcp_server/tests/test_catalog.py index 4d90414..1069649 100644 --- a/mcp_server/tests/test_catalog.py +++ b/mcp_server/tests/test_catalog.py @@ -280,3 +280,83 @@ def test_catalog_query_reports_cli_usage_errors_without_upstream_calls(): with pytest.raises(UsageError, match="not a read-only"): catalog.query(("skills", "add"), {}) assert transport.calls == [] + + +def test_catalog_canonicalises_numeric_ids_of_arxiv_papers(): + transport = StubTransport( + { + "papers/86122": {"id": "86122", "arxiv_id": "2505.18132v1"}, + "papers/2505.18132": {"id": "86122", "title": "BiggerGait"}, + "papers/4242": {"id": "4242", "arxiv_id": None, "source": "external"}, + } + ) + catalog = CatalogClient(transport=transport) + + # List tools hand out numeric IDs for arXiv papers too; the Markdown route + # only knows arXiv papers by arXiv ID, so numeric references canonicalise. + assert catalog.resolve_paper("86122") == "2505.18132" + assert catalog.query(("paper", "info"), {"paper": "86122"}) == { + "id": "86122", + "title": "BiggerGait", + } + # External papers have no arXiv ID and keep their numeric reference. + assert catalog.resolve_paper("4242") == "4242" + assert transport.calls[0] == ("papers/86122", {}) + + +def test_catalog_rejects_unsupported_urls_without_searching(): + transport = StubTransport({}) + catalog = CatalogClient(transport=transport) + + for url in ( + "https://doi.org/10.1007/s10479-024-06277-x", + "https://aclanthology.org/2026.acl-long.200/", + ): + with pytest.raises(ResponseError, match="Paper URL not supported"): + catalog.resolve_paper(url) + assert transport.calls == [] + + +def test_catalog_distinguishes_missing_papers_from_missing_markdown(): + from pwc_cli.transport import HTTPStatusError + + class MissingTransport: + def __init__(self, paper_exists): + self.paper_exists = paper_exists + + def get(self, path, params=None): + if path.endswith("/read"): + raise HTTPStatusError(404, "Paper Markdown was not found") + if self.paper_exists: + return Response(b'{"id": "1"}', {}) + raise HTTPStatusError(404, "No paper found") + + with pytest.raises(ResponseError, match="Paper not found: 2308.10195"): + CatalogClient(transport=MissingTransport(False)).read_paper_chunk( + "2308.10195", resolved=True + ) + with pytest.raises(HTTPStatusError) as missing_markdown: + CatalogClient(transport=MissingTransport(True)).read_paper_chunk( + "2505.18132", resolved=True + ) + assert missing_markdown.value.status == 404 + + +def test_catalog_reports_titles_missing_from_an_empty_search_as_not_found(): + # The public API answers an unknown title with an empty result list; that + # must surface as "title not found", not as a malformed response. + transport = StubTransport( + { + "papers/search": { + "next_page": None, + "previous_page": None, + "results": [], + "applied_filters": {}, + } + } + ) + catalog = CatalogClient(transport=transport) + + with pytest.raises(ResponseError, match="Paper title not found: Dropout"): + catalog.resolve_paper("Dropout: A Simple Way to Prevent Overfitting") + assert len(transport.calls) == 1 diff --git a/mcp_server/tests/test_server.py b/mcp_server/tests/test_server.py index 9ea738a..6df8b9e 100644 --- a/mcp_server/tests/test_server.py +++ b/mcp_server/tests/test_server.py @@ -733,3 +733,101 @@ async def exercise(): assert markdown.contents[0].text == "abcdefgh" assert '"slug":"image-classification"' in task.contents[0].text assert '"slug":"imagenet-1k"' in benchmark.contents[0].text + + +def test_upstream_validation_rate_limit_and_timeout_errors_are_actionable(caplog): + from pwc_cli.transport import TransportError + from pwc_mcp.server import catalog_error_message + + class ErrorCatalog(StubCatalog): + def __init__(self, error): + super().__init__() + self.error = error + + def query(self, command, options): + raise self.error + + cases = { + HTTPStatusError(422, '{"detail":"page_size must be <= 100"}\n\x00'): ( + 'invalid_argument: {"detail":"page_size must be <= 100"}' + ), + HTTPStatusError(429, "slow down"): ( + "rate_limited: the Papers With Code catalog is rate limiting; retry later" + ), + TransportError("API request timed out"): ( + "upstream_timeout: the Papers With Code catalog timed out" + ), + ResponseError("Paper not found: 2308.10195"): ( + "not_found: Paper not found: 2308.10195" + ), + ResponseError("Paper URL not supported: https://doi.org/x; only arXiv"): ( + "not_found: Paper URL not supported: https://doi.org/x; only arXiv" + ), + } + for error, expected in cases.items(): + (result,) = _call(ErrorCatalog(error), [("get_task", {"task": "x"})]) + assert result.is_error is True + assert result.content[0].text == f"Error executing tool get_task: {expected}" + + with caplog.at_level(logging.WARNING): + assert catalog_error_message(ResponseError("API returned invalid JSON")) == ( + "upstream_error: the Papers With Code catalog request failed" + ) + assert "pwc-mcp generic catalog error type=ResponseError" in caplog.text + assert "invalid JSON" not in caplog.text + + +def test_list_benchmarks_falls_back_instead_of_rejecting_argument_combinations(): + catalog = StubCatalog() + trending_search, area_with_search, area_only = _call( + catalog, + [ + ( + "list_benchmarks", + {"search": "COCO", "order_by": "trending", "order_direction": "desc"}, + ), + ("list_benchmarks", {"area": "Vision", "search": "HDR", "limit": 10}), + ("list_benchmarks", {"area": "Vision", "limit": 10}), + ], + ) + + # order_by=trending needs a task; the CLI would raise a usage error. + assert catalog.queries[0][1]["order_by"] is None + assert catalog.queries[0][1]["search"] == "COCO" + assert trending_search.is_error is False + assert "Note: order_by=trending needs task; ordered by name instead." in ( + trending_search.content[0].text + ) + assert trending_search.structured_content["items"][0]["slug"] == "imagenet-1k" + + # area cannot be combined with flat filters; the filters win. + assert catalog.queries[1][1]["area"] is None + assert catalog.queries[1][1]["search"] == "HDR" + assert catalog.queries[1][1]["page_size"] == 10 + assert "area cannot be combined with filters" in area_with_search.content[0].text + + # A bare limit does not conflict with grouping: the grouped listing ignores it. + assert catalog.queries[2][1]["area"] == "Vision" + assert catalog.queries[2][1]["page_size"] is None + assert "Note:" not in area_only.content[0].text + assert area_only.content[0].text == "Found 1 benchmarks." + + +def test_read_paper_names_an_identity_mismatch_instead_of_a_generic_failure(): + class DriftingCatalog(StubCatalog): + def read_paper_chunk(self, paper, **kwargs): + chunk = super().read_paper_chunk(paper, **kwargs) + return PaperMarkdownChunk( + paper="9999.99999", + source=chunk.source, + markdown=chunk.markdown, + content_version=chunk.content_version, + next_offset=chunk.next_offset, + ) + + (result,) = _call(DriftingCatalog(), [("read_paper", {"paper": "1706.03762"})]) + + assert result.is_error is True + assert result.content[0].text == ( + "Error executing tool read_paper: paper changed; restart reading from the beginning" + ) diff --git a/mcp_server/uv.lock b/mcp_server/uv.lock index 129c972..dc3a56b 100644 --- a/mcp_server/uv.lock +++ b/mcp_server/uv.lock @@ -438,12 +438,12 @@ wheels = [ [[package]] name = "pwc-cli" -version = "0.4.1" +version = "0.4.2" source = { editable = "../standalone_cli" } [[package]] name = "pwc-mcp" -version = "0.2.1" +version = "0.2.2" source = { editable = "." } dependencies = [ { name = "mcp" }, diff --git a/standalone_cli/SKILL.md b/standalone_cli/SKILL.md index 187307a..d532946 100644 --- a/standalone_cli/SKILL.md +++ b/standalone_cli/SKILL.md @@ -3,7 +3,7 @@ name: pwc-cli description: "Papers With Code CLI (`pwc`) for searching and reading AI/ML papers, discovering recent and trending research, finding related work and paper lineage, browsing tasks, methods, conferences, organizations, frameworks, and benchmark leaderboards, and submitting authenticated paper edits through the public Papers With Code catalog. Use whenever the user asks to find papers, survey literature, compare research, inspect an arXiv paper, explore AI/ML taxonomy or conferences, discover benchmarks or state-of-the-art models, or mentions Papers With Code, `pwc`, or `pwc-cli`." --- -Generated with `pwc v0.4.1`. Run `pwc skills add --force` to regenerate. +Generated with `pwc v0.4.2`. Run `pwc skills add --force` to regenerate. Research commands query the public [Papers With Code](https://paperswithcode.co) catalog anonymously. Paper editing requires explicit browser authorization through `pwc auth login --paper PAPER`. @@ -43,7 +43,7 @@ case-insensitive but exact; ambiguous titles fail with their matching IDs. - `pwc search QUERY [--limit LIMIT] [--page PAGE] [--mode hybrid|keyword|semantic] [--start-date START_DATE] [--end-date END_DATE] [--has-official-implementation] [--implementation-coverage] [--json]` — search papers. - `pwc paper info PAPER [--include-resources] [--include-evals] [--json]` — show paper metadata including abstract. -- `pwc paper evaluations PAPER [--page PAGE] [--page-size N] [--json]` — page through one paper's benchmark evaluations. +- `pwc paper evaluations PAPER [--page PAGE] [--page-size PAGE_SIZE] [--json]` — list one paper's benchmark evaluations. - `pwc paper read PAPER [--json]` — print stored paper Markdown. - `pwc paper list [--page PAGE] [--page-size PAGE_SIZE] [--search SEARCH] [--start-date START_DATE] [--end-date END_DATE] [--task TASK] [--method METHOD] [--conference CONFERENCE] [--framework FRAMEWORK] [--organization ORGANIZATION] [--author AUTHOR] [--all-versions] [--order-by trending|date_published|citation_count] [--order-dir asc|desc] [--include-resources] [--has-official-implementation] [--implementation-coverage] [--json]` — list and filter papers. - `pwc paper recent [--limit LIMIT] [--implementation-coverage] [--json]` — list recent papers. @@ -54,16 +54,16 @@ case-insensitive but exact; ambiguous titles fail with their matching IDs. - `pwc paper edit preview PAPER [--file FILE]`. - `pwc paper edit submit PAPER [--file FILE]`. - `pwc task [--name NAME] [--json]` — inspect or list research tasks. -- `pwc task list [--page PAGE] [--page-size PAGE_SIZE] [--group-by-area] [--flat] [--area AREA] [--level LEVEL] [--visible-only] [--order-by name|created_at|level|paper_count] [--order-dir asc|desc] [--json]` — list and filter research tasks. +- `pwc task list [--page PAGE] [--page-size PAGE_SIZE] [--search SEARCH] [--group-by-area] [--flat] [--area AREA] [--level LEVEL] [--visible-only] [--order-by name|created_at|level|paper_count] [--order-dir asc|desc] [--json]` — list and filter research tasks. - `pwc method [--name NAME] [--json]` — inspect or list research methods. -- `pwc method list [--page PAGE] [--page-size PAGE_SIZE] [--area AREA] [--introduced-year INTRODUCED_YEAR] [--order-by name|full_name|introduced_year|created_at|paper_count] [--order-dir asc|desc] [--json]` — list and filter research methods. +- `pwc method list [--page PAGE] [--page-size PAGE_SIZE] [--search SEARCH] [--area AREA] [--introduced-year INTRODUCED_YEAR] [--order-by name|full_name|introduced_year|created_at|paper_count] [--order-dir asc|desc] [--json]` — list and filter research methods. - `pwc conference [--name NAME] [--json]` — inspect or list conferences. - `pwc conference list [--year YEAR] [--json]` — list conferences with imported papers. - `pwc organization [--name NAME] [--json]` — inspect or list research organizations. - `pwc organization list [--featured-only] [--json]` — list research organizations. - `pwc framework [--name NAME] [--json]` — inspect or list research frameworks. - `pwc framework list [--domain DOMAIN] [--category CATEGORY] [--platform PLATFORM] [--json]` — list research frameworks. -- `pwc benchmark [--name NAME] [--limit LIMIT] [--is-open true|false] [--max-parameters SIZE] [--require-metrics METRIC[,METRIC]] [--min METRIC=VALUE] [--max METRIC=VALUE] [--sort METRIC[:ASC|DESC]] [--pareto METRIC:HIGHER,METRIC:LOWER] [--json]` — inspect benchmarks. +- `pwc benchmark [--name NAME] [--limit LIMIT] [--page PAGE] [--is-open true|false] [--max-parameters SIZE] [--require-metrics METRIC[,METRIC]] [--min METRIC=VALUE] [--max METRIC=VALUE] [--sort METRIC[:ASC|DESC]] [--pareto METRIC:HIGHER,METRIC:LOWER] [--json]` — inspect benchmarks. - `pwc benchmark list [--page PAGE] [--page-size PAGE_SIZE] [--search SEARCH] [--task TASK] [--group-by-area] [--flat] [--area AREA] [--benchmarks-per-task BENCHMARKS_PER_TASK] [--include-descendants] [--min-eval-count MIN_EVAL_COUNT] [--is-open true|false] [--order-by trending|name|full_name|created_at|paper_count] [--order-dir asc|desc] [--json]` — list and filter benchmarks. - `pwc skills add [--global] [--claude] [--dest DEST] [--force]` — install the version-matched pwc CLI Skill. - `pwc version` — show CLI and API contract versions. diff --git a/standalone_cli/pyproject.toml b/standalone_cli/pyproject.toml index 281940b..c8db285 100644 --- a/standalone_cli/pyproject.toml +++ b/standalone_cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pwc-cli" -version = "0.4.1" +version = "0.4.2" description = "Papers With Code research and paper-editing CLI" readme = "README.md" requires-python = ">=3.10" diff --git a/standalone_cli/src/pwc_cli/__init__.py b/standalone_cli/src/pwc_cli/__init__.py index 6bc0ac0..ab2fd92 100644 --- a/standalone_cli/src/pwc_cli/__init__.py +++ b/standalone_cli/src/pwc_cli/__init__.py @@ -1,4 +1,4 @@ """Standalone Papers With Code client.""" -__version__ = "0.4.1" +__version__ = "0.4.2" API_CONTRACT_VERSION = "v1" diff --git a/standalone_cli/src/pwc_cli/cli.py b/standalone_cli/src/pwc_cli/cli.py index 45069e2..e2edc73 100644 --- a/standalone_cli/src/pwc_cli/cli.py +++ b/standalone_cli/src/pwc_cli/cli.py @@ -183,6 +183,33 @@ def _rows(payload: Any) -> tuple[list[dict[str, Any]], int | None]: ) if total is not None else None +def _name_tokens(value: object) -> frozenset[str]: + return frozenset(re.findall(r"[a-z0-9]+", str(value or "").casefold())) + + +def _closest_names(reference: str, items: list[dict[str, Any]], limit: int = 3) -> str: + """Name the candidates sharing the most words with the reference. + + The search API ranks by its own relevance, which for short taxonomy names + can put an unrelated entry first; word overlap with the reference keeps the + hint useful ("person re-identification" suggests re-identification tasks, + not language identification). + """ + target = _name_tokens(reference) + ranked = sorted( + ( + ( + -len(target & _name_tokens(item.get("name") or item.get("slug"))), + index, + str(item.get("name") or item.get("slug")), + ) + for index, item in enumerate(items) + if item.get("name") or item.get("slug") + ), + ) + return ", ".join(name for _score, _index, name in ranked[:limit]) + + def _exact_entity_match( reference: str, items: list[dict[str, Any]], @@ -195,11 +222,7 @@ def _exact_entity_match( for item in items: if str(item.get(field) or "").strip().casefold() == target: return item - suggestions = ", ".join( - str(item.get("name") or item.get("slug")) - for item in items[:3] - if item.get("name") or item.get("slug") - ) + suggestions = _closest_names(reference, items) suffix = f"; closest results: {suggestions}" if suggestions else "" raise ResponseError(f"{label} not found: {reference}{suffix}") @@ -1016,9 +1039,7 @@ def task_detail(args: argparse.Namespace, client: Client) -> int: candidates, _total = _rows(search_payload) summary = _task_match(args.name, candidates) if summary is None: - suggestions = ", ".join( - str(item.get("name")) for item in candidates[:3] if item.get("name") - ) + suggestions = _closest_names(args.name, candidates) suffix = f"; closest results: {suggestions}" if suggestions else "" raise ResponseError(f"Task not found: {args.name}{suffix}") @@ -2048,6 +2069,67 @@ def _metric_requests(args: argparse.Namespace) -> tuple[str, ...]: return tuple(dict.fromkeys(name.casefold() for name in requested)) +# Normalised (lower-case alphanumeric) spellings agents use for the same metric. +METRIC_ALIASES: dict[str, tuple[str, ...]] = { + "ap": ("map", "boxap", "ap5095"), + "map": ("ap", "boxap"), + "maskap": ("map", "ap"), + "boxap": ("map", "ap"), + "top1": ("accuracy", "top1accuracy", "acc"), + "top1accuracy": ("accuracy", "top1", "acc"), + "acc": ("accuracy", "top1accuracy", "top1"), + "accuracy": ("top1accuracy", "top1", "acc"), + "top5": ("top5accuracy",), + "top5accuracy": ("top5",), + "auroc": ("auc", "rocauc"), + "auc": ("auroc", "rocauc"), + "rocauc": ("auc", "auroc"), + "f1": ("f1score",), + "f1score": ("f1",), + "miou": ("meaniou", "iou"), + "meaniou": ("miou", "iou"), + "bleu": ("bleuscore",), + "bleuscore": ("bleu",), + "totalscore": ("total", "overall", "overallscore", "score"), + "score": ("totalscore", "overallscore"), +} + + +def _normalize_metric_name(value: str) -> str: + return re.sub(r"[^a-z0-9]+", "", value.casefold()) + + +def _resolve_metric_name(requested: str, available: dict[str, str]) -> str | None: + """Map a requested metric onto the leaderboard's own name, or None.""" + key = requested.casefold() + if key in available: + return available[key] + normalized = _normalize_metric_name(requested) + by_normalized: dict[str, str] = {} + for name in available.values(): + by_normalized.setdefault(_normalize_metric_name(name), name) + if normalized in by_normalized: + return by_normalized[normalized] + for alias in METRIC_ALIASES.get(normalized, ()): + if alias in by_normalized: + return by_normalized[alias] + return None + + +def _apply_metric_names(args: argparse.Namespace, mapping: dict[str, str]) -> None: + def rename(name: str) -> str: + return mapping.get(name.casefold(), name) + + args.require_metrics = [rename(name) for name in args.require_metrics] + args.minimum_metrics = [(rename(n), t) for n, t in args.minimum_metrics] + args.maximum_metrics = [(rename(n), t) for n, t in args.maximum_metrics] + if args.sort_metric: + name, direction = args.sort_metric + args.sort_metric = (rename(name), direction) + if args.pareto: + args.pareto = [(rename(n), d) for n, d in args.pareto] + + def _select_metric_rows( items: list[dict[str, Any]], args: argparse.Namespace ) -> list[dict[str, Any]]: @@ -2055,6 +2137,15 @@ def _select_metric_rows( if not requested: return items available = _available_metrics(items) + renamed = { + name: resolved + for name in requested + if name not in available + and (resolved := _resolve_metric_name(name, available)) is not None + } + if renamed: + _apply_metric_names(args, renamed) + requested = _metric_requests(args) unknown = [name for name in requested if name not in available] if unknown: choices = ", ".join(sorted(available.values(), key=str.casefold)) or "none" @@ -2231,9 +2322,7 @@ def benchmark_detail(args: argparse.Namespace, client: Client) -> int: candidates, _total = _rows(search_payload) benchmark = _benchmark_match(args.name, candidates) if benchmark is None: - suggestions = ", ".join( - str(item.get("name")) for item in candidates[:3] if item.get("name") - ) + suggestions = _closest_names(args.name, candidates) suffix = f"; closest results: {suggestions}" if suggestions else "" raise ResponseError(f"Benchmark not found: {args.name}{suffix}") diff --git a/standalone_cli/src/pwc_cli/transport.py b/standalone_cli/src/pwc_cli/transport.py index 14eedfd..9b6a172 100644 --- a/standalone_cli/src/pwc_cli/transport.py +++ b/standalone_cli/src/pwc_cli/transport.py @@ -93,3 +93,8 @@ def get( raise HTTPStatusError(error.code, detail) from error except urllib.error.URLError as error: raise TransportError(f"API request failed: {error.reason}") from error + except TimeoutError as error: + # Read timeouts surface as a bare TimeoutError, not a URLError. + raise TransportError("API request timed out") from error + except OSError as error: + raise TransportError(f"API request failed: {error}") from error diff --git a/standalone_cli/tests/test_cli.py b/standalone_cli/tests/test_cli.py index 54ad4ca..8a168f9 100644 --- a/standalone_cli/tests/test_cli.py +++ b/standalone_cli/tests/test_cli.py @@ -1,19 +1,28 @@ from __future__ import annotations +import argparse import importlib.util import io import json import sys import urllib.error +import urllib.request from contextlib import redirect_stderr, redirect_stdout from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) -from pwc_cli.cli import build_parser, main # noqa: E402 +from pwc_cli.cli import UsageError, build_parser, main # noqa: E402 from pwc_cli.skills import build_skill_md # noqa: E402 -from pwc_cli.transport import Client, HTTPStatusError, Response # noqa: E402 +from pwc_cli.transport import ( # noqa: E402 + Client, + HTTPStatusError, + Response, + ResponseError, +) INSTALLER_SPEC = importlib.util.spec_from_file_location( "pwc_cli_installer", ROOT / "install.py" @@ -85,7 +94,7 @@ def test_generated_skill_matches_installed_cli_version_and_commands(): skill = build_skill_md() assert "name: pwc-cli" in skill - assert "Generated with `pwc v0.4.1`" in skill + assert "Generated with `pwc v0.4.2`" in skill assert "`pwc search QUERY" in skill assert "--include-evals" in skill assert "[--organization ORGANIZATION]" in skill @@ -2297,7 +2306,7 @@ def test_top_level_version_is_offline_and_stable(): build_parser().parse_args(["--version"]) except SystemExit as error: assert error.code == 0 - assert output.getvalue() == "pwc 0.4.1\tapi v1\n" + assert output.getvalue() == "pwc 0.4.2\tapi v1\n" def test_search_default_output_is_compact_deterministic_tsv(monkeypatch): @@ -2949,3 +2958,79 @@ def fake_urlopen(request, *, timeout): assert requests[0].full_url.endswith( "/api/v1/papers/?author=Kaiming+He&author=%40yilundu" ) + + +def test_transport_reports_read_timeouts_and_socket_errors_as_transport_errors( + monkeypatch, +): + from pwc_cli.transport import TransportError + + failures = iter([TimeoutError("The read operation timed out"), ConnectionResetError(104, "reset")]) + + def fake_urlopen(_request, *, timeout): + raise next(failures) + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + client = Client("https://example.test/api/v1") + + with pytest.raises(TransportError, match="timed out"): + client.get("health") + with pytest.raises(TransportError, match="API request failed"): + client.get("health") + + +def test_closest_results_rank_by_shared_words_not_search_order(): + from pwc_cli.cli import _closest_names, _exact_entity_match + + candidates = [ + {"name": "Spoken Language Identification"}, + {"name": "Face Recognition"}, + {"name": "Person Re-Identification (Video)"}, + {"name": "Unsupervised Person Re-Identification"}, + {"name": "Image Classification"}, + ] + + assert _closest_names("person re-identification", candidates) == ( + "Person Re-Identification (Video), " + "Unsupervised Person Re-Identification, " + "Spoken Language Identification" + ) + with pytest.raises(ResponseError, match="closest results: Person Re-Identification"): + _exact_entity_match("person re-identification", candidates, label="Task") + + +def test_metric_requests_resolve_through_case_and_common_aliases(): + from pwc_cli.cli import _resolve_metric_name, _select_metric_rows + + available = {"map": "mAP", "fps": "FPS", "top 1 accuracy": "Top 1 Accuracy"} + assert _resolve_metric_name("AP", available) == "mAP" + assert _resolve_metric_name("box ap", available) == "mAP" + assert _resolve_metric_name("top1", available) == "Top 1 Accuracy" + assert _resolve_metric_name("Top-1 Accuracy", available) == "Top 1 Accuracy" + assert _resolve_metric_name("fps", available) == "FPS" + assert _resolve_metric_name("latency", available) is None + + rows = [ + {"model_name": "Fast", "metrics": {"mAP": 58, "FPS": 100}}, + {"model_name": "Slow", "metrics": {"mAP": 62, "FPS": 20}}, + ] + args = argparse.Namespace( + require_metrics=["ap"], + minimum_metrics=[("AP", 60)], + maximum_metrics=[], + sort_metric=("ap", "desc"), + pareto=[], + ) + assert [row["model_name"] for row in _select_metric_rows(rows, args)] == ["Slow"] + assert args.sort_metric == ("mAP", "desc") + assert args.minimum_metrics == [("mAP", 60)] + + unknown = argparse.Namespace( + require_metrics=[], + minimum_metrics=[], + maximum_metrics=[], + sort_metric=("latency", "desc"), + pareto=[], + ) + with pytest.raises(UsageError, match="unknown metric\\(s\\): latency; available metrics: FPS, mAP"): + _select_metric_rows(rows, unknown) diff --git a/standalone_cli/tests/test_queries.py b/standalone_cli/tests/test_queries.py index 6b8cef8..a0fd9ce 100644 --- a/standalone_cli/tests/test_queries.py +++ b/standalone_cli/tests/test_queries.py @@ -211,3 +211,15 @@ def test_query_refuses_commands_that_are_not_read_only(): for command in (("skills", "add"), ("version",), ("paper", "edit", "export")): with pytest.raises(UsageError, match="not a read-only"): queries.query(command, {}, StubClient({})) + + +def test_query_resolves_metric_aliases_against_the_leaderboard(): + # Agents ask for "AP" on COCO leaderboards that report "mAP". + data = queries.query( + ("benchmark",), + {"name": "COCO", "sort_metric": "AP:asc"}, + StubClient(BENCHMARK_ROUTES), + ) + + assert data["matched_count"] == 2 + assert [row["model_name"] for row in data["results"]] == ["Small", "Smaller"]