From 774325f0921780fa1853ef154289084393edd32e Mon Sep 17 00:00:00 2001 From: Niels Rogge Date: Sat, 19 Sep 2026 13:02:55 +0000 Subject: [PATCH] Resolve titles, metrics, and entity names the way chat agents spell them Production chat traces (36 h, 2026-09-18/19) show the remaining MCP tool failures that are not catalog gaps: - `get_paper_info("Attention Is All You Need")` failed with "Too many results to resolve paper title safely": the exact title was the first keyword hit, but resolution paged through ten pages of common-word matches and gave up. Titles are now looked up as one quoted phrase first (websearch_to_tsquery phrase semantics, one small page), falling back to the plain query, and an exact match found before the page budget runs out is used. - `get_benchmark` rejected `sort_metric` spellings such as "Overall" for "GenEval Score", "D4RL Normalized Score" for "Normalized Score", "Pass@1" for "Accuracy", "Pass Rate" for "Success Rate", and "image-level AUROC" for "Detection AUROC" (12 rejections after the alias table shipped). Metric resolution now also accepts the one leaderboard metric that contains the request or one of its aliases, and the alias table covers Pass@1, pass and success rate, overall, F-score, image/pixel-level AUROC, WER, CER, and EM. - `get_method("mixture of experts")` reported "not found" although the only candidate was "Mixture-of-Experts (MoE)". Method, task, conference, organization, and framework lookups now fall back to a unique match on letters and digits alone, ignoring a trailing acronym but not a qualifier such as "(Video)". - `read_paper` rejected a valid continuation cursor when the agent continued with another reference to the same paper (the numeric catalog ID after starting from the arXiv ID). The cursor is honoured when the new reference resolves to the cursor's paper, and the invalid-cursor error now says what to pass instead. Bumps pwc-cli to 0.4.3 and pwc-mcp to 0.2.4. Co-Authored-By: Claude Fable 5.1 --- mcp_server/README.md | 3 +- mcp_server/SKILL.md | 2 +- mcp_server/SPEC.md | 6 +- mcp_server/pyproject.toml | 2 +- mcp_server/src/pwc_mcp/__init__.py | 2 +- mcp_server/src/pwc_mcp/catalog.py | 41 ++++++--- mcp_server/src/pwc_mcp/cursors.py | 20 ++++- mcp_server/src/pwc_mcp/server.py | 23 ++++- mcp_server/tests/test_app.py | 2 +- mcp_server/tests/test_catalog.py | 66 +++++++++++++- mcp_server/tests/test_cursors.py | 16 +++- mcp_server/tests/test_server.py | 40 ++++++++- mcp_server/uv.lock | 4 +- standalone_cli/SKILL.md | 2 +- standalone_cli/pyproject.toml | 2 +- standalone_cli/src/pwc_cli/__init__.py | 2 +- standalone_cli/src/pwc_cli/cli.py | 119 ++++++++++++++++++++++--- standalone_cli/tests/test_cli.py | 75 +++++++++++++++- 18 files changed, 384 insertions(+), 43 deletions(-) diff --git a/mcp_server/README.md b/mcp_server/README.md index 2027631..28f8e33 100644 --- a/mcp_server/README.md +++ b/mcp_server/README.md @@ -67,7 +67,8 @@ the caller controls hybrid (default), keyword, or semantic mode. `read_paper` fetches at most one 64 KiB catalog chunk per call and returns a signed, one-hour continuation cursor when more Markdown remains. Continuations stay pinned to the resolved paper and content version, so a changed paper fails with an -explicit restart response. +explicit restart response; any reference that resolves to the same paper (the +numeric catalog ID after starting from the arXiv ID, say) may carry the cursor. ## Resources diff --git a/mcp_server/SKILL.md b/mcp_server/SKILL.md index 8218277..a1f7db0 100644 --- a/mcp_server/SKILL.md +++ b/mcp_server/SKILL.md @@ -4,7 +4,7 @@ description: "Papers With Code MCP tools for searching and reading AI/ML papers, compatibility: "Requires an MCP client connected to https://paperswithcode.co/mcp with the Papers With Code tools available." --- -Generated for `pwc-mcp v0.2.3` and stock-client MCP protocol `2025-11-25`. +Generated for `pwc-mcp v0.2.4` and stock-client MCP protocol `2025-11-25`. The tools query the public [Papers With Code](https://paperswithcode.co) catalog anonymously and are read-only. Every tool runs the matching `pwc` CLI research diff --git a/mcp_server/SPEC.md b/mcp_server/SPEC.md index 10bfae7..584502e 100644 --- a/mcp_server/SPEC.md +++ b/mcp_server/SPEC.md @@ -65,8 +65,10 @@ Responses use stable, MCP-specific versioned structured outputs with a text fallback. `read_paper` performs one upstream read of at most 64 KiB per call and returns a signed opaque continuation cursor when more Markdown remains. The cursor binds the original reference, canonical paper, content version, byte -offset, chunk limit, key identifier, and fixed one-hour expiry. The current and -previous signing keys support rotation without accepting unsigned state. +offset, chunk limit, key identifier, and fixed one-hour expiry. A continuation +supplied with a different reference is honoured only when that reference +resolves to the cursor's canonical paper. The current and previous signing keys +support rotation without accepting unsigned state. Paper references accept arXiv IDs, numeric PwC external IDs, arXiv/Hugging Face/Papers With Code URLs, and exact titles. Ambiguous exact titles fail rather diff --git a/mcp_server/pyproject.toml b/mcp_server/pyproject.toml index 7fb5b70..bd55a1d 100644 --- a/mcp_server/pyproject.toml +++ b/mcp_server/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pwc-mcp" -version = "0.2.3" +version = "0.2.4" description = "Read-only Papers With Code MCP server" readme = "README.md" requires-python = ">=3.10" diff --git a/mcp_server/src/pwc_mcp/__init__.py b/mcp_server/src/pwc_mcp/__init__.py index 13777c6..faab903 100644 --- a/mcp_server/src/pwc_mcp/__init__.py +++ b/mcp_server/src/pwc_mcp/__init__.py @@ -1,3 +1,3 @@ """Read-only Papers With Code MCP server.""" -__version__ = "0.2.3" +__version__ = "0.2.4" diff --git a/mcp_server/src/pwc_mcp/catalog.py b/mcp_server/src/pwc_mcp/catalog.py index b39ae68..eadc6ad 100644 --- a/mcp_server/src/pwc_mcp/catalog.py +++ b/mcp_server/src/pwc_mcp/catalog.py @@ -272,6 +272,34 @@ def _resolve_paper(self, reference: str) -> str: return candidate query = candidate.replace("-", " ") if slug_from_url else candidate target = " ".join(candidate.split()).casefold() + # The keyword search treats a quoted query as one phrase, so an exact + # title comes back in a single small page even when its words are + # common ("Attention Is All You Need" has over a thousand keyword + # hits). The plain query remains the fallback for titles the phrase + # parser cannot express. + phrase = '"' + " ".join(query.replace('"', " ").split()) + '"' + exact, exhausted = self._exact_title_matches(phrase, target, slug_from_url) + if not exact: + exact, exhausted = self._exact_title_matches(query, target, slug_from_url) + if len(exact) == 1: + return next(iter(exact)) + if exact: + choices = "; ".join( + f"{item.get('title')} ({paper})" for paper, item in exact.items() + ) + raise ResponseError(f"Paper title is ambiguous: {candidate}; {choices}") + if not exhausted: + raise ResponseError("Too many results to resolve paper title safely") + raise ResponseError(f"Paper title not found: {candidate}") + + def _exact_title_matches( + self, query: str, target: str, slug_from_url: str | None + ) -> tuple[dict[str, dict[str, Any]], bool]: + """Collect papers whose title (or slug) equals the target. + + Returns the matches and whether the search was read to its end within + the page budget; an unexhausted search with no match is inconclusive. + """ exact: dict[str, dict[str, Any]] = {} page = 1 while page <= 10: @@ -292,18 +320,9 @@ def _resolve_paper(self, reference: str) -> str: exact.setdefault(paper, item) next_page = payload.get("next_page") if not isinstance(next_page, int) or next_page <= page: - break + return exact, True page = next_page - else: - raise ResponseError("Too many results to resolve paper title safely") - if len(exact) == 1: - return next(iter(exact)) - if exact: - choices = "; ".join( - f"{item.get('title')} ({paper})" for paper, item in exact.items() - ) - raise ResponseError(f"Paper title is ambiguous: {candidate}; {choices}") - raise ResponseError(f"Paper title not found: {candidate}") + return exact, False def _canonical_paper_id(self, catalog_id: str) -> str: """Prefer the arXiv ID for a numeric catalog ID so every route accepts it. diff --git a/mcp_server/src/pwc_mcp/cursors.py b/mcp_server/src/pwc_mcp/cursors.py index fcca79b..4494c95 100644 --- a/mcp_server/src/pwc_mcp/cursors.py +++ b/mcp_server/src/pwc_mcp/cursors.py @@ -18,6 +18,21 @@ _PAPER_ID = re.compile(r"(?:\d{4}\.\d{4,5}|\d{1,20})") +class CursorReferenceMismatch(ValueError): + """A valid cursor whose paper reference differs from the one supplied. + + The caller may still honour it when the new reference resolves to the + cursor's paper (an agent that started from an arXiv ID and continues with + the numeric catalog ID, say). + """ + + def __init__(self, state: CursorState) -> None: + super().__init__( + "invalid continuation cursor: it belongs to another paper reference" + ) + self.state = state + + @dataclass(frozen=True) class CursorState: reference: str @@ -129,15 +144,18 @@ def decode(self, token: str, *, reference: str) -> CursorState: "kid", "exp", } - or state.reference != reference.strip() or not self._valid_state(state) ): raise ValueError if state.expires_at <= int(self._now()): raise TimeoutError + if state.reference != reference.strip(): + raise CursorReferenceMismatch(state) return state except TimeoutError as error: raise ValueError("expired continuation cursor") from error + except CursorReferenceMismatch: + raise except ( ValueError, TypeError, diff --git a/mcp_server/src/pwc_mcp/server.py b/mcp_server/src/pwc_mcp/server.py index 79982d0..469782d 100644 --- a/mcp_server/src/pwc_mcp/server.py +++ b/mcp_server/src/pwc_mcp/server.py @@ -32,6 +32,7 @@ CURSOR_LIFETIME_SECONDS, MAX_CHUNK_BYTES, CursorCodec, + CursorReferenceMismatch, CursorState, ) from pwc_mcp.models import ( @@ -475,12 +476,28 @@ def get_paper_evaluations( @server.tool(annotations=READ_ONLY, structured_output=True) def read_paper(paper: Reference, cursor: str | None = None) -> PaperReadResult: - """Read stored paper Markdown, continuing oversized documents with a cursor (`pwc paper read`).""" + """Read stored paper Markdown, continuing oversized documents with a cursor (`pwc paper read`). Pass the next_cursor value from the previous result together with the same paper (any reference to that paper works); omit cursor to start from the beginning.""" reference = paper.strip() try: state = codec.decode(cursor, reference=reference) if cursor else None + except CursorReferenceMismatch as error: + # The agent continued with another spelling of the same paper, for + # example the numeric catalog ID after starting from the arXiv ID. + try: + canonical = catalog.resolve_paper(reference) + except (ResponseError, TransportError) as inner: + raise ToolError(catalog_error_message(inner)) from inner + if canonical != error.state.paper: + raise ToolError( + "continuation cursor belongs to a different paper; omit cursor " + "to start reading this paper from the beginning" + ) from error + state = error.state except ValueError as error: - raise ToolError(str(error)) from error + raise ToolError( + f"{error}; pass the next_cursor value from the previous read_paper " + "result, or omit cursor to start from the beginning" + ) from error if state is None: try: canonical = catalog.resolve_paper(reference) @@ -818,7 +835,7 @@ def get_benchmark( sort_metric: SortMetric | None = None, pareto: ParetoObjectives | None = None, ) -> BenchmarkResult: - """Get one exact benchmark and its leaderboard (`pwc benchmark --name`). Use max_parameters (for example "4B") to keep models at or below a size, sort_metric to rank by a metric, and minimum_metrics, maximum_metrics, require_metrics, or pareto to select rows; matched_count reports how many rows passed before limit. Metric names are matched case-insensitively and through common aliases (AP/mAP, top1/Accuracy, AUROC/AUC); an unknown metric error lists the leaderboard's actual metric names.""" + """Get one exact benchmark and its leaderboard (`pwc benchmark --name`). Use max_parameters (for example "4B") to keep models at or below a size, sort_metric to rank by a metric, and minimum_metrics, maximum_metrics, require_metrics, or pareto to select rows; matched_count reports how many rows passed before limit. Metric names are matched case-insensitively, through common aliases (AP/mAP, top1/Accuracy, AUROC/AUC, Pass@1/Pass Rate), and by the one leaderboard metric containing the request or an alias of it (Normalized Score for "D4RL Normalized Score", GenEval Score for "Overall"); an unknown metric error lists the leaderboard's actual metric names.""" data = run( "get_benchmark", benchmark=benchmark, diff --git a/mcp_server/tests/test_app.py b/mcp_server/tests/test_app.py index 3a2e165..c413ac9 100644 --- a/mcp_server/tests/test_app.py +++ b/mcp_server/tests/test_app.py @@ -43,7 +43,7 @@ def test_health_and_browser_origin_policy_are_explicit(): assert health.json() == { "status": "ok", "service": "pwc-mcp", - "version": "0.2.3", + "version": "0.2.4", "protocol": "2025-11-25", } assert rejected.status_code == 403 diff --git a/mcp_server/tests/test_catalog.py b/mcp_server/tests/test_catalog.py index 1069649..7e01894 100644 --- a/mcp_server/tests/test_catalog.py +++ b/mcp_server/tests/test_catalog.py @@ -359,4 +359,68 @@ def test_catalog_reports_titles_missing_from_an_empty_search_as_not_found(): with pytest.raises(ResponseError, match="Paper title not found: Dropout"): catalog.resolve_paper("Dropout: A Simple Way to Prevent Overfitting") - assert len(transport.calls) == 1 + # One phrase query, then one plain keyword query as the fallback. + assert [params["q"] for _path, params in transport.calls] == [ + '"Dropout: A Simple Way to Prevent Overfitting"', + "Dropout: A Simple Way to Prevent Overfitting", + ] + + +EXACT_TITLE_ROW = {"id": "755", "arxiv_id": "1706.03762", "title": "Attention Is All You Need"} + + +class TitleSearchTransport: + """Keyword search whose plain results never end; the phrase query is small.""" + + def __init__(self, *, phrase_results, plain_has_exact): + self.phrase_results = phrase_results + self.plain_has_exact = plain_has_exact + self.calls = [] + + def get(self, path, params=None): + params = dict(params or {}) + self.calls.append((path, params)) + assert path == "papers/search" + page = params["page"] + if params["q"].startswith('"'): + body = {"results": self.phrase_results, "next_page": None} + else: + rows = [ + {"id": str(page * 100 + i), "arxiv_id": f"2{page:03d}.{i:05d}", "title": f"Attention {i}"} + for i in range(100) + ] + if self.plain_has_exact and page == 1: + rows[0] = EXACT_TITLE_ROW + body = {"results": rows, "next_page": page + 1} + return Response(json.dumps(body).encode(), {"content-type": "application/json"}) + + +def test_catalog_resolves_common_word_titles_through_one_phrase_query(): + # "Attention Is All You Need" has over a thousand keyword hits; paging + # through them used to end in "Too many results" although the exact title + # was the first result. + transport = TitleSearchTransport( + phrase_results=[ + EXACT_TITLE_ROW, + {"id": "9", "arxiv_id": "2010.13154", "title": "Attention is All You Need in Speech Separation"}, + ], + plain_has_exact=True, + ) + catalog = CatalogClient(transport=transport) + + assert catalog.resolve_paper("Attention Is All You Need") == "1706.03762" + assert [params["q"] for _path, params in transport.calls] == ['"Attention Is All You Need"'] + + +def test_catalog_keeps_exact_matches_found_before_the_page_budget_runs_out(): + transport = TitleSearchTransport(phrase_results=[], plain_has_exact=True) + catalog = CatalogClient(transport=transport) + + assert catalog.resolve_paper("Attention Is All You Need") == "1706.03762" + assert len(transport.calls) == 11 + + inconclusive = CatalogClient( + transport=TitleSearchTransport(phrase_results=[], plain_has_exact=False) + ) + with pytest.raises(ResponseError, match="Too many results"): + inconclusive.resolve_paper("Attention Is All You Need") diff --git a/mcp_server/tests/test_cursors.py b/mcp_server/tests/test_cursors.py index d0054e9..ddfef66 100644 --- a/mcp_server/tests/test_cursors.py +++ b/mcp_server/tests/test_cursors.py @@ -1,7 +1,7 @@ from __future__ import annotations import pytest -from pwc_mcp.cursors import CursorCodec, CursorState +from pwc_mcp.cursors import CursorCodec, CursorReferenceMismatch, CursorState def _state(*, expires_at: int = 4600) -> CursorState: @@ -57,3 +57,17 @@ def test_cursor_rejects_oversized_or_invalid_state(): codec.encode(_state(expires_at=1000)) with pytest.raises(ValueError, match="cursor secret"): CursorCodec("") + + +def test_cursor_for_another_reference_reports_the_mismatch_with_its_state(): + codec = CursorCodec("current-secret", now=lambda: 1000) + token = codec.encode(_state()) + + with pytest.raises(CursorReferenceMismatch, match="invalid continuation cursor") as caught: + codec.decode(token, reference="755") + assert caught.value.state == _state() + + # Tampering and expiry still win over the reference check. + expired = CursorCodec("current-secret", now=lambda: 5000) + with pytest.raises(ValueError, match="expired continuation cursor"): + expired.decode(token, reference="755") diff --git a/mcp_server/tests/test_server.py b/mcp_server/tests/test_server.py index 00ef157..f25f582 100644 --- a/mcp_server/tests/test_server.py +++ b/mcp_server/tests/test_server.py @@ -482,10 +482,48 @@ def test_read_paper_rejects_invalid_continuation_as_an_expected_error(): assert result.is_error is True assert result.content[0].text == ( - "Error executing tool read_paper: invalid continuation cursor" + "Error executing tool read_paper: invalid continuation cursor; pass the " + "next_cursor value from the previous read_paper result, or omit cursor to " + "start from the beginning" ) +class AliasedStubCatalog(StubCatalog): + """Resolves the numeric catalog ID and the arXiv ID to the same paper.""" + + def resolve_paper(self, paper: str): + self.resolve_calls += 1 + return {"1706.03762": "1706.03762", "755": "1706.03762", "1810.04805": "1810.04805"}[paper] + + +def test_read_paper_continues_with_another_reference_to_the_same_paper(): + # Agents start from the arXiv ID and continue with the numeric ID that + # list tools hand out; the cursor must follow the paper, not the spelling. + catalog = AliasedStubCatalog() + + async def exercise(): + async with Client(build_server(catalog, read_chunk_bytes=5)) as client: + first = await client.call_tool("read_paper", {"paper": "1706.03762"}) + cursor = first.structured_content["next_cursor"] + same = await client.call_tool("read_paper", {"paper": "755", "cursor": cursor}) + other = await client.call_tool("read_paper", {"paper": "1810.04805", "cursor": cursor}) + return first, same, other + + first, same, other = asyncio.run(exercise()) + + assert first.structured_content["markdown"] == "abcde" + assert same.is_error is False + assert same.structured_content["paper"] == "755" + assert same.structured_content["markdown"] == "fgh" + assert same.structured_content["next_cursor"] is None + assert other.is_error is True + assert other.content[0].text == ( + "Error executing tool read_paper: continuation cursor belongs to a different " + "paper; omit cursor to start reading this paper from the beginning" + ) + assert catalog.read_calls == [(0, None, 5), (5, "a" * 64, 5)] + + def test_paper_listing_related_work_and_lineage_are_composable(): catalog = StubCatalog() listed, recent, trending, related, lineage = _call( diff --git a/mcp_server/uv.lock b/mcp_server/uv.lock index e54f7b6..f467dd6 100644 --- a/mcp_server/uv.lock +++ b/mcp_server/uv.lock @@ -438,12 +438,12 @@ wheels = [ [[package]] name = "pwc-cli" -version = "0.4.2" +version = "0.4.3" source = { editable = "../standalone_cli" } [[package]] name = "pwc-mcp" -version = "0.2.3" +version = "0.2.4" source = { editable = "." } dependencies = [ { name = "mcp" }, diff --git a/standalone_cli/SKILL.md b/standalone_cli/SKILL.md index d532946..989e43e 100644 --- a/standalone_cli/SKILL.md +++ b/standalone_cli/SKILL.md @@ -3,7 +3,7 @@ name: pwc-cli description: "Papers With Code CLI (`pwc`) for searching and reading AI/ML papers, discovering recent and trending research, finding related work and paper lineage, browsing tasks, methods, conferences, organizations, frameworks, and benchmark leaderboards, and submitting authenticated paper edits through the public Papers With Code catalog. Use whenever the user asks to find papers, survey literature, compare research, inspect an arXiv paper, explore AI/ML taxonomy or conferences, discover benchmarks or state-of-the-art models, or mentions Papers With Code, `pwc`, or `pwc-cli`." --- -Generated with `pwc v0.4.2`. Run `pwc skills add --force` to regenerate. +Generated with `pwc v0.4.3`. Run `pwc skills add --force` to regenerate. Research commands query the public [Papers With Code](https://paperswithcode.co) catalog anonymously. Paper editing requires explicit browser authorization through `pwc auth login --paper PAPER`. diff --git a/standalone_cli/pyproject.toml b/standalone_cli/pyproject.toml index c8db285..b3f01ee 100644 --- a/standalone_cli/pyproject.toml +++ b/standalone_cli/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pwc-cli" -version = "0.4.2" +version = "0.4.3" description = "Papers With Code research and paper-editing CLI" readme = "README.md" requires-python = ">=3.10" diff --git a/standalone_cli/src/pwc_cli/__init__.py b/standalone_cli/src/pwc_cli/__init__.py index ab2fd92..d163931 100644 --- a/standalone_cli/src/pwc_cli/__init__.py +++ b/standalone_cli/src/pwc_cli/__init__.py @@ -1,4 +1,4 @@ """Standalone Papers With Code client.""" -__version__ = "0.4.2" +__version__ = "0.4.3" API_CONTRACT_VERSION = "v1" diff --git a/standalone_cli/src/pwc_cli/cli.py b/standalone_cli/src/pwc_cli/cli.py index e2edc73..c4aeca1 100644 --- a/standalone_cli/src/pwc_cli/cli.py +++ b/standalone_cli/src/pwc_cli/cli.py @@ -210,6 +210,67 @@ def _closest_names(reference: str, items: list[dict[str, Any]], limit: int = 3) return ", ".join(name for _score, _index, name in ranked[:limit]) +_TRAILING_PARENTHETICAL = re.compile(r"\s*\(([^()]*)\)\s*$") + + +def _normalize_entity_name(value: object) -> str: + return re.sub(r"[^a-z0-9]+", "", str(value or "").casefold()) + + +def _without_abbreviation(name: str) -> str | None: + """Drop a trailing acronym such as "(MoE)" or "(ViT)", but not "(Video)". + + The parenthetical counts as an abbreviation when it is short, starts with + the name's first letter, and its letters appear in order inside the name. + """ + match = _TRAILING_PARENTHETICAL.search(name) + if match is None: + return None + stem = name[: match.start()] + abbreviation = _normalize_entity_name(match.group(1)) + normalized_stem = _normalize_entity_name(stem) + if not abbreviation or len(abbreviation) > 8 or not normalized_stem: + return None + if abbreviation[0] != normalized_stem[0]: + return None + position = 0 + for character in abbreviation: + position = normalized_stem.find(character, position) + if position < 0: + return None + position += 1 + return stem + + +def _normalized_entity_match( + reference: str, items: list[dict[str, Any]], fields: tuple[str, ...] +) -> dict[str, Any] | None: + """Match one candidate on letters and digits alone. + + Agents write "mixture of experts" for "Mixture-of-Experts (MoE)" and + "time-series forecasting" for "Time Series Forecasting"; punctuation, + spacing, and a trailing acronym must not turn those into lookup failures. + Only a unique match resolves; anything else keeps the exact-name error. + """ + target = _normalize_entity_name(reference) + if len(target) < 3: + return None + matches: list[dict[str, Any]] = [] + for item in items: + for field in fields: + value = str(item.get(field) or "") + if not value: + continue + forms = {_normalize_entity_name(value)} + stem = _without_abbreviation(value) + if stem is not None: + forms.add(_normalize_entity_name(stem)) + if target in forms: + matches.append(item) + break + return matches[0] if len(matches) == 1 else None + + def _exact_entity_match( reference: str, items: list[dict[str, Any]], @@ -222,6 +283,9 @@ def _exact_entity_match( for item in items: if str(item.get(field) or "").strip().casefold() == target: return item + normalized = _normalized_entity_match(reference, items, fields) + if normalized is not None: + return normalized suggestions = _closest_names(reference, items) suffix = f"; closest results: {suggestions}" if suggestions else "" raise ResponseError(f"{label} not found: {reference}{suffix}") @@ -945,11 +1009,12 @@ def render(items: list[dict[str, Any]]) -> None: def _task_match(name: str, items: list[dict[str, Any]]) -> dict[str, Any] | None: target = name.strip().casefold() - for field in ("name", "slug", "id"): + fields = ("name", "slug", "id") + for field in fields: for item in items: if str(item.get(field) or "").strip().casefold() == target: return item - return None + return _normalized_entity_match(name, items, fields) def _task_reference(item: dict[str, Any], *, markdown: bool) -> str: @@ -2084,14 +2149,29 @@ def _metric_requests(args: argparse.Namespace) -> tuple[str, ...]: "auroc": ("auc", "rocauc"), "auc": ("auroc", "rocauc"), "rocauc": ("auc", "auroc"), - "f1": ("f1score",), - "f1score": ("f1",), + "f1": ("f1score", "fscore", "fmeasure"), + "f1score": ("f1", "fscore", "fmeasure"), + "fscore": ("f1", "f1score", "fmeasure"), + "fmeasure": ("f1", "f1score", "fscore"), "miou": ("meaniou", "iou"), "meaniou": ("miou", "iou"), "bleu": ("bleuscore",), "bleuscore": ("bleu",), "totalscore": ("total", "overall", "overallscore", "score"), - "score": ("totalscore", "overallscore"), + "score": ("totalscore", "overallscore", "overall"), + "overall": ("overallscore", "totalscore", "score", "total", "average"), + "overallscore": ("overall", "totalscore", "score"), + "pass1": ("passrate", "pass1accuracy", "accuracy", "successrate", "resolved"), + "passrate": ("pass1", "successrate", "accuracy"), + "successrate": ("passrate", "pass1", "success"), + "imagelevelauroc": ("detectionauroc", "imageauroc", "auroc", "iauroc"), + "pixellevelauroc": ("segmentationauroc", "pixelauroc", "pauroc"), + "wer": ("worderrorrate",), + "worderrorrate": ("wer",), + "cer": ("charactererrorrate",), + "charactererrorrate": ("cer",), + "em": ("exactmatch",), + "exactmatch": ("em",), } @@ -2100,7 +2180,14 @@ def _normalize_metric_name(value: str) -> str: def _resolve_metric_name(requested: str, available: dict[str, str]) -> str | None: - """Map a requested metric onto the leaderboard's own name, or None.""" + """Map a requested metric onto the leaderboard's own name, or None. + + Resolution order: exact name, spelling ignoring case and punctuation, + common aliases, then the one leaderboard name containing the request or + an alias of it (or contained in the request): "Normalized Score" answers + "D4RL Normalized Score", "GenEval Score" answers "Overall" through the + "score" alias, "Detection AUROC" answers "image-level AUROC". + """ key = requested.casefold() if key in available: return available[key] @@ -2108,11 +2195,21 @@ def _resolve_metric_name(requested: str, available: dict[str, str]) -> str | Non by_normalized: dict[str, str] = {} for name in available.values(): by_normalized.setdefault(_normalize_metric_name(name), name) - if normalized in by_normalized: - return by_normalized[normalized] - for alias in METRIC_ALIASES.get(normalized, ()): - if alias in by_normalized: - return by_normalized[alias] + spellings = (normalized, *METRIC_ALIASES.get(normalized, ())) + for spelling in spellings: + if spelling in by_normalized: + return by_normalized[spelling] + for spelling in spellings: + if len(spelling) < 2: + continue + containing = [ + name + for candidate, name in by_normalized.items() + if len(candidate) >= 2 + and (spelling in candidate or candidate in spelling) + ] + if len(containing) == 1: + return containing[0] return None diff --git a/standalone_cli/tests/test_cli.py b/standalone_cli/tests/test_cli.py index 8a168f9..089fc4a 100644 --- a/standalone_cli/tests/test_cli.py +++ b/standalone_cli/tests/test_cli.py @@ -94,7 +94,7 @@ def test_generated_skill_matches_installed_cli_version_and_commands(): skill = build_skill_md() assert "name: pwc-cli" in skill - assert "Generated with `pwc v0.4.2`" in skill + assert "Generated with `pwc v0.4.3`" in skill assert "`pwc search QUERY" in skill assert "--include-evals" in skill assert "[--organization ORGANIZATION]" in skill @@ -2306,7 +2306,7 @@ def test_top_level_version_is_offline_and_stable(): build_parser().parse_args(["--version"]) except SystemExit as error: assert error.code == 0 - assert output.getvalue() == "pwc 0.4.2\tapi v1\n" + assert output.getvalue() == "pwc 0.4.3\tapi v1\n" def test_search_default_output_is_compact_deterministic_tsv(monkeypatch): @@ -3034,3 +3034,74 @@ def test_metric_requests_resolve_through_case_and_common_aliases(): ) with pytest.raises(UsageError, match="unknown metric\\(s\\): latency; available metrics: FPS, mAP"): _select_metric_rows(rows, unknown) + + +def test_metric_requests_resolve_by_containment_and_alias_containment(): + from pwc_cli.cli import _resolve_metric_name + + # Requests seen from chat agents against production leaderboards. + assert ( + _resolve_metric_name("D4RL Normalized Score", {"normalized score": "Normalized Score"}) + == "Normalized Score" + ) + assert _resolve_metric_name("Overall", {"geneval score": "GenEval Score"}) == "GenEval Score" + assert _resolve_metric_name("Score", {"dpg-bench score": "DPG-Bench Score"}) == "DPG-Bench Score" + assert _resolve_metric_name("Pass@1", {"accuracy": "Accuracy"}) == "Accuracy" + assert _resolve_metric_name("Pass Rate", {"success rate": "Success Rate"}) == "Success Rate" + mvtec = { + "detection auroc": "Detection AUROC", + "segmentation auroc": "Segmentation AUROC", + "detection ap": "Detection AP", + } + assert _resolve_metric_name("image-level AUROC", mvtec) == "Detection AUROC" + assert _resolve_metric_name("pixel-level AUROC", mvtec) == "Segmentation AUROC" + # Ambiguous or unrelated requests still surface the leaderboard's names. + assert _resolve_metric_name("AUROC", mvtec) is None + assert ( + _resolve_metric_name( + "F-score", + {"mean f1 (advanced)": "Mean F1 (Advanced)", "mean f1 (intermediate)": "Mean F1 (Intermediate)"}, + ) + is None + ) + assert _resolve_metric_name("latency", {"map": "mAP"}) is None + + +def test_entity_names_match_ignoring_punctuation_and_trailing_acronyms(): + from pwc_cli.cli import _exact_entity_match, _task_match, _without_abbreviation + + methods = [ + {"name": "Mixture-of-Experts (MoE)", "slug": "moe"}, + {"name": "Vision Transformer (ViT)", "slug": "vit"}, + ] + assert _exact_entity_match("mixture of experts", methods, label="Method")["slug"] == "moe" + assert _exact_entity_match("vision transformer", methods, label="Method")["slug"] == "vit" + with pytest.raises(ResponseError, match="Method not found: experts"): + _exact_entity_match("experts", methods, label="Method") + + tasks = [ + {"name": "Time Series Forecasting", "slug": "time-series-forecasting"}, + {"name": "Person Re-Identification (Video)", "slug": "person-re-identification-video"}, + {"name": "Unsupervised Person Re-Identification", "slug": "unsupervised-person-re-identification"}, + ] + assert _task_match("time-series forecasting", tasks)["slug"] == "time-series-forecasting" + # "(Video)" qualifies the task instead of abbreviating it, so the bare name + # must not silently pick the video variant. + assert _task_match("person re-identification", tasks) is None + # Two candidates that normalize alike stay unresolved. + assert ( + _task_match( + "image-classification", + [{"name": "Image Classification"}, {"name": "Image, Classification"}], + ) + is None + ) + + assert ( + _without_abbreviation("Physics-Informed Neural Networks (PINNs)") + == "Physics-Informed Neural Networks" + ) + assert _without_abbreviation("Aspect-Based Sentiment Analysis (ABSA)") == "Aspect-Based Sentiment Analysis" + assert _without_abbreviation("Person Re-Identification (Video)") is None + assert _without_abbreviation("MVTec AD (Unified Multi-Class)") is None + assert _without_abbreviation("Knowledge Distillation") is None