From 59cd2985a3bc4b014bd4781b70350bc9a43b12f6 Mon Sep 17 00:00:00 2001 From: Vladimir Iglovikov Date: Tue, 12 May 2026 10:53:20 +0300 Subject: [PATCH] test: harden platform recommendation coverage --- tests/test_paper_assets.py | 108 ++++++++++++++++++++++++++++++------- tools/paper_assets.py | 22 ++++++-- 2 files changed, 106 insertions(+), 24 deletions(-) diff --git a/tests/test_paper_assets.py b/tests/test_paper_assets.py index f02d1e2..f400db9 100644 --- a/tests/test_paper_assets.py +++ b/tests/test_paper_assets.py @@ -5,6 +5,7 @@ import pandas as pd import pytest +from tools import paper_assets from tools._results import load_dataloader_results, load_results from tools.paper_assets import ( EXPECTED_DATALOADER_DECODERS, @@ -13,6 +14,9 @@ PAPER_PLATFORMS, ROBUSTNESS_DECODERS, WORKERS_ORDER, + _latex_table_row, + _platform_recommendation_choices, + _platform_recommendation_rows, generate_platform_recommendation_table, generate_robustness_table, validate_paper_data, @@ -153,22 +157,88 @@ def test_platform_recommendation_table_has_three_zero_skip_choices_per_platform( assert r"\texttt{pyvips}:" not in text assert r"\texttt{tensorflow}:" not in text - expected_choices = [ - r"\texttt{simplejpeg}: 1754 img/s ($w=8$)", - r"\texttt{opencv}: 1707 img/s ($w=8$)", - r"\texttt{imagecodecs}: 1677 img/s ($w=8$)", - r"\texttt{torchvision}: 1596 img/s ($w=8$)", - r"\texttt{imagecodecs}: 1543 img/s ($w=4$)", - r"\texttt{simplejpeg}: 1521 img/s ($w=4$)", - r"\texttt{torchvision}: 2920 img/s ($w=8$)", - r"\texttt{opencv}: 2814 img/s ($w=8$)", - r"\texttt{simplejpeg}: 2739 img/s ($w=8$)", - r"\texttt{imageio}: 2561 img/s ($w=8$)", - r"\texttt{torchvision}: 2557 img/s ($w=8$)", - r"\texttt{simplejpeg}: 2421 img/s ($w=8$)", - r"\texttt{simplejpeg}: 1557 img/s ($w=8$)", - r"\texttt{torchvision}: 1504 img/s ($w=8$)", - r"\texttt{imageio}: 1466 img/s ($w=8$)", - ] - for choice in expected_choices: - assert choice in text + choices_by_platform = _platform_recommendation_choices(load_dataloader_results(root)) + assert [platform for platform, _ in choices_by_platform] == list(PAPER_PLATFORMS) + for _, choices in choices_by_platform: + assert len(choices) == 3 + assert choices["library"].is_unique + assert choices["peak_ips"].tolist() == sorted(choices["peak_ips"], reverse=True) + assert choices["peak_workers"].isin(WORKERS_ORDER).all() + + +def test_latex_table_row_formats_cells_with_indentation_and_linebreak() -> None: + assert _latex_table_row(["A", "B", "C"]) == r" A & B & C \\" + + +def test_platform_recommendation_rows_requires_three_zero_skip_choices() -> None: + platform = PAPER_PLATFORMS[0] + dl = pd.DataFrame( + [ + { + "platform": platform, + "library": "opencv", + "num_workers": 0, + "images_per_second": 100.0, + "cpu_brand": "Synthetic CPU", + }, + { + "platform": platform, + "library": "imageio", + "num_workers": 2, + "images_per_second": 90.0, + "cpu_brand": "Synthetic CPU", + }, + ], + ) + + with pytest.raises(ValueError, match="fewer than three zero-skip DataLoader choices"): + _platform_recommendation_rows(dl) + + +def test_platform_recommendation_choices_ignore_skip_decoders(monkeypatch: pytest.MonkeyPatch) -> None: + platform = PAPER_PLATFORMS[0] + monkeypatch.setattr(paper_assets, "PAPER_PLATFORMS", (platform,)) + dl = pd.DataFrame( + [ + { + "platform": platform, + "library": "jpeg4py", + "num_workers": 8, + "images_per_second": 500.0, + "cpu_brand": "Synthetic CPU", + }, + { + "platform": platform, + "library": "turbojpeg", + "num_workers": 8, + "images_per_second": 450.0, + "cpu_brand": "Synthetic CPU", + }, + { + "platform": platform, + "library": "opencv", + "num_workers": 4, + "images_per_second": 300.0, + "cpu_brand": "Synthetic CPU", + }, + { + "platform": platform, + "library": "imageio", + "num_workers": 2, + "images_per_second": 200.0, + "cpu_brand": "Synthetic CPU", + }, + { + "platform": platform, + "library": "pillow", + "num_workers": 0, + "images_per_second": 100.0, + "cpu_brand": "Synthetic CPU", + }, + ], + ) + + [(selected_platform, choices)] = _platform_recommendation_choices(dl) + + assert selected_platform == platform + assert choices["library"].tolist() == ["opencv", "imageio", "pillow"] diff --git a/tools/paper_assets.py b/tools/paper_assets.py index 26c68e0..e00ffec 100644 --- a/tools/paper_assets.py +++ b/tools/paper_assets.py @@ -430,19 +430,27 @@ def _format_platform_choice(row: pd.Series) -> str: return rf"\texttt{{{lib}}}: {ips:.0f} img/s ($w={workers}$)" -def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]: +def _platform_recommendation_choices(dl: pd.DataFrame) -> list[tuple[str, pd.DataFrame]]: dl = _paper_scope(dl) peak = _peak_dataloader_rows(dl) - plat_labels = _platform_labels(_cpu_by_platform(dl)) zero_skip_loader_decoders = EXPECTED_DATALOADER_DECODERS - EXPECTED_SKIP_DECODERS - rows: list[list[str]] = [] + choices_by_platform: list[tuple[str, pd.DataFrame]] = [] for plat in PAPER_PLATFORMS: sub = peak[(peak["platform"] == plat) & (peak["library"].isin(zero_skip_loader_decoders))] sub = sub.sort_values(["peak_ips", "library"], ascending=[False, True]) if len(sub) < 3: raise ValueError(f"{plat} has fewer than three zero-skip DataLoader choices") - choices = [_format_platform_choice(sub.iloc[i]) for i in range(3)] + choices_by_platform.append((plat, sub.head(3).reset_index(drop=True))) + return choices_by_platform + + +def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]: + plat_labels = _platform_labels(_cpu_by_platform(_paper_scope(dl))) + + rows: list[list[str]] = [] + for plat, choices_df in _platform_recommendation_choices(dl): + choices = [_format_platform_choice(choices_df.iloc[i]) for i in range(3)] rows.append( [ _latex_escape(plat_labels[plat]), @@ -453,6 +461,10 @@ def _platform_recommendation_rows(dl: pd.DataFrame) -> list[list[str]]: return rows +def _latex_table_row(row: list[str]) -> str: + return " " + " & ".join(row) + r" \\" + + def generate_platform_recommendation_table(dl: pd.DataFrame, dest: Path) -> None: headers = [ "Platform", @@ -462,7 +474,7 @@ def generate_platform_recommendation_table(dl: pd.DataFrame, dest: Path) -> None "Note", ] rows = _platform_recommendation_rows(dl) - body = "\n".join(" " + " & ".join(row) + r" \\" for row in rows) + body = "\n".join(_latex_table_row(row) for row in rows) header = " & ".join(headers) caption = ( r" \caption{Per-platform zero-skip DataLoader starting points. Values are measured peak PyTorch "