diff --git a/every_eval_ever/adapters/arc_agi/adapter.py b/every_eval_ever/adapters/arc_agi/adapter.py index a3d7dc400..0d90a76fe 100644 --- a/every_eval_ever/adapters/arc_agi/adapter.py +++ b/every_eval_ever/adapters/arc_agi/adapter.py @@ -9,6 +9,13 @@ id -> display name). ``datasets.json`` names the ARC-AGI-1/2/3 splits. - The older ``/media/data/leaderboard/evaluations.json`` endpoint is gone (404 as of 2026-08-12); this adapter fetches the current files. +- Published run logs for the two public sets: the Hugging Face datasets + ``arcprize/arc_agi_v1_public_eval`` and ``arcprize/arc_agi_v2_public_eval``, + read at pinned commits (see ``run_logs.py``). A ``v1_Public_Eval`` or + ``v2_Public_Eval`` score whose chosen row's ``modelId`` names a run + directory that passes every gate gets the logged output cap, a uniform + logged temperature and the scored task count; a directory that fails a gate + is recorded as ``request_args_unavailable``. Each evaluation row has shape: { @@ -30,7 +37,8 @@ uv run python -m every_eval_ever.adapters.arc_agi.adapter \\ --output-dir data/arc-agi uv run python -m every_eval_ever.adapters.arc_agi.adapter \\ - --input-json /tmp/arc_agi_payload.json --output-dir /tmp/arc-smoke + --input-json /tmp/arc_agi_payload.json --output-dir /tmp/arc-smoke \\ + --arc-runs-dir /tmp/arc_public_eval """ from __future__ import annotations @@ -39,16 +47,28 @@ import json import math import re +import sys +import tempfile import time -from collections import defaultdict +from collections import Counter, defaultdict from pathlib import Path from typing import Any +from every_eval_ever.adapters.arc_agi.run_logs import ( + RUN_DATASETS, + RunLogs, + RunLogsUnavailable, + fetch_run_logs, + loads_stated, + local_run_logs, +) from every_eval_ever.eval_types import ( EvalLibrary, EvaluationLog, EvaluationResult, EvaluatorRelationship, + GenerationArgs, + GenerationConfig, MetricConfig, ModelInfo, ScoreDetails, @@ -56,6 +76,7 @@ SourceDataUrl, SourceMetadata, SourceType, + Uncertainty, ) from every_eval_ever.helpers import ( SCHEMA_VERSION, @@ -66,7 +87,7 @@ save_evaluation_logs, save_failure_report, ) -from every_eval_ever.helpers.fetch import fetch_json +from every_eval_ever.helpers.fetch import fetch_text SOURCE_NAME = 'ARC Prize leaderboard' SOURCE_ORGANIZATION = 'ARC Prize Foundation' @@ -135,6 +156,28 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: 'future runs can replay offline with --input-json.' ), ) + parser.add_argument( + '--arc-runs-dir', + type=Path, + help=( + 'Read the published run logs from this directory, holding v1/ ' + 'and v2/ checkouts of the ARC public-eval datasets, instead of ' + 'cloning them.' + ), + ) + parser.add_argument( + '--arc-history-dir', + type=Path, + help=( + 'With --arc-runs-dir whose v1/ and v2/ are not git checkouts: ' + 'a directory of v1/ and v2/ clones supplying their commit history.' + ), + ) + parser.add_argument( + '--no-run-logs', + action='store_true', + help='Skip the run-log join; score results carry no request args.', + ) parser.add_argument( '--failure-report', type=Path, @@ -146,18 +189,34 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def fetch_payload(base_url: str) -> dict[str, Any]: +PAYLOAD_FILES = ('evaluations', 'models', 'providers', 'datasets') + + +def fetch_payload_texts(base_url: str) -> dict[str, str]: + """Fetch the leaderboard's JSON files as their raw text.""" base = base_url.rstrip('/') - return { - 'evaluations': fetch_json(f'{base}/evaluations.json'), - 'models': fetch_json(f'{base}/models.json'), - 'providers': fetch_json(f'{base}/providers.json'), - 'datasets': fetch_json(f'{base}/datasets.json'), - } + return {name: fetch_text(f'{base}/{name}.json') for name in PAYLOAD_FILES} + + +def parse_payload_texts(texts: dict[str, str]) -> dict[str, Any]: + """Parse the fetched files, keeping each float's written precision.""" + return {name: loads_stated(text) for name, text in texts.items()} + + +def combined_payload_text( + texts: dict[str, str], base_url: str, fetched_at: str +) -> str: + """Return the --save-raw-json file: the fetched files embedded verbatim.""" + parts = [ + f' {json.dumps("source_base_url")}: {json.dumps(base_url)}', + f' {json.dumps("fetched_at")}: {json.dumps(fetched_at)}', + *(f' {json.dumps(name)}: {texts[name].strip()}' for name in texts), + ] + return '{\n' + ',\n'.join(parts) + '\n}\n' def load_payload_file(path: Path) -> dict[str, Any]: - payload = json.loads(path.read_text(encoding='utf-8')) + payload = loads_stated(path.read_text(encoding='utf-8')) if not isinstance(payload, dict) or 'evaluations' not in payload: raise ValueError( f'{path}: expected a combined payload with an "evaluations" key ' @@ -292,11 +351,52 @@ def _cost_result( ) +def run_log_fields( + run_logs: RunLogs | None, + dataset_id: str, + row: dict, + retrieved: float, +) -> tuple[GenerationConfig | None, dict[str, str]]: + """Return the score result's generation config and extra source details. + + Both are empty when the set publishes no logs or no run directory is + named after the row's ``modelId``. + """ + if run_logs is None: + return None, {} + outcome = run_logs.join(dataset_id, row, retrieved) + if outcome is None: + return None, {} + if outcome.run is None: + return ( + GenerationConfig( + additional_details={ + 'request_args_unavailable': str(outcome.rejection), + 'request_log': outcome.request_log, + } + ), + {}, + ) + run = outcome.run + details = {'cap_key': run.cap_key, 'request_log': outcome.request_log} + return ( + GenerationConfig( + generation_args=GenerationArgs( + max_tokens=run.cap_value, temperature=run.temperature + ), + additional_details=details, + ), + {'scored_tasks': str(run.task_count)}, + ) + + def make_results( rows_for_canonical: list[dict], developer_name: str, metric_bounds: dict[str, dict[str, float]], dataset_names: dict[str, str], + run_logs: RunLogs | None = None, + retrieved: float = math.inf, ) -> list[EvaluationResult]: results = [] by_dataset = defaultdict(list) @@ -309,12 +409,17 @@ def make_results( {r['modelId'] for r in by_dataset[dataset_id]} ) dataset_display_name = dataset_names.get(dataset_id) + generation_config, run_source_details = run_log_fields( + run_logs, dataset_id, row, retrieved + ) + score_source_data = make_source_data(dataset_id, dataset_display_name) + score_source_data.additional_details.update(run_source_details) results.append( EvaluationResult( evaluation_result_id=f'{dataset_id}::score', evaluation_name=dataset_id, - source_data=make_source_data(dataset_id, dataset_display_name), + source_data=score_source_data, metric_config=MetricConfig( metric_id='score', metric_name='ARC score', @@ -327,6 +432,11 @@ def make_results( ), score_details=ScoreDetails( score=float(row['score']), + uncertainty=Uncertainty( + num_samples=int(run_source_details['scored_tasks']) + ) + if 'scored_tasks' in run_source_details + else None, details={ **stringify_details( row, exclude_keys={'score', 'modelId'} @@ -337,6 +447,7 @@ def make_results( ), }, ), + generation_config=generation_config, ) ) @@ -377,6 +488,7 @@ def make_log( model_entry: dict | None, provider_entry: dict | None, dataset_names: dict[str, str], + run_logs: RunLogs | None = None, ) -> EvaluationLog: primary_raw_model_id = choose_primary_raw_model_id( rows_for_canonical, developer_name @@ -437,7 +549,12 @@ def make_log( additional_details=model_details, ), evaluation_results=make_results( - rows_for_canonical, developer_name, metric_bounds, dataset_names + rows_for_canonical, + developer_name, + metric_bounds, + dataset_names, + run_logs, + float(retrieved_timestamp), ), ) @@ -451,6 +568,7 @@ def _valid_number(value: Any) -> bool: def convert_logs( payload: dict[str, Any], retrieved_timestamp: str | None = None, + run_logs: RunLogs | None = None, ) -> SourceConversionResult[tuple[EvaluationLog, str, str]]: timestamp = retrieved_timestamp or str(time.time()) models = {m['id']: m for m in payload.get('models') or []} @@ -578,6 +696,7 @@ def convert_logs( model_entry, provider_entry, dataset_names, + run_logs, ) bundles.append((log, developer_name, model_name)) @@ -605,36 +724,165 @@ def export( ) +def public_model_ids(payload: dict[str, Any]) -> dict[str, set[str]]: + """Return the displayed rows' ``modelId``s per set that publishes run logs.""" + model_ids: dict[str, set[str]] = defaultdict(set) + for row in payload.get('evaluations') or []: + if ( + isinstance(row, dict) + and row.get('display') is True + and row.get('datasetId') in RUN_DATASETS + and isinstance(row.get('modelId'), str) + ): + model_ids[row['datasetId']].add(row['modelId']) + return dict(model_ids) + + +#: Gates whose failure means the logs and the leaderboard disagree, as +#: opposed to logs that were never published complete. +REGRESSION_GATES = frozenset( + { + 'score_mismatch', + 'cost_mismatch', + 'task_score_mismatch', + 'task_attempts_mismatch', + 'task_set_mismatch', + 'run_score_mismatch', + 'cap_conflict', + 'cap_key_mixed', + 'pair_index_mismatch', + 'task_id_mismatch', + 'test_id_mismatch', + 'run_rewritten', + } +) + + +def _rejection_entry(outcome: Any) -> dict[str, Any]: + return { + 'model_id': outcome.model_id, + 'dataset_id': outcome.dataset_id, + 'directory': outcome.model_id, + 'gate': outcome.rejection, + 'detail': outcome.detail, + } + + +def run_log_report(run_logs: RunLogs) -> dict[str, Any]: + """Return the run-level join report: sources, gate counts, rows. + + ``gate_counts`` counts every joined score row by outcome (``filled`` or + the gate that rejected it) plus ``no_run_directory`` for rows whose + ``modelId`` names no directory. Rejections are split into + ``regressions`` (:data:`REGRESSION_GATES`) and ``unavailable``. + """ + counts = Counter( + 'filled' if outcome.run is not None else str(outcome.rejection) + for outcome in run_logs.outcomes + ) + if run_logs.unjoined: + counts['no_run_directory'] = len(run_logs.unjoined) + ordered = sorted( + run_logs.outcomes, key=lambda o: (o.dataset_id, o.model_id) + ) + return { + 'sources': { + dataset_id: { + 'dataset': source.dataset.repo_id, + 'revision': source.revision, + } + for dataset_id, source in sorted(run_logs.sources.items()) + }, + 'gate_counts': dict(sorted(counts.items())), + 'regressions': [ + _rejection_entry(o) + for o in ordered + if o.run is None and o.rejection in REGRESSION_GATES + ], + 'unavailable': [ + _rejection_entry(o) + for o in ordered + if o.run is None and o.rejection not in REGRESSION_GATES + ], + 'filled': [ + { + 'model_id': o.model_id, + 'dataset_id': o.dataset_id, + 'directory': o.model_id, + 'cap_key': o.run.cap_key, + 'max_tokens': o.run.cap_value, + 'temperature': o.run.temperature, + 'scored_tasks': o.run.task_count, + } + for o in ordered + if o.run is not None + ], + } + + +def summarize_joins(run_logs: RunLogs) -> str: + """Return a one-line count of run-log joins by outcome.""" + counts = Counter( + 'filled' if outcome.run is not None else str(outcome.rejection) + for outcome in run_logs.outcomes + ) + parts = ', '.join(f'{key} {counts[key]}' for key in sorted(counts)) + return f'ARC run-log joins: {len(run_logs.outcomes)} ({parts or "none"})' + + def run(args: argparse.Namespace) -> int: if args.input_json is not None: payload = load_payload_file(args.input_json) else: - payload = fetch_payload(args.base_url) + texts = fetch_payload_texts(args.base_url) + payload = parse_payload_texts(texts) if args.save_raw_json is not None: args.save_raw_json.parent.mkdir(parents=True, exist_ok=True) args.save_raw_json.write_text( - json.dumps( - { - 'source_base_url': args.base_url, - 'fetched_at': str(time.time()), - **payload, - }, - indent=2, - allow_nan=False, - ), + combined_payload_text(texts, args.base_url, str(time.time())), encoding='utf-8', ) - result = convert_logs(payload) + with tempfile.TemporaryDirectory(prefix='arc-run-logs-') as work_dir: + if args.no_run_logs: + run_logs = None + elif args.arc_runs_dir is not None: + try: + run_logs = local_run_logs( + args.arc_runs_dir, args.arc_history_dir + ) + except RunLogsUnavailable as exc: + raise SystemExit(f'ARC run logs: {exc}') from exc + else: + run_logs = fetch_run_logs( + public_model_ids(payload), Path(work_dir) + ) + result = convert_logs(payload, run_logs=run_logs) paths = export(result.records, args.output_dir) for path in paths: print(path) - if result.failures: + if run_logs is not None: + print(summarize_joins(run_logs)) + for entry in run_log_report(run_logs)['regressions']: + print( + 'ARC run-log regression: ' + f'{entry["model_id"]} {entry["dataset_id"]} {entry["gate"]}', + file=sys.stderr, + ) + if result.failures or run_logs is not None: report_path = save_failure_report( result, args.failure_report or default_failure_report_path(args.output_dir), ) - print(f'Failure report: {report_path}') + if run_logs is not None: + report = json.loads(report_path.read_text(encoding='utf-8')) + report['run_log_evidence'] = run_log_report(run_logs) + report_path.write_text( + json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + + '\n', + encoding='utf-8', + ) + print(f'Adapter report: {report_path}') result.raise_if_incomplete() return len(paths) diff --git a/every_eval_ever/adapters/arc_agi/run_logs.py b/every_eval_ever/adapters/arc_agi/run_logs.py new file mode 100644 index 000000000..60cf4d777 --- /dev/null +++ b/every_eval_ever/adapters/arc_agi/run_logs.py @@ -0,0 +1,635 @@ +"""Join ARC Prize leaderboard score rows to the published run logs. + +ARC Prize publishes the outputs of its public-evaluation runs as two Hugging +Face datasets, one per public set. Each holds one directory per run, named +after the leaderboard ``modelId``, with one JSON file per task (a list of test +pairs, each mapping ``attempt_`` to the logged request and its outcome) and +a ``results.json`` summarising the run. + +A directory speaks for a leaderboard score only when it passes every gate in +:func:`read_run` and :func:`row_rejection`. What it then states (the output +cap every logged request carried, a temperature every request shared, the +number of tasks scored) is written onto that score result only. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path +from typing import Any + +from every_eval_ever.helpers import raw_capture + + +@dataclass(frozen=True) +class RunDataset: + """One published run-output dataset, pinned to a commit.""" + + dataset_id: str + repo_id: str + revision: str + subdir: str + + @property + def url(self) -> str: + return f'https://huggingface.co/datasets/{self.repo_id}' + + +#: Leaderboard ``datasetId`` -> the dataset holding that set's run logs. Only +#: the public sets publish outputs; a directory fills its own set only. +RUN_DATASETS: dict[str, RunDataset] = { + 'v1_Public_Eval': RunDataset( + dataset_id='v1_Public_Eval', + repo_id='arcprize/arc_agi_v1_public_eval', + revision='3e9c9d1a8402aff82356815c106cecaea65cb7d9', + subdir='v1', + ), + 'v2_Public_Eval': RunDataset( + dataset_id='v2_Public_Eval', + repo_id='arcprize/arc_agi_v2_public_eval', + revision='026789c1c12a4c34580a32e84dcaf5630d7e8f31', + subdir='v2', + ), +} + +#: Output-cap spellings a logged request may carry. A run states exactly one, +#: under the same key and value, on every recorded request. +CAP_KEYS = ('max_tokens', 'max_completion_tokens', 'max_output_tokens') +RESULTS_FILE = 'results.json' +_ATTEMPT_KEY = re.compile(r'^attempt_\d+$') + + +class StatedFloat(float): + """A float parsed from JSON that keeps the text it was written as.""" + + text: str + + def __new__(cls, text: str) -> StatedFloat: + value = super().__new__(cls, text) + value.text = text + return value + + +def loads_stated(text: str) -> Any: + """Parse JSON keeping each float's written form (see :class:`StatedFloat`).""" + return json.loads(text, parse_float=StatedFloat) + + +def stated_decimal(value: float | int) -> Decimal: + """Return ``value`` as a Decimal at the precision it was written with. + + A :class:`StatedFloat` keeps its JSON text, so ``0.470`` stays three + decimals; any other number falls back to its shortest ``repr``. + """ + if isinstance(value, StatedFloat): + return Decimal(value.text) + if isinstance(value, int): + return Decimal(value) + return Decimal(repr(float(value))) + + +class RunRejected(Exception): + """A run directory failed a gate; ``reason`` names the gate.""" + + def __init__(self, reason: str, detail: str = '') -> None: + super().__init__(f'{reason}: {detail}' if detail else reason) + self.reason = reason + self.detail = detail + + +@dataclass(frozen=True) +class PublishedRun: + """One run as its logs state it, after the directory-level gates.""" + + directory: str + cap_key: str + cap_value: int + temperature: float | None + score: float + avg_cost_per_task: Any + latest_end: datetime + task_count: int + attempt_count: int + + +@dataclass(frozen=True) +class RunLogSource: + """A local checkout of one run dataset and, if known, its path history.""" + + dataset: RunDataset + root: Path + revision: str + history: dict[str, int] + + +@dataclass(frozen=True) +class JoinOutcome: + """What one score row's join produced.""" + + dataset_id: str + model_id: str + run: PublishedRun | None = None + rejection: str | None = None + detail: str = '' + request_log: str = '' + + +@dataclass +class RunLogs: + """Run-log sources keyed by leaderboard ``datasetId``, plus join outcomes.""" + + sources: dict[str, RunLogSource] + outcomes: list[JoinOutcome] = field(default_factory=list) + unjoined: list[tuple[str, str]] = field(default_factory=list) + _runs: dict[tuple[str, str], PublishedRun | RunRejected] = field( + default_factory=dict + ) + + def join( + self, dataset_id: str, row: dict[str, Any], retrieved: float + ) -> JoinOutcome | None: + """Join one chosen score row to its run directory. + + Returns ``None`` when the set publishes no logs or no directory is + named after the row's ``modelId``. + """ + source = self.sources.get(dataset_id) + model_id = row.get('modelId') + if source is None or not isinstance(model_id, str): + return None + directory = source.root / model_id + if ( + not model_id + or model_id.startswith('.') + or '/' in model_id + or not directory.is_dir() + ): + self.unjoined.append((dataset_id, model_id)) + return None + key = (dataset_id, model_id) + if key not in self._runs: + try: + self._runs[key] = read_run(directory, source.history) + except RunRejected as exc: + self._runs[key] = exc + read = self._runs[key] + request_log = f'{source.dataset.repo_id}@{source.revision}/{model_id}' + if isinstance(read, RunRejected): + outcome = JoinOutcome( + dataset_id, + model_id, + rejection=read.reason, + detail=read.detail, + request_log=request_log, + ) + else: + reason = row_rejection(read, row, retrieved) + outcome = JoinOutcome( + dataset_id, + model_id, + run=None if reason else read, + rejection=reason, + request_log=request_log, + ) + self.outcomes.append(outcome) + return outcome + + +def rounds_to(value: float, stated: float | int) -> bool: + """Whether ``value`` rounds to ``stated`` at the precision ``stated`` spells out.""" + target = stated_decimal(stated) + half = Decimal(1).scaleb(target.as_tuple().exponent) / 2 + return abs(Decimal(repr(float(value))) - target) <= half + + +def _sums_to(value: float, stated: float, terms: int) -> bool: + """Whether a sum of ``terms`` task scores equals one summed in another order.""" + spread = terms * sys.float_info.epsilon * max(abs(value), abs(stated), 1.0) + return abs(value - stated) <= spread + + +def parse_timestamp(value: Any) -> datetime | None: + """Parse an epoch number or ISO-8601 string to an aware UTC datetime.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return datetime.fromtimestamp(float(value), tz=timezone.utc) + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromtimestamp(float(value), tz=timezone.utc) + except ValueError: + pass + try: + parsed = datetime.fromisoformat(value.replace('Z', '+00:00')) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _attempt_cap(kwargs: dict[str, Any], where: str) -> tuple[str, int]: + stated = [key for key in CAP_KEYS if key in kwargs] + if not stated: + raise RunRejected('cap_absent', where) + if len(stated) > 1: + raise RunRejected('cap_ambiguous', f'{where}: {", ".join(stated)}') + value = kwargs[stated[0]] + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise RunRejected('cap_not_an_integer', f'{where}: {value!r}') + return stated[0], value + + +def read_history(repo_dir: Path, revision: str) -> dict[str, int]: + """Count the commits up to ``revision`` that touched each path. + + Reads commits and trees only, so it works in a blobless clone without + fetching file contents. + """ + log = _git( + '-C', + str(repo_dir), + 'log', + '--name-only', + '--no-renames', + '--pretty=format:', + revision, + '--', + ) + return dict(Counter(line for line in log.splitlines() if line)) + + +def read_run(directory: Path, history: dict[str, int]) -> PublishedRun: + """Reconstruct one run from its directory, or raise :class:`RunRejected`. + + Gates: no path the directory ever held was touched by a second commit + according to ``history``; every recorded attempt names this directory + as its ``test_id``, its file's task id, and its pair index; every recorded + request states one output cap under one key with one value; the task set, + each task's score and attempt count, the total attempts and the aggregate + all recompute from the task files to ``results.json``. + + A pair counts as solved when any attempt is correct, and a task scores the + fraction of its pairs solved. An attempt recorded as ``null`` rejects the + run: its request arguments, and so its cap, are not logged. + """ + results_path = directory / RESULTS_FILE + if not results_path.is_file(): + raise RunRejected('no_results_file') + try: + published = json.loads(results_path.read_text(encoding='utf-8')) + except ValueError as exc: + raise RunRejected('results_file_unreadable', str(exc)) from exc + task_results = ( + published.get('task_results') if isinstance(published, dict) else None + ) + if not isinstance(task_results, dict): + raise RunRejected('results_file_without_task_results') + + prefix = f'{directory.name}/' + rewritten = sorted( + path + for path, commits in history.items() + if path.startswith(prefix) and commits > 1 + ) + if rewritten: + raise RunRejected('run_rewritten', ', '.join(rewritten[:5])) + + task_paths = sorted( + path + for path in directory.glob('*.json') + if path.is_file() and path.name != RESULTS_FILE + ) + if not task_paths: + raise RunRejected('no_task_files') + + scores: dict[str, float] = {} + attempts_per_task: dict[str, int] = {} + cap: tuple[str, int] | None = None + temperatures: list[Any] = [] + latest_end: datetime | None = None + + for path in task_paths: + task_id = path.stem + try: + pairs = json.loads(path.read_text(encoding='utf-8')) + except ValueError as exc: + raise RunRejected( + 'task_file_unreadable', f'{path.name}: {exc}' + ) from exc + if not isinstance(pairs, list) or not pairs: + raise RunRejected('task_file_not_a_pair_list', path.name) + solved = [] + task_attempts = 0 + for index, pair in enumerate(pairs): + if not isinstance(pair, dict) or not pair: + raise RunRejected('task_file_not_a_pair_list', path.name) + correct = False + for key, attempt in pair.items(): + where = f'{path.name}: {key}' + if not _ATTEMPT_KEY.match(key): + raise RunRejected('attempt_key_unknown', where) + task_attempts += 1 + if not isinstance(attempt, dict): + raise RunRejected('attempt_not_recorded', where) + metadata = attempt.get('metadata') + if not isinstance(metadata, dict): + raise RunRejected('attempt_without_metadata', where) + if metadata.get('task_id') != task_id: + raise RunRejected( + 'task_id_mismatch', + f'{where}: {metadata.get("task_id")!r}', + ) + if metadata.get('test_id') != directory.name: + raise RunRejected( + 'test_id_mismatch', + f'{where}: {metadata.get("test_id")!r}', + ) + pair_index = metadata.get('pair_index') + if ( + isinstance(pair_index, bool) + or not isinstance(pair_index, int) + or pair_index != index + ): + raise RunRejected( + 'pair_index_mismatch', + f'{where}: {pair_index!r} at {index}', + ) + end = parse_timestamp(metadata.get('end_timestamp')) + if end is None: + raise RunRejected('attempt_without_end_time', where) + latest_end = end if latest_end is None else max(latest_end, end) + kwargs = metadata.get('kwargs') + if not isinstance(kwargs, dict): + raise RunRejected('attempt_without_kwargs', where) + attempt_cap = _attempt_cap(kwargs, where) + if cap is None: + cap = attempt_cap + elif attempt_cap[0] != cap[0]: + raise RunRejected( + 'cap_key_mixed', f'{cap[0]} vs {attempt_cap[0]}' + ) + elif attempt_cap[1] != cap[1]: + raise RunRejected( + 'cap_conflict', f'{cap[1]} vs {attempt_cap[1]}' + ) + temperatures.append(kwargs.get('temperature')) + if not isinstance(attempt.get('correct'), bool): + raise RunRejected('correct_not_recorded', where) + correct = correct or attempt['correct'] + solved.append(correct) + scores[task_id] = sum(solved) / len(solved) + attempts_per_task[task_id] = task_attempts + + if cap is None or latest_end is None: + raise RunRejected('no_recorded_attempts') + if set(scores) != set(task_results): + missing = len(set(task_results) - set(scores)) + extra = len(set(scores) - set(task_results)) + raise RunRejected( + 'task_set_mismatch', + f'{missing} not published as files, {extra} not in results.json', + ) + for task_id, score in sorted(scores.items()): + stated = task_results[task_id] + stated = stated if isinstance(stated, dict) else {} + stated_score = stated.get('score') + if not _is_number(stated_score) or not rounds_to(score, stated_score): + raise RunRejected( + 'task_score_mismatch', f'{task_id}: {stated_score!r}' + ) + stated_attempts = stated.get('attempts') + if ( + stated_attempts is not None + and stated_attempts != attempts_per_task[task_id] + ): + raise RunRejected( + 'task_attempts_mismatch', + f'{task_id}: {stated_attempts!r} vs {attempts_per_task[task_id]}', + ) + total_attempts = sum(attempts_per_task.values()) + for name, recomputed, terms in ( + ('total_tasks', len(scores), 0), + ('total_attempts', total_attempts, 0), + ('score', sum(scores.values()), len(scores)), + ): + stated = published.get(name) + matches = _is_number(stated) and ( + _sums_to(float(recomputed), float(stated), terms) + if terms + else float(recomputed) == float(stated) + ) + if not matches: + raise RunRejected( + 'run_score_mismatch', f'{name}: {stated!r} vs {recomputed}' + ) + + temperature = None + if ( + temperatures + and all(_is_number(t) for t in temperatures) + and len({float(t) for t in temperatures}) == 1 + ): + temperature = float(temperatures[0]) + + return PublishedRun( + directory=directory.name, + cap_key=cap[0], + cap_value=cap[1], + temperature=temperature, + score=sum(scores.values()) / len(scores), + avg_cost_per_task=published.get('avg_cost_per_task'), + latest_end=latest_end, + task_count=len(scores), + attempt_count=total_attempts, + ) + + +def row_rejection( + run: PublishedRun, row: dict[str, Any], retrieved: float +) -> str | None: + """Name the gate a leaderboard row fails against a run, or ``None``. + + The run's recomputed score must round to the row's ``score`` and its + ``avg_cost_per_task`` to the row's ``costPerTask``, each at the precision + the row spells out, and the run must have ended before retrieval. + """ + if not _is_number(row.get('score')): + return 'score_not_stated' + if not rounds_to(run.score, row['score']): + return 'score_mismatch' + if not _is_number(row.get('costPerTask')): + return 'cost_not_stated' + if not _is_number(run.avg_cost_per_task) or not rounds_to( + run.avg_cost_per_task, row['costPerTask'] + ): + return 'cost_mismatch' + if run.latest_end.timestamp() > retrieved: + return 'run_ended_after_retrieval' + return None + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _git_head(path: Path) -> str | None: + try: + completed = subprocess.run( + ['git', '-C', str(path), 'rev-parse', 'HEAD'], + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + return completed.stdout.strip() or None + + +def _is_git_root(path: Path) -> bool: + return (path / '.git').exists() + + +class RunLogsUnavailable(RuntimeError): + """Local run logs cannot be read under the same gates as a live run.""" + + +def load_history_file(path: Path, revision: str) -> dict[str, int]: + """Read a path-history file: ``{"revision": sha, "paths": {path: commits}}``. + + The file is what :func:`read_history` returns for ``revision``, saved as + JSON; its ``revision`` must equal the one the run logs are read at. + """ + try: + data = json.loads(Path(path).read_text(encoding='utf-8')) + except (OSError, ValueError) as exc: + raise RunLogsUnavailable(f'{path}: unreadable history file: {exc}') + if not isinstance(data, dict) or data.get('revision') != revision: + raise RunLogsUnavailable( + f'{path}: history is not for revision {revision}' + ) + paths = data.get('paths') + if not isinstance(paths, dict) or not all( + isinstance(k, str) and isinstance(v, int) for k, v in paths.items() + ): + raise RunLogsUnavailable(f'{path}: "paths" must map path to count') + return paths + + +def local_run_logs(runs_dir: Path, history_dir: Path | None = None) -> RunLogs: + """Use run logs already on disk under ``runs_dir/``. + + A subdirectory that is itself a git checkout supplies its own commit and + history. Otherwise the commit is taken to be the pinned one and its + history must come from ``history_dir``: a clone at ``/`` or a file + ``.json`` (see :func:`load_history_file`). A subdirectory with no + history raises :class:`RunLogsUnavailable`, so an offline run applies the + same gates as a live one. A missing subdirectory leaves that set unread. + """ + sources = {} + for dataset_id, dataset in RUN_DATASETS.items(): + root = Path(runs_dir) / dataset.subdir + if not root.is_dir(): + print( + f'ARC run logs: {root} not found; {dataset_id} not joined', + file=sys.stderr, + ) + continue + revision = dataset.revision + if _is_git_root(root): + revision = _git_head(root) or dataset.revision + history = read_history(root, revision) + elif history_dir is None: + raise RunLogsUnavailable( + f'{root} is not a git checkout, so its history is unknown; ' + 'pass --arc-history-dir with a clone or history file for ' + f'{dataset.subdir}' + ) + elif _is_git_root(Path(history_dir) / dataset.subdir): + history = read_history(Path(history_dir) / dataset.subdir, revision) + elif (Path(history_dir) / f'{dataset.subdir}.json').is_file(): + history = load_history_file( + Path(history_dir) / f'{dataset.subdir}.json', revision + ) + else: + raise RunLogsUnavailable( + f'{history_dir} holds neither a {dataset.subdir}/ clone nor ' + f'{dataset.subdir}.json' + ) + sources[dataset_id] = RunLogSource(dataset, root, revision, history) + return RunLogs(sources) + + +def _git(*args: str, cwd: Path | None = None, stdin: str | None = None) -> str: + env = {**os.environ, 'GIT_LFS_SKIP_SMUDGE': '1', 'GIT_TERMINAL_PROMPT': '0'} + try: + completed = subprocess.run( + ['git', *args], + cwd=cwd, + input=stdin, + capture_output=True, + text=True, + check=True, + env=env, + ) + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f'git {" ".join(args)} failed ({exc.returncode}): ' + f'{(exc.stderr or "").strip()}' + ) from exc + return completed.stdout + + +def fetch_run_logs( + model_ids: dict[str, set[str]], + work_dir: Path, + datasets: dict[str, RunDataset] | None = None, +) -> RunLogs: + """Clone each run dataset and check out its wanted directories at the + pinned commit. + + One full ``--no-checkout`` clone per dataset (a single pack, so a pinned + commit that is no longer HEAD still checks out), a cone-mode sparse + checkout of ``model_ids[datasetId]`` that exist at the commit, then a + detached checkout of it. The same clone's history backs the + rewritten-path gate. + """ + sources = {} + for dataset_id, dataset in (datasets or RUN_DATASETS).items(): + dest = Path(work_dir) / dataset.subdir + _git('clone', '--quiet', '--no-checkout', dataset.url, str(dest)) + present = set( + _git( + 'ls-tree', '-d', '--name-only', dataset.revision, cwd=dest + ).splitlines() + ) + wanted = sorted(model_ids.get(dataset_id, set()) & present) + _git( + 'sparse-checkout', + 'set', + '--cone', + '--stdin', + cwd=dest, + stdin=''.join(f'{name}\n' for name in wanted), + ) + _git('checkout', '--quiet', '--detach', dataset.revision, cwd=dest) + raw_capture.record_git_checkout( + dataset.url, + dest, + ref=dataset.revision, + label=f'{dataset.repo_id} run logs ({len(wanted)} directories)', + ) + sources[dataset_id] = RunLogSource( + dataset, + dest, + dataset.revision, + read_history(dest, dataset.revision), + ) + return RunLogs(sources) diff --git a/every_eval_ever/adapters/catalog.py b/every_eval_ever/adapters/catalog.py index e0bcefd7a..a6a90ff16 100644 --- a/every_eval_ever/adapters/catalog.py +++ b/every_eval_ever/adapters/catalog.py @@ -250,7 +250,11 @@ def _helm(key: str, leaderboard: str, weekday: int) -> AdapterSpec: notes=( 'Repointed 2026-08-12 to the JSON files behind ' 'arcprize.org/leaderboard; the old ' - '/media/data/leaderboard/evaluations.json endpoint is gone.' + '/media/data/leaderboard/evaluations.json endpoint is gone. ' + 'Also clones the two arcprize public-eval log datasets (full ' + 'no-checkout clones, about 130 MB of packs, then a sparse ' + 'checkout at the pinned commits); a live run measured 24 s, so ' + 'the default timeout stands.' ), ), AdapterSpec( @@ -315,6 +319,7 @@ def _helm(key: str, leaderboard: str, weekday: int) -> AdapterSpec: weekday=4, timeout_minutes=45, runnable=False, + with_packages=('pyarrow',), unrunnable_reason=( 'the Open LLM Leaderboard is no longer maintained upstream ' '(space discussion 1135); the archive is frozen, so a scheduled ' @@ -322,7 +327,11 @@ def _helm(key: str, leaderboard: str, weekday: int) -> AdapterSpec: ), notes=( 'The adapter still works for a one-off manual conversion of the ' - 'frozen archive (around 4,576 models).' + 'frozen archive (4,576 model and precision rows): it reads the ' + 'leaderboard table from the open-llm-leaderboard/contents ' + 'parquet and the harness dumps from open-llm-leaderboard/results, ' + 'both at pinned revisions (pyarrow, from with_packages, or `uv run --with pyarrow`). The Space API is ' + 'no longer served and is only read with --source-api.' ), ), AdapterSpec( diff --git a/every_eval_ever/adapters/hfopenllm_v2/adapter.py b/every_eval_ever/adapters/hfopenllm_v2/adapter.py index 223769451..493a9fe95 100644 --- a/every_eval_ever/adapters/hfopenllm_v2/adapter.py +++ b/every_eval_ever/adapters/hfopenllm_v2/adapter.py @@ -1,29 +1,65 @@ """ Script to convert HuggingFace Open LLM Leaderboard v2 data to the EvalEval schema format. -Data source: -- HF Open LLM Leaderboard v2 API: https://open-llm-leaderboard-open-llm-leaderboard.hf.space/api/leaderboard/formatted +Data sources, both pinned Hugging Face datasets: +- ``open-llm-leaderboard/contents`` at ``CONTENTS_REVISION``: the leaderboard + table (scores, one row per model and precision). Reading it needs pyarrow. +- ``open-llm-leaderboard/results`` at ``RESULTS_REVISION``: the harness result + dumps (harness settings, sample counts, split) + +``--source-api`` reads the table from the leaderboard Space API instead, which +is unpinned and no longer served. + +The default results fetch requires Git and transfers the pinned revision in one +shallow fetch. Only result JSON files are checked out, into a temporary directory. Usage: uv run python -m every_eval_ever.adapters.hfopenllm_v2.adapter uv run python -m every_eval_ever.adapters.hfopenllm_v2.adapter \ - --output-dir /tmp/smoke/data/hfopenllm_v2 + --output-dir /tmp/smoke/data/hfopenllm_v2 \ + --contents-parquet /data/train-00000-of-00001.parquet \ + --results-dir """ import argparse +import json +import os +import re +import subprocess +import tempfile import time +from collections import Counter +from contextlib import nullcontext from pathlib import Path from typing import Any, Dict, List +from every_eval_ever.adapters.hfopenllm_v2.contents import ( + CONTENTS_REPO, + CONTENTS_REVISION, + download_contents, + load_contents, +) +from every_eval_ever.adapters.hfopenllm_v2.dump_index import ( + RESULTS_PATTERN, + RESULTS_REPO, + RESULTS_REVISION, + DumpIndex, + HarnessGroup, + ModelEvidence, + build_dump_index, +) from every_eval_ever.eval_types import ( EvalLibrary, EvaluationLog, EvaluationResult, EvaluatorRelationship, + GenerationArgs, + GenerationConfig, MetricConfig, ScoreDetails, ScoreType, SourceDataHf, + Uncertainty, ) from every_eval_ever.helpers import ( SCHEMA_VERSION, @@ -34,6 +70,7 @@ fetch_json, make_model_info, make_source_metadata, + raw_capture, save_evaluation_logs, save_failure_report, ) @@ -41,6 +78,7 @@ # Source URL SOURCE_URL = 'https://open-llm-leaderboard-open-llm-leaderboard.hf.space/api/leaderboard/formatted' OUTPUT_DIR = 'data/hfopenllm_v2' +RESULTS_URL = f'https://huggingface.co/datasets/{RESULTS_REPO}' # Evaluation name mapping from API keys to display names EVALUATION_MAPPING = { @@ -98,23 +136,102 @@ } +# eval_key -> (metric_id, metric_name, metric_kind, metric_unit) +METRIC_MAPPING = { + 'ifeval': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'bbh': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'math': ('exact_match', 'Exact Match', 'exact_match', 'proportion'), + 'gpqa': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'musr': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'mmlu_pro': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), +} + +HARNESS_FORK = 'https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess' + + +def _slug(name: str) -> str: + return '_'.join( + ''.join(c if c.isalnum() else ' ' for c in name.lower()).split() + ) + + +def _source_data( + base: SourceDataHf, group: HarnessGroup | None +) -> SourceDataHf: + """``base`` with the split and counts the dumps agree on. + + The dataset the harness loaded is recorded as ``harness_dataset_path`` + where it differs from ``base.hf_repo``. + """ + if group is None: + return base + update: Dict[str, Any] = { + 'hf_split': group.hf_split, + 'samples_number': group.samples_number, + } + details = {} + if group.dataset_path is not None and group.dataset_path != base.hf_repo: + details['harness_dataset_path'] = group.dataset_path + if group.dataset_size is not None: + details['dataset_size'] = str(group.dataset_size) + if details: + update['additional_details'] = details + return base.model_copy(update=update) + + +def _generation_config(group: HarnessGroup | None) -> GenerationConfig | None: + """The harness settings the dumps agree on, if any.""" + if group is None: + return None + details = { + name: value if isinstance(value, str) else json.dumps(value) + for name, value in ( + ('output_type', group.output_type), + ('num_fewshot', group.num_fewshot), + ('do_sample', group.do_sample), + ) + if value is not None + } + args = None + if group.temperature is not None or group.max_tokens is not None: + args = GenerationArgs( + temperature=group.temperature, max_tokens=group.max_tokens + ) + if args is None and not details: + return None + return GenerationConfig( + generation_args=args, additional_details=details or None + ) + + def convert_model( model_data: Dict[str, Any], retrieved_timestamp: str, *, source_ref: str | None = None, failures: list[SourceRecordFailure] | None = None, + evidence: ModelEvidence | None = None, ) -> EvaluationLog: """Convert one model, optionally retaining unusable metric provenance. The strict public behavior is unchanged when ``failures`` is omitted: any unusable metric rejects the model. Batch conversion supplies a failure list so valid metrics from the same model can still be published. + ``evidence`` is the dump evidence for this model and precision; without + it the harness fields are left out. """ model_id = model_data['model']['name'] if '/' not in model_id: raise ValueError(f"Expected 'org/model' format, got: {model_id}") developer, model_name = model_id.split('/', 1) + precision = model_data['model'].get('precision') + if not isinstance(precision, str) or not precision.strip(): + raise ValueError(f'precision is missing; got {precision!r}') + evaluation_id = ( + f'hfopenllm_v2/{developer}_{model_name}/{precision}/' + f'{retrieved_timestamp}' + ) + groups = evidence.groups if evidence is not None else {} # Build evaluation results eval_results: List[EvaluationResult] = [] @@ -129,26 +246,41 @@ def convert_model( display_name, f'Accuracy on {display_name}' ) source_data = SOURCE_DATA_MAPPING.get(eval_key) - if source_data is None: + if source_data is None or eval_key not in METRIC_MAPPING: raise ValueError( f"unknown evaluation key; add '{eval_key}' to " - 'SOURCE_DATA_MAPPING' + 'SOURCE_DATA_MAPPING and METRIC_MAPPING' ) + metric_id, metric_name, metric_kind, metric_unit = METRIC_MAPPING[ + eval_key + ] + group = groups.get(eval_key) eval_results.append( EvaluationResult( + evaluation_result_id=( + f'{evaluation_id}#{_slug(display_name)}#{metric_id}' + ), evaluation_name=display_name, - source_data=source_data, + source_data=_source_data(source_data, group), metric_config=MetricConfig( evaluation_description=description, lower_is_better=False, score_type=ScoreType.continuous, min_score=0.0, max_score=1.0, + metric_id=metric_id, + metric_name=metric_name, + metric_kind=metric_kind, + metric_unit=metric_unit, ), score_details=ScoreDetails( score=round(float(eval_data['value']), 4), + uncertainty=Uncertainty(num_samples=group.samples_number) + if group is not None and group.samples_number is not None + else None, ), + generation_config=_generation_config(group), ) ) except Exception as exc: @@ -193,10 +325,14 @@ def convert_model( additional_details=additional_details if additional_details else None, ) - # Build evaluation ID - evaluation_id = ( - f'hfopenllm_v2/{developer}_{model_name}/{retrieved_timestamp}' - ) + library_details = {'fork': HARNESS_FORK} + if evidence is not None: + if evidence.git_hash is not None: + library_details['git_hash'] = evidence.git_hash + if evidence.transformers_version is not None: + library_details['transformers_version'] = ( + evidence.transformers_version + ) return EvaluationLog( schema_version=SCHEMA_VERSION, @@ -210,20 +346,31 @@ def convert_model( eval_library=EvalLibrary( name='lm-evaluation-harness', version='0.4.0', - additional_details={ - 'fork': 'https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess' - }, + additional_details=library_details, ), model_info=model_info, evaluation_results=eval_results, ) +def _lookup( + dump_index: DumpIndex | None, model_data: Dict[str, Any] +) -> ModelEvidence | None: + if dump_index is None: + return None + model = model_data.get('model') or {} + return dump_index.lookup(model.get('name'), model.get('precision')) + + def convert_models( models_data: List[Dict[str, Any]], retrieved_timestamp: str | None = None, + dump_index: DumpIndex | None = None, ) -> SourceConversionResult[EvaluationLogOutput]: - """Convert all usable models and preserve every rejected source row.""" + """Convert all usable models and preserve every rejected source row. + + Each row is joined to ``dump_index`` on its exact (model, precision). + """ timestamp = retrieved_timestamp or str(time.time()) outputs = [] failures: list[SourceRecordFailure] = [] @@ -242,6 +389,7 @@ def convert_models( timestamp, source_ref=source_ref, failures=failures, + evidence=_lookup(dump_index, model_data), ) outputs.append( EvaluationLogOutput( @@ -273,11 +421,61 @@ def convert_models( ) +def harness_gaps( + models_data: List[Dict[str, Any]], dump_index: DumpIndex | None +) -> Dict[str, Any]: + """Every harness field left null, with the reason, per model row.""" + unfilled = [] + for index, model_data in enumerate(models_data): + model = model_data.get('model') or {} + evidence = _lookup(dump_index, model_data) + for eval_key in model_data.get('evaluations') or {}: + if evidence is None: + gaps = {'all': 'no_dump'} + elif eval_key not in evidence.groups: + gaps = {'all': 'task_group_not_run'} + else: + gaps = evidence.groups[eval_key].gaps + for field_name, reason in sorted(gaps.items()): + unfilled.append( + { + 'source_ref': f'model row {index}', + 'model': model.get('name'), + 'precision': model.get('precision'), + 'evaluation_key': eval_key, + 'field': field_name, + 'reason': reason, + } + ) + counts = Counter( + f'{entry["evaluation_key"]}.{entry["field"]}: {entry["reason"]}' + for entry in unfilled + ) + return { + 'results_dataset': RESULTS_REPO, + 'results_revision': ( + dump_index.revision if dump_index else RESULTS_REVISION + ), + 'results_dir_unpinned': bool(dump_index and dump_index.unpinned), + 'dumps_read': dump_index.files_read if dump_index else 0, + 'dumps_rejected': dump_index.rejected if dump_index else [], + 'unfilled_counts': dict(sorted(counts.items())), + 'unfilled': unfilled, + } + + def process_models( - models_data: List[Dict[str, Any]], output_dir: str | Path = OUTPUT_DIR + models_data: List[Dict[str, Any]], + output_dir: str | Path = OUTPUT_DIR, + dump_index: DumpIndex | None = None, + leaderboard_input: Dict[str, Any] | None = None, ) -> int: - """Save valid models, report rejected rows, and signal incompleteness.""" - result = convert_models(models_data) + """Save valid models, report rejected rows and unfilled harness fields. + + The report is written whenever a row failed or a harness field stayed + null; only failed rows signal incompleteness. + """ + result = convert_models(models_data, dump_index=dump_index) outputs = [ EvaluationLogOutput( eval_log=record.eval_log, @@ -290,11 +488,21 @@ def process_models( paths = save_evaluation_logs(outputs) for path in paths: print(f'Saved: {path}') - if result.failures: + gaps = harness_gaps(models_data, dump_index) + if result.failures or gaps['unfilled'] or gaps['dumps_rejected']: report_path = save_failure_report( result, default_failure_report_path(output_dir), ) + report = json.loads(report_path.read_text(encoding='utf-8')) + report['harness_evidence'] = gaps + if leaderboard_input is not None: + report['leaderboard_input'] = leaderboard_input + report_path.write_text( + json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + + '\n', + encoding='utf-8', + ) print(f'Failure report: {report_path}') result.raise_if_incomplete() return len(paths) @@ -312,17 +520,254 @@ def parse_args(argv: List[str] | None = None) -> argparse.Namespace: default=Path(OUTPUT_DIR), help=f'Datastore collection directory (default: {OUTPUT_DIR}).', ) + parser.add_argument( + '--results-dir', + type=Path, + default=None, + help=( + f'Local Git checkout or Hub snapshot of {RESULTS_REPO}, at ' + f'{RESULTS_REVISION}. Git checkouts use HEAD; Hub snapshots use ' + 'the commit sha in the directory name. Default: one shallow ' + 'Git fetch into a temporary directory.' + ), + ) + parser.add_argument( + '--allow-unpinned-results-dir', + action='store_true', + help=( + '(Debug) Accept a --results-dir at another commit, or with no ' + 'commit in its path; the revision is recorded and reported as is.' + ), + ) + table = parser.add_mutually_exclusive_group() + table.add_argument( + '--contents-parquet', + type=Path, + default=None, + help=( + f'Local copy of {CONTENTS_REPO} data/train-00000-of-00001.parquet ' + 'inside a Hub snapshot directory named after its commit sha, ' + f'which must be {CONTENTS_REVISION}. Default: download it once ' + 'from the Hub.' + ), + ) + table.add_argument( + '--source-api', + action='store_true', + help=( + '(Legacy) Read the table from the leaderboard Space API, which ' + 'is unpinned and no longer served, instead of the contents ' + 'dataset.' + ), + ) + parser.add_argument( + '--allow-unpinned-contents-parquet', + action='store_true', + help=( + '(Debug) Accept a --contents-parquet at another commit, or with ' + 'no commit in its path; the revision is recorded and reported ' + 'as is.' + ), + ) return parser.parse_args(argv) +_SHA = re.compile(r'[0-9a-f]{40}') + + +def _snapshot_revision( + snapshot_dir: Path, + *, + option: str, + repo: str, + pinned: str, + allow_unpinned: bool, +) -> tuple[str, bool]: + name = snapshot_dir.name + sha = name if _SHA.fullmatch(name) else None + if sha == pinned: + return sha, False + if not allow_unpinned: + found = sha or f'no commit sha (directory {name!r})' + raise ValueError( + f'{option} is at {found}, but this adapter is pinned to ' + f'{repo}@{pinned}. Pass a snapshot at that revision, or ' + f'--allow-unpinned-{option[2:]}.' + ) + return sha or 'local', True + + +def results_dir_revision( + results_dir: str | Path, *, allow_unpinned: bool = False +) -> tuple[str, bool]: + """The commit a local results snapshot is at, and whether it is unpinned. + + Git checkouts use their actual HEAD. For existing Hub snapshots the commit + is the directory's own name, as in ``snapshots/``. Other revisions + require ``allow_unpinned`` and are reported without claiming the pin. + """ + root = Path(results_dir).resolve() + if (root / '.git').exists(): + revision = _git('rev-parse', 'HEAD^{commit}', cwd=root) + if revision != RESULTS_REVISION and not allow_unpinned: + raise ValueError( + f'--results-dir is at {revision}, but this adapter is pinned ' + f'to {RESULTS_REPO}@{RESULTS_REVISION}. Pass a checkout at ' + 'that revision, or --allow-unpinned-results-dir.' + ) + return revision, revision != RESULTS_REVISION + return _snapshot_revision( + root, + option='--results-dir', + repo=RESULTS_REPO, + pinned=RESULTS_REVISION, + allow_unpinned=allow_unpinned, + ) + + +def contents_parquet_revision( + parquet: str | Path, *, allow_unpinned: bool = False +) -> tuple[str, bool]: + """The commit a local contents parquet is at, and whether it is unpinned. + + The commit is the name of the snapshot directory holding ``data/``, as + in the Hub cache layout ``snapshots//data/``; the file itself + may be a symlink into the cache's blobs. Otherwise as + ``results_dir_revision``. + """ + return _snapshot_revision( + Path(parquet).absolute().parent.parent, + option='--contents-parquet', + repo=CONTENTS_REPO, + pinned=CONTENTS_REVISION, + allow_unpinned=allow_unpinned, + ) + + +def _git(*args: str, cwd: Path | None = None) -> str: + try: + result = subprocess.run( + ['git', *args], + cwd=cwd, + env={ + **os.environ, + 'GIT_LFS_SKIP_SMUDGE': '1', + 'GIT_TERMINAL_PROMPT': '0', + }, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise RuntimeError( + 'Git is required to fetch HF leaderboard results' + ) from exc + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f'git {args[0]} failed: {exc.stderr.strip()}' + ) from exc + return result.stdout.strip() + + +def download_results(destination: str | Path) -> Path: + """Fetch the pinned Git revision once into an empty, caller-owned directory.""" + root = Path(destination).resolve() + if root.exists() and any(root.iterdir()): + raise ValueError(f'Results destination must be empty: {root}') + root.mkdir(parents=True, exist_ok=True) + _git('init', cwd=root) + _git('fetch', '--depth', '1', RESULTS_URL, RESULTS_REVISION, cwd=root) + fetched = _git('rev-parse', 'FETCH_HEAD^{commit}', cwd=root) + if fetched != RESULTS_REVISION: + raise RuntimeError( + f'Fetched results commit {fetched}, expected {RESULTS_REVISION}' + ) + _git('sparse-checkout', 'set', '--no-cone', RESULTS_PATTERN, cwd=root) + _git('checkout', '--detach', RESULTS_REVISION, cwd=root) + checked_out = _git('rev-parse', 'HEAD^{commit}', cwd=root) + if checked_out != RESULTS_REVISION: + raise RuntimeError( + f'Checked out results commit {checked_out}, expected {RESULTS_REVISION}' + ) + raw_capture.record_git_checkout( + RESULTS_URL, + root, + ref=RESULTS_REVISION, + label='Open LLM Leaderboard v2 harness result dumps', + ) + return root + + def main(argv: List[str] | None = None) -> int: args = parse_args(argv) + revision, unpinned = RESULTS_REVISION, False + contents_revision, contents_unpinned = CONTENTS_REVISION, False + try: + if args.results_dir is not None: + revision, unpinned = results_dir_revision( + args.results_dir, + allow_unpinned=args.allow_unpinned_results_dir, + ) + if args.contents_parquet is not None: + contents_revision, contents_unpinned = contents_parquet_revision( + args.contents_parquet, + allow_unpinned=args.allow_unpinned_contents_parquet, + ) + except ValueError as exc: + raise SystemExit(str(exc)) from exc - print(f'Fetching data from {SOURCE_URL}...') - all_models = fetch_json(SOURCE_URL) + if args.source_api: + print(f'Fetching data from {SOURCE_URL}...') + all_models = fetch_json(SOURCE_URL) + leaderboard_input = {'source_url': SOURCE_URL} + else: + raw_capture.record_hf_dataset( + CONTENTS_REPO, + revision=contents_revision, + label='Open LLM Leaderboard v2 table', + ) + parquet = args.contents_parquet or download_contents() + print(f'Reading {parquet} ({contents_revision})...') + all_models = load_contents(parquet) + leaderboard_input = { + 'dataset': CONTENTS_REPO, + 'revision': contents_revision, + 'unpinned': contents_unpinned, + } + + if args.results_dir is not None: + if (args.results_dir / '.git').exists(): + raw_capture.record_git_checkout( + RESULTS_URL, + args.results_dir, + ref=revision, + label='Open LLM Leaderboard v2 harness result dumps', + ) + else: + raw_capture.record_hf_dataset( + RESULTS_REPO, + revision=revision, + label='Open LLM Leaderboard v2 harness result dumps', + ) + workspace = ( + nullcontext(args.results_dir) + if args.results_dir is not None + else tempfile.TemporaryDirectory(prefix='eee-hfopenllm-v2-') + ) + with workspace as root: + results_dir = args.results_dir or download_results(root) + print(f'Indexing harness dumps in {results_dir} ({revision})...') + dump_index = build_dump_index(results_dir) + dump_index.revision, dump_index.unpinned = revision, unpinned + print( + f'Read {dump_index.files_read} dumps for {len(dump_index.entries)} ' + f'(model, precision) pairs; rejected {len(dump_index.rejected)}.' + ) print(f'Processing {len(all_models)} models...') - count = process_models(all_models, args.output_dir) + count = process_models( + all_models, args.output_dir, dump_index, leaderboard_input + ) print(f'Done! Processed {count} models.') return count diff --git a/every_eval_ever/adapters/hfopenllm_v2/contents.py b/every_eval_ever/adapters/hfopenllm_v2/contents.py new file mode 100644 index 000000000..36f2dba75 --- /dev/null +++ b/every_eval_ever/adapters/hfopenllm_v2/contents.py @@ -0,0 +1,75 @@ +"""The Open LLM Leaderboard v2 table, read from ``open-llm-leaderboard/contents``. + +The leaderboard publishes its final table as one parquet file. Each row is +one (model, precision) evaluation. ``load_contents`` turns the rows into the +shape the leaderboard's Space API served, which is what the adapter converts. +""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +CONTENTS_REPO = 'open-llm-leaderboard/contents' +CONTENTS_REVISION = '9c09a7cae43334062a82cb164f2ef255013dafa2' +CONTENTS_FILE = 'data/train-00000-of-00001.parquet' + +# eval_key -> (score column on the 0-1 scale, evaluation name) +SCORE_COLUMNS = { + 'ifeval': ('IFEval Raw', 'IFEval'), + 'bbh': ('BBH Raw', 'BBH'), + 'math': ('MATH Lvl 5 Raw', 'MATH Level 5'), + 'gpqa': ('GPQA Raw', 'GPQA'), + 'musr': ('MUSR Raw', 'MUSR'), + 'mmlu_pro': ('MMLU-PRO Raw', 'MMLU-PRO'), +} + + +def _stated(value: Any) -> Any: + if isinstance(value, float) and not math.isfinite(value): + return None + return value + + +def contents_row(row: dict[str, Any]) -> dict[str, Any]: + """One parquet row in the Space API's row shape.""" + model = { + 'name': row.get('fullname'), + 'precision': row.get('Precision'), + 'architecture': row.get('Architecture'), + } + metadata = {} + params = _stated(row.get('#Params (B)')) + if params is not None: + metadata['params_billions'] = params + return { + 'model': model, + 'metadata': metadata, + 'evaluations': { + key: {'name': name, 'value': _stated(row.get(column))} + for key, (column, name) in SCORE_COLUMNS.items() + }, + } + + +def load_contents(parquet_path: str | Path) -> list[dict[str, Any]]: + """Every row of the contents parquet, in the Space API's row shape.""" + import pyarrow.parquet as pq + + table = pq.read_table(parquet_path) + return [contents_row(row) for row in table.to_pylist()] + + +def download_contents() -> Path: + """Fetch the pinned contents parquet in one Hub download.""" + from huggingface_hub import hf_hub_download + + return Path( + hf_hub_download( + repo_id=CONTENTS_REPO, + filename=CONTENTS_FILE, + repo_type='dataset', + revision=CONTENTS_REVISION, + ) + ) diff --git a/every_eval_ever/adapters/hfopenllm_v2/dump_index.py b/every_eval_ever/adapters/hfopenllm_v2/dump_index.py new file mode 100644 index 000000000..309f2da38 --- /dev/null +++ b/every_eval_ever/adapters/hfopenllm_v2/dump_index.py @@ -0,0 +1,351 @@ +"""Index of the Open LLM Leaderboard v2 harness result dumps. + +The leaderboard publishes the lm-evaluation-harness ``results_*.json`` dump of +every run in ``open-llm-leaderboard/results``. The index states, per +``(model, precision)`` and per leaderboard benchmark, the harness settings and +sample counts every dump for that pair agrees on. A field the dumps disagree +on, or do not state, is ``None`` with a reason in ``gaps``. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +RESULTS_REPO = 'open-llm-leaderboard/results' +RESULTS_REVISION = 'aa81ecc38fdc5708254b833923368970efdf5ef5' +RESULTS_PATTERN = '*/**/results_*.json' + +DTYPE_PRECISION = { + 'torch.bfloat16': 'bfloat16', + 'torch.float16': 'float16', +} + + +@dataclass(frozen=True) +class TaskGroup: + """The harness tasks that make up one leaderboard benchmark.""" + + prefix: str + subtasks: int + generative: bool = False + suffix: str = '' + + def tasks_of(self, configs: dict[str, Any]) -> dict[str, Any]: + """The group's tasks among a dump's ``configs``.""" + exact = not self.prefix.endswith('_') + return { + task: config + for task, config in configs.items() + if (task == self.prefix if exact else task.startswith(self.prefix)) + and task.endswith(self.suffix) + } + + +# Keyed on the leaderboard API's evaluation keys. +TASK_GROUPS: dict[str, TaskGroup] = { + 'ifeval': TaskGroup('leaderboard_ifeval', 1, generative=True), + 'bbh': TaskGroup('leaderboard_bbh_', 24), + 'math': TaskGroup('leaderboard_math_', 7, generative=True, suffix='_hard'), + 'gpqa': TaskGroup('leaderboard_gpqa_', 3), + 'musr': TaskGroup('leaderboard_musr_', 3), + 'mmlu_pro': TaskGroup('leaderboard_mmlu_pro', 1), +} + +CONFIG_FIELDS = ('output_type', 'num_fewshot') +DECODING_FIELDS = ('temperature', 'max_tokens', 'do_sample') + +_IDENTITY_ARGS = ('pretrained', 'peft', 'delta') + + +@dataclass +class HarnessGroup: + """What every dump of one (model, precision) states for one benchmark. + + The harness settings (output type, few-shot count, decoding) resolve as + one: a conflict on any of them leaves all of them null. + """ + + output_type: str | None = None + num_fewshot: int | None = None + temperature: float | None = None + max_tokens: int | None = None + do_sample: bool | None = None + samples_number: int | None = None + dataset_size: int | None = None + dataset_path: str | None = None + hf_split: str | None = None + gaps: dict[str, str] = field(default_factory=dict) + + +@dataclass +class ModelEvidence: + """Dump evidence for one (model, precision).""" + + dumps: list[str] + groups: dict[str, HarnessGroup] + git_hash: str | None = None + transformers_version: str | None = None + + +@dataclass +class DumpIndex: + """All dumps of a results snapshot, keyed on ``(model, precision)``.""" + + entries: dict[tuple[str, str], ModelEvidence] + files_read: int = 0 + rejected: list[dict[str, Any]] = field(default_factory=list) + revision: str = RESULTS_REVISION + unpinned: bool = False + + def lookup(self, model: str, precision: str | None) -> ModelEvidence | None: + """The evidence for an exact (model, precision), if any dump names it.""" + if precision is None: + return None + return self.entries.get((model, precision)) + + +def _launched_models(model_args: Any) -> dict[str, str]: + if not isinstance(model_args, str): + return {} + launched = {} + for part in model_args.split(','): + key, sep, value = part.partition('=') + if sep and key in _IDENTITY_ARGS: + launched[key] = value + return launched + + +def dump_identity(dump: dict[str, Any]) -> tuple[str | None, str | None]: + """The model a dump is about, or ``None`` and the reason it is rejected. + + Identity is the dump's own ``model_name``, which must equal the weights + the run loaded: ``peft=`` or ``delta=`` when given, else ``pretrained=``. + The directory a dump sits in is never used. + """ + name = dump.get('model_name') + if not isinstance(name, str) or not name: + return None, 'no_model_name' + launched = _launched_models((dump.get('config') or {}).get('model_args')) + if 'pretrained' not in launched: + return None, 'no_pretrained' + if 'peft' in launched and 'delta' in launched: + return None, 'ambiguous_adapter_args' + expected = launched.get( + 'peft', launched.get('delta', launched['pretrained']) + ) + if name != expected: + return None, 'identity_contradicts_model_args' + return name, None + + +def _unanimous(values: list[Any]) -> tuple[bool, Any]: + distinct = {json.dumps(v, sort_keys=True) for v in values} + if len(distinct) != 1: + return False, None + return True, values[0] + + +def _dump_statement( + group: TaskGroup, dump: dict[str, Any] +) -> dict[str, Any] | None: + """One dump's statement for one group, or ``None`` if it did not run it. + + Each field maps to ``(value, None)`` or ``(None, reason)``. + """ + tasks = group.tasks_of(dump.get('configs') or {}) + if not tasks: + return None + if len(tasks) != group.subtasks: + names = CONFIG_FIELDS + (DECODING_FIELDS if group.generative else ()) + return { + name: (None, 'incomplete_task_group') + for name in names + ('counts', 'dataset_path', 'hf_split') + } + + statement: dict[str, tuple[Any, str | None]] = {} + + def config_of(task_config: dict[str, Any]) -> dict[str, Any]: + kwargs = task_config.get('generation_kwargs') or {} + config = { + 'output_type': task_config.get('output_type'), + 'num_fewshot': task_config.get('num_fewshot'), + } + if group.generative: + config['temperature'] = kwargs.get('temperature') + config['max_tokens'] = kwargs.get('max_gen_toks') + config['do_sample'] = kwargs.get('do_sample') + return config + + agreed, config = _unanimous( + [config_of(c) for _, c in sorted(tasks.items())] + ) + for name in CONFIG_FIELDS + (DECODING_FIELDS if group.generative else ()): + if not agreed: + statement[name] = (None, 'subtask_conflict') + elif config[name] is None: + statement[name] = (None, 'not_stated') + else: + statement[name] = (config[name], None) + + n_samples = dump.get('n-samples') + counts: dict[str, tuple[int, int]] = {} + count_gap = None + if not isinstance(n_samples, dict): + count_gap = 'n_samples_absent' + else: + for task in sorted(tasks): + entry = n_samples.get(task) + entry = entry if isinstance(entry, dict) else {} + original, effective = entry.get('original'), entry.get('effective') + if not all( + isinstance(v, int) and not isinstance(v, bool) and v >= 1 + for v in (original, effective) + ): + count_gap = 'sample_count_absent' + break + counts[task] = (original, effective) + if count_gap: + statement['counts'] = (None, count_gap) + else: + statement['counts'] = (counts, None) + + paths = {c.get('dataset_path') for c in tasks.values()} + path = next(iter(paths)) + if len(paths) != 1: + statement['dataset_path'] = (None, 'subtask_conflict') + elif not isinstance(path, str) or not path: + statement['dataset_path'] = (None, 'not_stated') + else: + statement['dataset_path'] = (path, None) + + splits = {c.get('test_split') for c in tasks.values()} + split = next(iter(splits)) + if any(not isinstance(s, str) or not s for s in splits): + statement['hf_split'] = (None, 'not_stated') + elif len(splits) != 1: + statement['hf_split'] = (None, 'subtask_conflict') + else: + statement['hf_split'] = (split, None) + return statement + + +def _resolve_group(statements: list[dict[str, Any]]) -> HarnessGroup: + resolved = HarnessGroup() + names = ( + CONFIG_FIELDS + DECODING_FIELDS + ('counts', 'dataset_path', 'hf_split') + ) + for name in names: + stated = [s[name] for s in statements if name in s] + if not stated: + continue + reasons = {reason for _, reason in stated} + agreed, value = _unanimous([v for v, _ in stated]) + if reasons == {None} and agreed: + gap = None + elif len(reasons) == 1: + gap = reasons.pop() or 'cross_dump_conflict' + else: + gap = 'cross_dump_conflict' + if name == 'counts': + if gap: + resolved.gaps['samples_number'] = gap + continue + original = sum(o for o, _ in value.values()) + effective = sum(e for _, e in value.values()) + resolved.samples_number = effective + if effective != original: + resolved.dataset_size = original + continue + if gap: + resolved.gaps[name] = gap + else: + setattr(resolved, name, value) + config_names = CONFIG_FIELDS + DECODING_FIELDS + conflict = next( + ( + resolved.gaps[name] + for name in config_names + if resolved.gaps.get(name, '').endswith('conflict') + ), + None, + ) + if conflict: + for name in config_names: + if getattr(resolved, name) is not None or name in resolved.gaps: + setattr(resolved, name, None) + resolved.gaps[name] = conflict + return resolved + + +def build_dump_index(results_dir: str | Path) -> DumpIndex: + """Read every ``results_*.json`` under ``results_dir`` into a ``DumpIndex``.""" + root = Path(results_dir) + index = DumpIndex(entries={}) + by_key: dict[tuple[str, str], list[tuple[str, dict[str, Any]]]] = ( + defaultdict(list) + ) + for path in sorted(root.rglob('results_*.json')): + index.files_read += 1 + rel = path.relative_to(root).as_posix() + try: + content = path.read_text(encoding='utf-8') + if content.startswith('version https://git-lfs.github.com/spec/v1'): + raise RuntimeError( + f'Result dump {rel} is a Git LFS pointer, not JSON. ' + 'Bulk results fetching requires Git-tracked JSON blobs.' + ) + dump = json.loads(content) + except ValueError as exc: + index.rejected.append( + {'path': rel, 'reason': 'unreadable', 'detail': str(exc)} + ) + continue + if not isinstance(dump, dict): + index.rejected.append({'path': rel, 'reason': 'not_an_object'}) + continue + model, reason = dump_identity(dump) + config = dump.get('config') or {} + if model is None: + index.rejected.append( + { + 'path': rel, + 'reason': reason, + 'model_name': dump.get('model_name'), + 'model_args': config.get('model_args'), + } + ) + continue + precision = DTYPE_PRECISION.get(config.get('model_dtype')) + if precision is None: + index.rejected.append( + { + 'path': rel, + 'reason': 'unmapped_model_dtype', + 'model_dtype': config.get('model_dtype'), + } + ) + continue + by_key[(model, precision)].append((rel, dump)) + + for key, dumps in sorted(by_key.items()): + groups = {} + for eval_key, group in TASK_GROUPS.items(): + statements = [ + s + for s in (_dump_statement(group, dump) for _, dump in dumps) + if s is not None + ] + if statements: + groups[eval_key] = _resolve_group(statements) + evidence = ModelEvidence(dumps=[rel for rel, _ in dumps], groups=groups) + for name in ('git_hash', 'transformers_version'): + values = {dump.get(name) for _, dump in dumps} + value = next(iter(values)) + if len(values) == 1 and isinstance(value, str) and value: + setattr(evidence, name, value) + index.entries[key] = evidence + return index diff --git a/every_eval_ever/adapters/hle/adapter.py b/every_eval_ever/adapters/hle/adapter.py index 2dea092ce..df08619da 100644 --- a/every_eval_ever/adapters/hle/adapter.py +++ b/every_eval_ever/adapters/hle/adapter.py @@ -85,12 +85,24 @@ DEFAULT_OUTPUT_DIR = 'data/hle' JUDGE_MODEL_ID = 'openai/o3-mini-2025-01-31' JUDGE_PROMPT_DESCRIPTION = ( - "Scale SEAL evaluates Humanity's Last Exam at temperature 0.0. The " - 'judge model o3-mini-2025-01-31 acts as an automatic answer extractor ' + 'The judge model o3-mini-2025-01-31 acts as an automatic answer extractor ' 'and grader against ground-truth solutions for each of the 2,500 ' 'frozen questions.' ) DATASET_TOTAL_QUESTIONS = 2500 +SOURCE_GENERATION_POLICY = ( + 'Each model on the leaderboard is evaluated on all public questions of ' + "Humanity's Last Exam with temperature 0.0 when configurable or stated " + 'otherwise.' +) +_NUMBER = r'\d+(?:\.\d+)?' +ROW_NOTE_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile( + rf'Sampled at temperature = (?P{_NUMBER}), ' + rf'top_p = (?P{_NUMBER})\.' + ), + re.compile(rf'Sampled at temperature (?P{_NUMBER})\.?'), +) # Map the leaderboard's lowercase company slug to the canonical developer # slug used elsewhere in the EEE data tree (matches helpers/developer.py @@ -143,7 +155,7 @@ def created_at(self) -> str | None: return str(value) if value else None @property - def contamination_message(self) -> str | None: + def row_note(self) -> str | None: value = self.raw.get('contaminationMessage') return str(value) if value else None @@ -305,13 +317,39 @@ def make_judge_model_info() -> ModelInfo: ) -def make_generation_config() -> GenerationConfig: - """Per Scale SEAL's published methodology, the model under evaluation - runs at temperature 0.0. Per the schema, that belongs on - ``EvaluationResult.generation_config.generation_args.temperature`` - (eval.schema.json L400) — not in source/metric ``additional_details``. +def parse_row_note(note: str) -> GenerationArgs | None: + """Return the sampling settings stated by a per-row note. + + Only notes that fully match one of ``ROW_NOTE_PATTERNS`` yield values; + any other note yields ``None``. + """ + stripped = note.strip() + for pattern in ROW_NOTE_PATTERNS: + match = pattern.fullmatch(stripped) + if match is None: + continue + values = { + key: float(value) + for key, value in match.groupdict().items() + if value is not None + } + return GenerationArgs(**values) + return None + + +def make_generation_config(row: LeaderboardRow) -> GenerationConfig | None: + """Build the generation config from the row's own note, if any. + + Sampling values are set only when the note states them in a reviewed + sentence form; the note itself is kept verbatim as ``source_row_note``. """ - return GenerationConfig(generation_args=GenerationArgs(temperature=0.0)) + note = row.row_note + if note is None: + return None + return GenerationConfig( + generation_args=parse_row_note(note), + additional_details={'source_row_note': note}, + ) def make_source_data() -> SourceDataUrl: @@ -383,7 +421,7 @@ def make_accuracy_result( }, ), score_details=ScoreDetails(**score_details_kwargs), - generation_config=make_generation_config(), + generation_config=make_generation_config(row), ) @@ -417,7 +455,7 @@ def make_calibration_result( ), ), score_details=ScoreDetails(score=row.calibration_error), - generation_config=make_generation_config(), + generation_config=make_generation_config(row), ) @@ -441,9 +479,9 @@ def make_log( 'raw_model_display_name': row.model_display, 'rank': str(row.raw.get('rank')), } - contamination = row.contamination_message - if contamination: - additional_details['contamination_message'] = contamination + note = row.row_note + if note: + additional_details['source_row_note'] = note deprecated = row.raw.get('deprecated') if deprecated is not None: additional_details['deprecated'] = 'true' if deprecated else 'false' @@ -464,6 +502,7 @@ def make_log( 'leaderboard_url': LEADERBOARD_URL, 'hle_home_url': HLE_HOME_URL, 'hle_dataset_hf_url': HLE_DATASET_HF_URL, + 'source_generation_policy': SOURCE_GENERATION_POLICY, }, ), eval_library=EvalLibrary( diff --git a/every_eval_ever/adapters/mmlu_pro/adapter.py b/every_eval_ever/adapters/mmlu_pro/adapter.py index 66bd0b28c..51521f4d8 100644 --- a/every_eval_ever/adapters/mmlu_pro/adapter.py +++ b/every_eval_ever/adapters/mmlu_pro/adapter.py @@ -12,8 +12,9 @@ Each CSV row carries a model's overall accuracy plus 14 per-subject accuracies (Biology, Business, Chemistry, Computer Science, Economics, Engineering, Health, History, Law, Math, Philosophy, Physics, Psychology, -Other), all reported as proportions in [0, 1] from a 5-shot CoT setup -described in Wang et al. 2024. +Other), all reported as proportions in [0, 1]. The leaderboard has no +prompt-setup column; ``prompt_style`` is recorded only when the model name +carries an ``(N-shot)`` marker, e.g. ``Athene-V2-Chat (0-shot)``. For every model the adapter emits one ``EvaluationLog`` with 15 ``EvaluationResult`` entries: ``mmlu_pro/overall`` and one @@ -158,6 +159,8 @@ 'rho': 'microsoft', } +SHOT_MARKER = re.compile(r'\((\d+)-shot\)', re.IGNORECASE) + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -251,21 +254,29 @@ def normalize_developer(model_name: str) -> str: return get_developer(model_name) -def make_source_data() -> SourceDataHf: +def prompt_style_from_name(model_name: str) -> str | None: + """Return the ``N-shot`` label stated by a ``(N-shot)`` name marker.""" + match = SHOT_MARKER.search(model_name) + return f'{int(match.group(1))}-shot' if match else None + + +def make_source_data(prompt_style: str | None = None) -> SourceDataHf: + additional_details = { + 'results_csv_url': RESULTS_CSV_URL, + 'leaderboard_space_url': LEADERBOARD_SPACE_URL, + 'benchmark_hf_repo': BENCHMARK_HF_REPO, + 'paper_url': PAPER_URL, + 'github_url': GITHUB_URL, + 'dataset_total_questions': str(DATASET_TOTAL_QUESTIONS), + } + if prompt_style: + additional_details['prompt_style'] = prompt_style return SourceDataHf( dataset_name='MMLU-Pro leaderboard submissions (TIGER-Lab)', source_type='hf_dataset', hf_repo=RESULTS_HF_REPO, hf_split='train', - additional_details={ - 'results_csv_url': RESULTS_CSV_URL, - 'leaderboard_space_url': LEADERBOARD_SPACE_URL, - 'benchmark_hf_repo': BENCHMARK_HF_REPO, - 'paper_url': PAPER_URL, - 'github_url': GITHUB_URL, - 'dataset_total_questions': str(DATASET_TOTAL_QUESTIONS), - 'prompt_style': '5-shot CoT', - }, + additional_details=additional_details, ) @@ -274,7 +285,11 @@ def make_metric_config( metric_id: str, metric_name: str, description: str, + prompt_style: str | None = None, ) -> MetricConfig: + additional_details = {'aggregation': 'accuracy_over_subset'} + if prompt_style: + additional_details['prompt_style'] = prompt_style return MetricConfig( evaluation_description=description, metric_id=metric_id, @@ -285,24 +300,27 @@ def make_metric_config( score_type=ScoreType.continuous, min_score=0.0, max_score=1.0, - additional_details={ - 'aggregation': 'accuracy_over_subset', - 'prompt_style': '5-shot CoT', - }, + additional_details=additional_details, ) def make_evaluation_result( - *, result_id: str, name: str, description: str, score: float + *, + result_id: str, + name: str, + description: str, + score: float, + prompt_style: str | None = None, ) -> EvaluationResult: return EvaluationResult( evaluation_result_id=result_id, evaluation_name=name, - source_data=make_source_data(), + source_data=make_source_data(prompt_style), metric_config=make_metric_config( metric_id=result_id, metric_name=name, description=description, + prompt_style=prompt_style, ), score_details=ScoreDetails(score=score), ) @@ -354,6 +372,7 @@ def make_log( raw_data_source = row.get('Data Source', '').strip() data_source = normalize_data_source(raw_data_source) size_b = parse_size(row.get('Model Size(B)', '')) + prompt_style = prompt_style_from_name(model_name) results: list[EvaluationResult] = [ make_evaluation_result( @@ -361,9 +380,10 @@ def make_log( name='MMLU-Pro (overall)', description=( 'Overall accuracy across the ~12,000-question MMLU-Pro ' - 'benchmark, evaluated 5-shot with chain-of-thought.' + 'benchmark.' ), score=overall, + prompt_style=prompt_style, ) ] for subject in SUBJECTS: @@ -375,11 +395,9 @@ def make_log( make_evaluation_result( result_id=f'mmlu_pro/{slug}', name=f'MMLU-Pro ({subject})', - description=( - f'Accuracy on the MMLU-Pro {subject} subset, ' - 'evaluated 5-shot with chain-of-thought.' - ), + description=f'Accuracy on the MMLU-Pro {subject} subset.', score=subject_score, + prompt_style=prompt_style, ) ) diff --git a/every_eval_ever/adapters/vals_ai/adapter.py b/every_eval_ever/adapters/vals_ai/adapter.py index c627c381b..1fdd81c2a 100644 --- a/every_eval_ever/adapters/vals_ai/adapter.py +++ b/every_eval_ever/adapters/vals_ai/adapter.py @@ -12,9 +12,11 @@ from pathlib import Path from typing import Any from urllib.error import URLError +from urllib.parse import urljoin from urllib.request import Request, urlopen from every_eval_ever.eval_types import ( + AgenticEvalConfig, EvalLibrary, EvaluationLog, EvaluationResult, @@ -61,6 +63,95 @@ 'qwen': 'alibaba', 'togethercomputer': 'together', } +METHODOLOGY_BLOCK_TAGS = frozenset( + { + 'blockquote', + 'br', + 'dd', + 'div', + 'dl', + 'dt', + 'h3', + 'h4', + 'h5', + 'h6', + 'hr', + 'li', + 'ol', + 'p', + 'pre', + 'table', + 'td', + 'th', + 'tr', + 'ul', + } +) +METHODOLOGY_SKIPPED_TAGS = frozenset( + {'astro-island', 'script', 'style', 'template'} +) + + +class ValsMethodologyStatementMissing(RuntimeError): + """A benchmark page no longer states a value the adapter fills from it.""" + + +@dataclass(frozen=True) +class PageStatement: + """A generation setting stated as a number in a benchmark's methodology.""" + + field: str + value: float | int + sentence: str + + +_SAGE_SENTENCE = ( + 'All models are evaluated with temperature 1, and produce at most 30K ' + 'tokens (the length of a short book - more than enough to adequately ' + 'grade student work).' +) +_MED_SENTENCE = ( + 'All models are evaluated with temperature 1, and produce at most 30k ' + 'tokens.' +) +_MMMU_SENTENCE = 'All models were ran with a temperature of 0.' +_MMMU_CAP_SENTENCE = ( + 'We established a consistent baseline by using a default configuration ' + 'of 8192 maximum output tokens for all models to ensure that outputs ' + 'were not truncated.' +) + +PAGE_STATEMENTS: dict[str, tuple[PageStatement, ...]] = { + 'sage': ( + PageStatement('temperature', 1.0, _SAGE_SENTENCE), + PageStatement('max_tokens', 30000, _SAGE_SENTENCE), + ), + 'medcode': ( + PageStatement('temperature', 1.0, _MED_SENTENCE), + PageStatement('max_tokens', 30000, _MED_SENTENCE), + ), + 'medscribe': ( + PageStatement('temperature', 1.0, _MED_SENTENCE), + PageStatement('max_tokens', 30000, _MED_SENTENCE), + ), + 'mmmu': ( + PageStatement('temperature', 0.0, _MMMU_SENTENCE), + PageStatement('max_tokens', 8192, _MMMU_CAP_SENTENCE), + ), +} +PAGE_METADATA_DETAILS = { + 'version': 'benchmark_version', + 'runner': 'vals_runner', + 'mode': 'vals_mode', + 'family': 'vals_family', + 'archived': 'vals_archived', +} +ROW_OUTCOME_FIELDS = ( + 'token_totals', + 'usage', + 'task_results', + 'tie_breaker_score', +) @dataclass(frozen=True) @@ -75,6 +166,9 @@ class ValsMetric: model_id: str metrics: dict[str, Any] source_url: str + page_details: dict[str, Any] | None = None + page_fills: dict[str, float | int] | None = None + harness_label: str | None = None @dataclass(frozen=True) @@ -126,6 +220,73 @@ def handle_starttag( self.props.append(props) +class MethodologyParser(HTMLParser): + """Collect the text of the methodology section of a benchmark page.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.parts: list[str] = [] + self.found = False + self._article_depth = 0 + self._capturing = False + self._heading_open = False + self._skip_depth = 0 + + def handle_starttag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + attr_map = {name: value or '' for name, value in attrs} + if tag == 'article': + if self._article_depth: + self._article_depth += 1 + elif 'benchmark-details' in attr_map.get('class', '').split(): + self._article_depth = 1 + return + if not self._article_depth: + return + if tag == 'h2': + if self._capturing: + self._capturing = False + elif not self.found and attr_map.get('id') == 'methodology': + self.found = True + self._capturing = True + self._heading_open = True + return + if tag in METHODOLOGY_SKIPPED_TAGS: + self._skip_depth += 1 + elif tag in METHODOLOGY_BLOCK_TAGS: + self.parts.append(' ') + + def handle_endtag(self, tag: str) -> None: + if not self._article_depth: + return + if tag == 'article': + self._article_depth -= 1 + if not self._article_depth: + self._capturing = False + elif tag == 'h2': + self._heading_open = False + elif tag in METHODOLOGY_SKIPPED_TAGS: + self._skip_depth = max(0, self._skip_depth - 1) + elif tag in METHODOLOGY_BLOCK_TAGS: + self.parts.append(' ') + + def handle_data(self, data: str) -> None: + if self._capturing and not self._heading_open and not self._skip_depth: + self.parts.append(data) + + +def extract_methodology_text(page_html: str) -> str | None: + """Return the whitespace-normalised text under the methodology heading.""" + parser = MethodologyParser() + parser.feed(page_html) + parser.close() + if not parser.found: + return None + text = ' '.join(''.join(parser.parts).split()) + return text or None + + def fetch_text(url: str) -> str: request = Request(url, headers={'User-Agent': USER_AGENT}) try: @@ -204,18 +365,45 @@ def parse_astro_props(raw_props: str) -> dict[str, Any]: return json.loads(unescape(raw_props)) -def normalize_benchmark_page(page_html: str, source_url: str) -> dict[str, Any]: - view = extract_benchmark_view(page_html) +def extract_benchmark_view_url(page_html: str, source_url: str) -> str | None: + """Return the full table payload URL named by the benchmark loader.""" + parser = AstroIslandParser() + parser.feed(page_html) + for raw_props in parser.props: + props = decode_astro_value(parse_astro_props(raw_props)) + if isinstance(url := props.get('benchmarkViewUrl'), str): + return urljoin(source_url, url) + return None + + +def normalize_benchmark_page( + page_html: str, + source_url: str, + *, + benchmark_view: dict[str, Any] | None = None, +) -> dict[str, Any]: + view = ( + benchmark_view + if benchmark_view is not None + else extract_benchmark_view(page_html) + ) + view = view.get('default') or view + if not isinstance(view, dict) or not {'metadata', 'tasks'}.issubset(view): + raise ValueError('BenchmarkView payload lacks metadata/tasks') metadata = view.get('metadata') or {} tasks = view.get('tasks') or {} if not isinstance(metadata, dict) or not isinstance(tasks, dict): raise ValueError('BenchmarkView payload has invalid metadata/tasks') - return { + normalized = { 'metadata': metadata, 'tasks': tasks, 'source_url': source_url, } + methodology_text = extract_methodology_text(page_html) + if methodology_text is not None: + normalized['methodology_text'] = methodology_text + return normalized def extract_collection( @@ -235,7 +423,14 @@ def extract_collection( source_url = f'{base_url.rstrip("/")}/benchmarks/{slug}' try: page_html = fetch_text(source_url) - benchmarks.append(normalize_benchmark_page(page_html, source_url)) + view_url = extract_benchmark_view_url(page_html, source_url) + # Graph props are projections and may omit settings and subtasks. + view = json.loads(fetch_text(view_url)) if view_url else None + benchmarks.append( + normalize_benchmark_page( + page_html, source_url, benchmark_view=view + ) + ) except Exception as exc: source_failures.append( SourceRecordFailure( @@ -273,7 +468,11 @@ def iter_vals_metrics_result( metrics: list[ValsMetric] = [] failures: list[SourceRecordFailure] = [] total_records = 0 - for benchmark_index, benchmark in enumerate(payload.get('benchmarks', [])): + slug_routes: dict[str, str] = {} + for benchmark_index, benchmark in sorted( + enumerate(payload.get('benchmarks', [])), + key=lambda item: _benchmark_order(item[1]), + ): if not isinstance(benchmark, dict): failures.append( SourceRecordFailure( @@ -311,6 +510,32 @@ def iter_vals_metrics_result( total_records += 1 continue benchmark_slug = str(raw_benchmark_slug) + first_route = slug_routes.setdefault(benchmark_slug, source_url) + if first_route != source_url: + failures.append( + SourceRecordFailure( + source_ref=source_url, + reason=( + f'Vals.ai route {source_url!r} serves benchmark slug ' + f'{benchmark_slug!r}, already converted from ' + f'{first_route!r}; route skipped' + ), + source_record={ + 'benchmark_slug': benchmark_slug, + 'source_url': source_url, + 'first_source_url': first_route, + }, + ) + ) + skipped = _skipped_row_failures( + source_url, + tasks, + f'Vals.ai route {source_url!r} skipped: benchmark slug ' + f'{benchmark_slug!r} already converted from {first_route!r}', + ) + failures.extend(skipped) + total_records += 1 + len(skipped) + continue benchmark_name = str(metadata.get('benchmark') or benchmark_slug) task_names = metadata.get('tasks') or {} if not isinstance(tasks, dict): @@ -324,6 +549,33 @@ def iter_vals_metrics_result( total_records += 1 continue + methodology_text = _optional_str(benchmark.get('methodology_text')) + statements = PAGE_STATEMENTS.get(benchmark_slug, ()) + missing_sentences = sorted( + { + statement.sentence + for statement in statements + if statement.sentence not in (methodology_text or '') + } + ) + if missing_sentences: + raise ValsMethodologyStatementMissing( + f'Vals.ai benchmark {benchmark_slug!r} ({source_url}): the ' + 'methodology section no longer contains the sentence the ' + f'page fill relies on: {missing_sentences!r}. Update ' + 'PAGE_STATEMENTS before publishing.' + ) + page_fills = { + statement.field: statement.value for statement in statements + } or None + page_details = { + detail_key: metadata.get(key) + for key, detail_key in PAGE_METADATA_DETAILS.items() + } + if methodology_text is not None: + page_details['vals_methodology_url'] = f'{source_url}#methodology' + harness_label = _optional_str(metadata.get('harness_label')) + for task_key, model_rows in tasks.items(): if not isinstance(model_rows, dict): failures.append( @@ -386,6 +638,9 @@ def iter_vals_metrics_result( model_id=str(model_id), metrics=row, source_url=source_url, + page_details=page_details, + page_fills=page_fills, + harness_label=harness_label, ) ) @@ -397,6 +652,40 @@ def iter_vals_metrics_result( ) +def _benchmark_order(benchmark: Any) -> tuple[bool, str]: + """Sort key preferring the route whose last segment is the embedded slug.""" + if not isinstance(benchmark, dict): + return (True, '') + route = str(benchmark.get('source_url') or BENCHMARKS_URL) + metadata = benchmark.get('metadata') + slug = ( + metadata.get('slug') or metadata.get('benchmark_id') + if isinstance(metadata, dict) + else None + ) + return (route.rstrip('/').rsplit('/', 1)[-1] != str(slug), route) + + +def _skipped_row_failures( + ref_prefix: str, + tasks: Any, + reason: str, +) -> list[SourceRecordFailure]: + """One failure per model/task row of a benchmark page that was skipped.""" + if not isinstance(tasks, dict): + return [] + return [ + SourceRecordFailure( + source_ref=f'{ref_prefix}/{task_key}/{model_id}', + reason=reason, + source_record=row, + ) + for task_key, model_rows in tasks.items() + if isinstance(model_rows, dict) + for model_id, row in model_rows.items() + ] + + def build_index( payload: dict[str, Any], ) -> dict[tuple[str, str], list[ValsMetric]]: @@ -533,6 +822,7 @@ def _make_bundle( 'industry': first.industry, 'leaderboard_page_url': first.source_url, 'extraction_method': 'static_astro_benchmark_view_props', + **(first.page_details or {}), } ), ), @@ -679,6 +969,8 @@ def make_result( 'verbosity': row.metrics.get('verbosity'), 'compute_effort': row.metrics.get('compute_effort'), 'provider': row.metrics.get('provider'), + 'harness': row.metrics.get('harness'), + **{field: row.metrics.get(field) for field in ROW_OUTCOME_FIELDS}, } ) @@ -738,26 +1030,65 @@ def make_result( details=details, uncertainty=uncertainty, ), - generation_config=make_generation_config(row.metrics), + generation_config=make_generation_config( + row.metrics, + page_fills=row.page_fills, + harness_label=row.harness_label, + ), ) -def make_generation_config(metrics: dict[str, Any]) -> GenerationConfig | None: - explicit_generation_args = { - 'temperature': _optional_float(metrics.get('temperature')), - 'top_p': _optional_float(metrics.get('top_p')), - 'max_tokens': _optional_positive_int(metrics.get('max_output_tokens')), +def make_generation_config( + metrics: dict[str, Any], + *, + page_fills: dict[str, float | int] | None = None, + harness_label: str | None = None, +) -> GenerationConfig | None: + """Map one Vals.ai row to generation settings, filling nulls from the page. + + ``page_fills`` holds values the benchmark's methodology section states for + every model and fills only a field the row leaves null; a row value that is + present but unusable (unparseable, or a non-positive cap) stays unset. + ``_source`` records which one was used. + """ + row_fields = { + 'temperature': ('temperature', _optional_float), + 'max_tokens': ('max_output_tokens', _optional_positive_int), } - generation_args = None - if any(value is not None for value in explicit_generation_args.values()): - generation_args = GenerationArgs(**explicit_generation_args) + values: dict[str, Any] = {} + sources: dict[str, str] = {} + for field, (row_key, parse) in row_fields.items(): + raw_value = metrics.get(row_key) + if raw_value is None or raw_value is ASTRO_UNDEFINED: + page_value = (page_fills or {}).get(field) + if page_value is not None: + values[field] = page_value + sources[f'{field}_source'] = 'page_methodology' + continue + row_value = parse(raw_value) + if row_value is not None: + values[field] = row_value + sources[f'{field}_source'] = 'row' + + top_p = _optional_float(metrics.get('top_p')) + if top_p is not None: + values['top_p'] = top_p + reasoning = metrics.get('reasoning') + if isinstance(reasoning, bool): + values['reasoning'] = reasoning + harness = _optional_str(metrics.get('harness')) + if harness is not None: + values['agentic_eval_config'] = AgenticEvalConfig( + additional_details={_harness_detail_key(harness_label): harness} + ) + generation_args = GenerationArgs(**values) if values else None additional_details = _clean_details( { - 'reasoning': metrics.get('reasoning'), 'reasoning_effort': metrics.get('reasoning_effort'), 'verbosity': metrics.get('verbosity'), 'compute_effort': metrics.get('compute_effort'), + **sources, } ) if generation_args is not None or additional_details: @@ -768,6 +1099,11 @@ def make_generation_config(metrics: dict[str, Any]) -> GenerationConfig | None: return None +def _harness_detail_key(harness_label: str | None) -> str: + label = re.sub(r'[^a-z0-9]+', '_', (harness_label or '').lower()).strip('_') + return f'vals_{label or "harness"}' + + def make_source_data(row: ValsMetric) -> SourceDataPrivate | SourceDataUrl: details = _clean_details( { diff --git a/every_eval_ever/converters/helm/adapter.py b/every_eval_ever/converters/helm/adapter.py index 9c400aaea..d35f0fadf 100644 --- a/every_eval_ever/converters/helm/adapter.py +++ b/every_eval_ever/converters/helm/adapter.py @@ -90,6 +90,14 @@ require_uuid4, ) +_SCHEMA_SPLITS = { + 'train': 'train', + 'val': 'val', + 'valid': 'val', + 'validation': 'val', + 'test': 'test', +} + def _instance_counts_by_result_id(per_instance_stats: List) -> Dict[str, int]: """How many distinct instances each aggregate result was computed over. @@ -533,7 +541,11 @@ def _transform_single( samples_number=len( set(state.instance.id for state in request_states) ), - sample_ids=[str(state.instance.id) for state in request_states], + sample_ids=list( + dict.fromkeys( + str(state.instance.id) for state in request_states + ) + ), additional_details={ 'scenario_name': str(run_spec.scenario_spec.class_name), 'scenario_args': json.dumps(run_spec.scenario_spec.args) @@ -617,11 +629,17 @@ def _transform_single( or None ) + hf_split = _SCHEMA_SPLITS.get(_stat_name_part(split) or '') + evaluation_results.append( EvaluationResult( evaluation_result_id=evaluation_result_id, evaluation_name=source_data.dataset_name, - source_data=source_data, + source_data=( + source_data.model_copy(update={'hf_split': hf_split}) + if hf_split + else source_data + ), evaluation_timestamp=evaluation_timestamp, metric_config=metric_config, score_details=ScoreDetails( @@ -664,6 +682,11 @@ def _transform_single( 'num_completions': str( request_states[0].request.num_completions ), + **( + {'adapter_method': str(adapter_spec.method)} + if adapter_spec.method + else {} + ), }, ), ) diff --git a/every_eval_ever/converters/inspect/adapter.py b/every_eval_ever/converters/inspect/adapter.py index ac4b6f17f..a32687c7c 100644 --- a/every_eval_ever/converters/inspect/adapter.py +++ b/every_eval_ever/converters/inspect/adapter.py @@ -117,6 +117,23 @@ def _require_inspect_dependencies() -> None: _STDERR_PREFERENCE = ('analytic', 'bootstrap') _UNCERTAINTY_METRICS = _STDDEV_METRICS | frozenset(_STDERR_METHODS) +# Plan-step parameter names that override a generation field for the calls +# that step makes, keyed by the `GenerationArgs` field they override. +_STEP_OVERRIDE_PARAMS = { + 'temperature': frozenset({'temperature'}), + 'max_tokens': frozenset( + {'max_tokens', 'max_completion_tokens', 'max_output_tokens'} + ), + 'top_p': frozenset({'top_p'}), + 'top_k': frozenset({'top_k'}), +} +_SCHEMA_SPLITS = { + 'train': 'train', + 'val': 'val', + 'validation': 'val', + 'test': 'test', +} + class InspectAIAdapter(BaseEvaluationAdapter): """ @@ -170,7 +187,11 @@ def _extract_uncertainty( stddev_value: float | None, num_samples: int | None, ) -> Uncertainty | None: - if stderr_value is None and stddev_value is None and not num_samples: + if ( + stderr_value is None + and stddev_value is None + and num_samples is None + ): return None return Uncertainty( # A standard error of exactly 0.0 is what a task every sample @@ -301,9 +322,21 @@ def _extract_evaluation_results( # score, and states how many those were. The run-wide count includes # the samples this scorer returned no value for, and is what a log # from before Inspect reported this per scorer leaves us with. - scored_samples = ( - getattr(scorer, 'scored_samples', None) or num_samples - ) + scored_samples = getattr(scorer, 'scored_samples', None) + if scored_samples is None: + scored_samples = num_samples + scorer_source_data = source_data.model_copy(deep=True) + if isinstance(scorer_source_data, SourceDataHf): + scorer_source_data.samples_number = scored_samples + elif scored_samples is not None: + scorer_source_data.additional_details = { + **(scorer_source_data.additional_details or {}), + 'samples_number': str(scored_samples), + } + if (scorer_source_data.additional_details or {}).get( + 'dataset_size' + ) == str(scored_samples): + scorer_source_data.additional_details.pop('dataset_size') # Inspect reports dispersion as a metric of the scorer, and the # schema carries it on the score it describes rather than beside it. @@ -329,7 +362,7 @@ def _extract_evaluation_results( scorer_name=scorer_name, metric_info=metric_info, llm_grader=llm_grader, - source_data=source_data, + source_data=scorer_source_data, evaluation_timestamp=timestamp, generation_config=generation_config, stderr_value=None if describes_itself else stderr_value, @@ -369,13 +402,26 @@ def _looks_like_local_path(cls, location: str | None) -> bool: return not cls._HF_REPO_RE.match(location) def _extract_source_data( - self, dataset: EvalDataset, task_name: str + self, + dataset: EvalDataset, + task_name: str, + task_args: Dict[str, Any] | None = None, ) -> SourceDataHf | SourceDataPrivate: sample_ids = ( [str(sid) for sid in dataset.sample_ids] if dataset.sample_ids is not None else None ) + samples_number = ( + len(sample_ids) if sample_ids is not None else dataset.samples + ) + dataset_size = ( + str(dataset.samples) + if dataset.samples is not None + else None + ) + split = (task_args or {}).get('split') + split = _SCHEMA_SPLITS.get(split) if isinstance(split, str) else None if self._looks_like_local_path(dataset.location): # dataset.location is not a valid HF repo identifier: it may @@ -403,10 +449,14 @@ def _extract_source_data( ) if dataset.name: additional_details['inspect_dataset_name'] = dataset.name - if dataset.samples is not None: - additional_details['samples_number'] = str(dataset.samples) + if samples_number is not None: + additional_details['samples_number'] = str(samples_number) + if dataset_size is not None: + additional_details['dataset_size'] = dataset_size if sample_ids is not None: additional_details['sample_ids'] = ','.join(sample_ids) + if split is not None: + additional_details['split'] = split return SourceDataPrivate( source_type='other', dataset_name=dataset_name, @@ -421,13 +471,17 @@ def _extract_source_data( if dataset.name else task_name.split('/')[-1] ) - return SourceDataHf( # TODO add hf_split + additional_details = {'shuffled': str(dataset.shuffled)} + if dataset_size is not None: + additional_details['dataset_size'] = dataset_size + return SourceDataHf( source_type='hf_dataset', dataset_name=dataset_name, hf_repo=dataset.location, - samples_number=dataset.samples, + hf_split=split, + samples_number=samples_number, sample_ids=sample_ids, - additional_details={'shuffled': str(dataset.shuffled)}, + additional_details=additional_details, ) def _safe_get(self, obj: Any, field: str): @@ -477,15 +531,66 @@ def _extract_prompt_template(self, plan: InspectEvalPlan) -> str | None: return None + @staticmethod + def _param_names(params: Any) -> set[str]: + """Every dict key in a step's parameters, at any depth, outside tool definitions.""" + if isinstance(params, dict): + if params.get('type') == 'tool': + return set() + names = {str(key) for key in params} + for key, value in params.items(): + if key != 'tools': + names |= InspectAIAdapter._param_names(value) + return names + if isinstance(params, (list, tuple)): + names = set() + for item in params: + names |= InspectAIAdapter._param_names(item) + return names + return set() + + def _overridden_generation_fields( + self, inspect_plan: InspectEvalPlan + ) -> set[str]: + """Generation fields a plan step may set per call, out of the plan config's reach.""" + steps = list(inspect_plan.steps) + named: set[str] = set() + for step in steps: + named |= self._param_names(step.params) + named |= self._param_names(step.params_passed) + overridden = { + field + for field, params in _STEP_OVERRIDE_PARAMS.items() + if named & params + } + if steps and ( + (steps[-1].params or {}).get('model') is not None + or (steps[-1].params_passed or {}).get('model') is not None + ): + overridden |= set(_STEP_OVERRIDE_PARAMS) + return overridden + def _extract_generation_config( self, spec: EvalSpec, inspect_plan: InspectEvalPlan ) -> GenerationConfig: - eval_config = spec.model_generate_config + unverified_packages = sorted( + set(spec.packages) - {'inspect_ai', 'inspect_evals'} + ) + # Other scaffolds can override the plan's settings inside their calls. + eval_config = ( + spec.model_generate_config + if unverified_packages + else spec.model_generate_config.merge(inspect_plan.config) + ) eval_generation_config = { gen_config: json.dumps(value) for gen_config, value in vars(eval_config).items() if value is not None } + if unverified_packages: + eval_generation_config['plan_config_unverified_packages'] = ','.join( + unverified_packages + ) eval_sandbox = spec.task_args.get('sandbox', None) if eval_sandbox and not isinstance(eval_sandbox, list): eval_sandbox = [eval_sandbox] @@ -515,25 +620,56 @@ def _extract_generation_config( ) max_attempts = ( - spec.task_args.get('max_attempts') or eval_config.max_retries + spec.task_args.get('max_attempts') + or spec.model_generate_config.max_retries ) # TODO not sure if max_attempts == max_retries in this case + reasoning_effort = eval_config.reasoning_effort reasoning = ( - True - if eval_config.reasoning_effort - and eval_config.reasoning_effort.lower() != 'none' - else False + reasoning_effort.lower() != 'none' + if isinstance(reasoning_effort, str) + else None ) + overridden = self._overridden_generation_fields(inspect_plan) + filled = { + field: getattr(eval_config, field) + for field in _STEP_OVERRIDE_PARAMS + if field not in overridden + and getattr(eval_config, field) is not None + } + for field in overridden: + eval_generation_config.pop(field, None) + if overridden: + eval_generation_config['overridden_by_step'] = ','.join( + sorted(overridden) + ) + if filled: + from_plan = { + not unverified_packages + and getattr(inspect_plan.config, field) is not None + for field in filled + } + eval_generation_config['generation_args_source'] = ( + 'mixed' + if len(from_plan) > 1 + else 'plan_config' + if True in from_plan + else 'model_config' + ) + + if steps := inspect_plan.steps: + eval_generation_config['terminal_solver'] = steps[-1].solver + available_tools: List[AvailableTool] = self._extract_available_tools( inspect_plan ) generation_args = GenerationArgs( - temperature=eval_config.temperature, - top_p=eval_config.top_p, - top_k=eval_config.top_k, - max_tokens=eval_config.max_tokens, + temperature=filled.get('temperature'), + top_p=filled.get('top_p'), + top_k=filled.get('top_k'), + max_tokens=filled.get('max_tokens'), reasoning=reasoning, prompt_template=self._extract_prompt_template(inspect_plan), agentic_eval_config=AgenticEvalConfig( @@ -552,11 +688,16 @@ def _extract_generation_config( additional_details=additional_details or None, ) - def _extract_library_version(self, packages: Dict[str, str]) -> str: - parts = [ - f'{name}:{version}' for name, version in packages.items() if version - ] - return ','.join(parts) + def _extract_library_version( + self, packages: Dict[str, str] + ) -> tuple[str | None, dict[str, str] | None]: + """The inspect_ai version, and the other packages the log records.""" + others = { + str(name): str(version) + for name, version in packages.items() + if version and name != 'inspect_ai' + } + return packages.get('inspect_ai') or None, others or None def transform_from_directory( self, dir_path: Union[str, Path], metadata_args: Dict[str, Any] = None @@ -680,11 +821,14 @@ def _transform_single( if not evaluation_unix_timestamp: evaluation_unix_timestamp = retrieved_unix_timestamp - library_version = self._extract_library_version(eval_spec.packages) + library_version, library_packages = self._extract_library_version( + eval_spec.packages + ) eval_library = EvalLibrary( name=metadata_args.get('eval_library_name', 'inspect_ai'), version=library_version or metadata_args.get('eval_library_version', 'unknown'), + additional_details=library_packages, ) evaluator_relationship = metadata_args.get( @@ -696,7 +840,7 @@ def _transform_single( ) source_data = self._extract_source_data( - eval_spec.dataset, eval_spec.task + eval_spec.dataset, eval_spec.task, eval_spec.task_args ) model_path = eval_spec.model diff --git a/every_eval_ever/converters/lm_eval/adapter.py b/every_eval_ever/converters/lm_eval/adapter.py index 406fea52f..51c5ca936 100644 --- a/every_eval_ever/converters/lm_eval/adapter.py +++ b/every_eval_ever/converters/lm_eval/adapter.py @@ -163,10 +163,23 @@ def _get_tasks(self, raw_data: Dict[str, Any]) -> List[str]: tasks.append(task_name) return tasks - def _build_source_data(self, task_config: Dict[str, Any], task_name: str): - """Build source_data from task config.""" + def _build_source_data( + self, + task_config: Dict[str, Any], + task_name: str, + n_samples: Dict[str, Any] | None = None, + ): + """Build source_data from task config and the task's `n-samples` entry.""" dataset_path = task_config.get('dataset_path', '') dataset_name = task_config.get('task', task_name) + n_samples = n_samples or {} + original = n_samples.get('original') + original = original if isinstance(original, int) else None + effective = n_samples.get('effective') + samples_number = effective if isinstance(effective, int) else original + additional: Dict[str, str] = {} + if original is not None and original != samples_number: + additional['dataset_size'] = str(original) if ( dataset_path @@ -181,33 +194,51 @@ def _build_source_data(self, task_config: Dict[str, Any], task_name: str): task_config.get('test_split') or task_config.get('validation_split') ), + samples_number=samples_number, + additional_details=additional or None, ) + if samples_number is not None: + additional['samples_number'] = str(samples_number) return SourceDataPrivate( dataset_name=dataset_name, source_type='other', + additional_details=additional or None, ) def _build_generation_config( self, task_config: Dict[str, Any] ) -> Optional[GenerationConfig]: """Build generation config from task config.""" - gen_kwargs = task_config.get('generation_kwargs', {}) - if not gen_kwargs: - return None + gen_kwargs = task_config.get('generation_kwargs') or {} + output_type = task_config.get('output_type') + output_type = output_type if isinstance(output_type, str) else None + scored_by_likelihood = output_type in ( + 'loglikelihood', + 'loglikelihood_rolling', + 'multiple_choice', + ) - args = GenerationArgs( - temperature=gen_kwargs.get('temperature'), - top_p=gen_kwargs.get('top_p'), - top_k=gen_kwargs.get('top_k'), - max_tokens=gen_kwargs.get('max_gen_toks'), + args = ( + GenerationArgs( + temperature=gen_kwargs.get('temperature'), + top_p=gen_kwargs.get('top_p'), + top_k=gen_kwargs.get('top_k'), + max_tokens=gen_kwargs.get('max_gen_toks'), + ) + if gen_kwargs and not scored_by_likelihood + else None ) additional = {} + if output_type is not None: + additional['output_type'] = output_type for k, v in gen_kwargs.items(): if k not in ('temperature', 'top_p', 'top_k', 'max_gen_toks'): additional[k] = json.dumps(v) if not isinstance(v, str) else v if task_config.get('num_fewshot') is not None: additional['num_fewshot'] = str(task_config['num_fewshot']) + if args is None and not additional: + return None return GenerationConfig( generation_args=args, @@ -229,7 +260,7 @@ def _build_evaluation_results( if not isinstance(bootstrap_iters, int): bootstrap_iters = None - source_data = self._build_source_data(task_config, task_name) + source_data = self._build_source_data(task_config, task_name, n_samples) gen_config = self._build_generation_config(task_config) eval_timestamp = raw_data.get('date') if eval_timestamp is not None: diff --git a/every_eval_ever/cron/README.md b/every_eval_ever/cron/README.md index 5c9fef4b3..8735a4b1b 100644 --- a/every_eval_ever/cron/README.md +++ b/every_eval_ever/cron/README.md @@ -428,6 +428,59 @@ Unreadable retention entries are kept with a warning naming the snapshot and failure; unscanned snapshots are listed in the summary. Unchanged same-size source files still use the agreed size-based change detection. +### Supersession + +Every row carries `evaluation_id`, `retrieved_timestamp`, `cron_adapter` +and `cron_run_date` (read from the record; the cron fields only when its +`type_of_addition` is `cron`) and `superseded_by` / `superseded_reason`, +recomputed on every run from two rules, in this order: + +- **Retired.** `flat/retired.jsonl` in the datastore, maintained by pull + request, lists `{"object_uuid", "reason", "replaced_by"?}` per line: + `superseded_by` = `replaced_by`, or `"retired"` without one, and reason + `retired`. A malformed line, a duplicate UUID or a self-replacement fails + the run; an entry whose UUID is not in the index is skipped and listed as + stale in the summary, and one whose `replaced_by` is not in the index is + applied and listed as dangling. +- **Newer capture.** Only cron-ingested rows join a series, keyed by + collection, `cron_adapter` and `evaluation_id` minus its trailing + segment, and only when that segment is an epoch number equal to + `retrieved_timestamp`. Among the series' rows that are not retired, the + newest by `retrieved_timestamp` (then UUID) is current; the others get + `superseded_by` = its UUID and reason `newer_capture`. Retiring the + newest capture therefore makes the previous one current again. A series + whose newest row drops any older result identity is left unmarked and + listed as regressed, for a maintainer to settle with a retire entry. + `result_identities` stores a sorted multiset of SHA-256 hashes of result + metadata: all result fields except `score_details`, `evaluation_timestamp` + and the run-local `evaluation_result_id`. Thus benchmark names, metrics, + dataset/split, generation settings and duplicate multiplicity must all be + covered. A metadata correction may require reviewed retirement; changed + scores alone do not. Unreadable identities also leave the series unmarked. + `result_count` remains diagnostic and never establishes coverage. A series + where two rows share a `cron_run_date` is ambiguous and left unmarked, and + a collection whose rows still have ids pending is skipped; both are + listed in the summary. Hand-submitted and converted records are only ever + superseded by the retire list. + +Consumers keep rows where `superseded_by` is null. Nothing is deleted: +`data/` and `flat/objects/` are untouched, and the fields enter the +manifest core hash through the entries hash, so a supersession change +publishes a new snapshot while an unchanged rebuild stays a no-op. + +Rows from snapshots older than either record ids or `result_identities` +serialize without identity fields until filled. Every run fills all missing +identity fields. One shallow Git fetch reads the published aggregate objects at +a single resolved datastore commit; semantic comparisons reuse that checkout. +Git is required. LFS smudging is disabled, and only LFS-backed aggregate objects +actually needed by the run are downloaded individually at that same commit and +verified against their pointer hashes. Sample sidecars are never read for +identity backfill. The summary reports the pin, Git pack count, and required LFS +payload paths. Temporary checkouts are removed even if the rebuild fails. +`--bootstrap-ids` remains a compatibility alias for the normal behavior; +`--backfill-limit` has been removed. Existing daily diff, deep verification, +retention, and recovery reads keep their existing fetch paths. + ### Reviewer comments and changes The review found that companion reattachment could overwrite historical diff --git a/every_eval_ever/cron/flat_rebuild.py b/every_eval_ever/cron/flat_rebuild.py index 8e401e727..7f767aa50 100644 --- a/every_eval_ever/cron/flat_rebuild.py +++ b/every_eval_ever/cron/flat_rebuild.py @@ -1,4 +1,4 @@ -"""Rebuild the datastore's flat view against the Hub, without a clone. +"""Rebuild the datastore's flat view using the Hub and pinned bulk reads. ``tools/build_flat_datastore.py`` in the EEE_datastore repository defines the flat layout and rebuilds it from a local checkout. That checkout no longer @@ -13,6 +13,8 @@ behind ``--allow-bootstrap``); 4. rebuild rows, collection indexes and the manifest, preserving published bytes, accepting reserialization and reporting conflicting UUID reuse; + mark superseded rows (see ``apply_supersession``), backfilling the + ``evaluation_id`` of inherited rows that predate the field; 5. retire collection indexes whose collection no longer exists under ``data/`` (moved to ``flat/indexes/retired/``), trim snapshots past the retention window, and commit everything in bounded batches with the @@ -37,13 +39,16 @@ import hashlib import json import os +import re import sys import time import warnings -from collections import defaultdict +from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta +from decimal import Decimal from pathlib import Path, PurePosixPath from typing import Any, Callable, Sequence from uuid import UUID @@ -58,6 +63,10 @@ # Imported as a module, not by name: the retry helpers are replaced in tests # and the commit path has to see the replacement. from every_eval_ever.cron import store +from every_eval_ever.cron.flat_snapshot import ( + FlatSnapshotError, + FlatSnapshotReader, +) from every_eval_ever.cron.submit import ( DEFAULT_BATCH_SIZE, DEFAULT_DATASTORE_REPO, @@ -68,6 +77,7 @@ RETIRE_PREFIX = 'flat/indexes/retired' BY_LEGACY_PATH = 'flat/indexes/by_legacy_path.jsonl' MANIFESTS_PREFIX = 'flat/manifests' +RETIRED_LIST_PATH = 'flat/retired.jsonl' DATA_PREFIX = 'data/' #: Fields of a manifest excluded from its content hash, mirroring #: ``manifest_core`` in ``tools/build_flat_datastore.py``. @@ -80,6 +90,9 @@ #: decide between trim and pin; unscanned ones are conservatively kept. DEFAULT_SCAN_BUDGET = 20 DOWNLOAD_WORKERS = 8 +BACKFILL_CHUNK = 500 +SUPERSEDED_REASONS = frozenset({'newer_capture', 'retired'}) +_NUMERIC = re.compile(r'\d+(?:\.\d+)?') #: Excluded paths a maintainer has acknowledged; see ``exit_code_for``. ACKNOWLEDGED_CONFLICTS = Path(__file__).with_name( 'flat_acknowledged_conflicts.txt' @@ -105,8 +118,36 @@ class Row: instance_level_path: str | None = None instance_sha: str | None = None instance_level_size_bytes: int | None = None + evaluation_id: str | None = None + retrieved_timestamp: str | None = None + #: Set only for records whose provenance says ``type_of_addition`` is + #: ``cron``. + cron_adapter: str | None = None + cron_run_date: str | None = None + #: ``len(evaluation_results)``; None when the record has no such list. + result_count: int | None = None + #: Sorted metadata hashes, preserving duplicate result identities. + result_identities: tuple[str, ...] | None = None + #: False for an inherited row written before the record ids were + #: indexed; such a row serializes without them until backfilled. + ids_recorded: bool = True + superseded_by: str | None = None + superseded_reason: str | None = None def __post_init__(self) -> None: + if (self.superseded_by is None) != (self.superseded_reason is None): + raise FlatRebuildError( + f'{self.object_uuid}: superseded_by and superseded_reason ' + 'are set together' + ) + if ( + self.superseded_reason is not None + and self.superseded_reason not in SUPERSEDED_REASONS + ): + raise FlatRebuildError( + f'{self.object_uuid}: unknown superseded_reason ' + f'{self.superseded_reason!r}' + ) if self.instance_level_available: if ( self.instance_level_path is None @@ -137,6 +178,19 @@ def to_dict(self) -> dict[str, Any]: row['instance_level_path'] = self.instance_level_path row['instance_sha'] = self.instance_sha row['instance_level_size_bytes'] = self.instance_level_size_bytes + if self.ids_recorded: + row['evaluation_id'] = self.evaluation_id + row['retrieved_timestamp'] = self.retrieved_timestamp + row['cron_adapter'] = self.cron_adapter + row['cron_run_date'] = self.cron_run_date + row['result_count'] = self.result_count + row['result_identities'] = ( + list(self.result_identities) + if self.result_identities is not None + else None + ) + row['superseded_by'] = self.superseded_by + row['superseded_reason'] = self.superseded_reason return row @classmethod @@ -153,9 +207,76 @@ def from_dict(cls, row: dict[str, Any]) -> 'Row': instance_level_path=row.get('instance_level_path'), instance_sha=row.get('instance_sha'), instance_level_size_bytes=row.get('instance_level_size_bytes'), + evaluation_id=row.get('evaluation_id'), + retrieved_timestamp=row.get('retrieved_timestamp'), + cron_adapter=row.get('cron_adapter'), + cron_run_date=row.get('cron_run_date'), + result_count=row.get('result_count'), + result_identities=( + tuple(row['result_identities']) + if row.get('result_identities') is not None + else None + ), + ids_recorded=( + 'evaluation_id' in row and 'result_identities' in row + ), + superseded_by=row.get('superseded_by'), + superseded_reason=row.get('superseded_reason'), ) +@dataclass(frozen=True) +class RecordIds: + """The identity and provenance fields a row copies from its record.""" + + evaluation_id: str | None = None + retrieved_timestamp: str | None = None + cron_adapter: str | None = None + cron_run_date: str | None = None + result_count: int | None = None + #: Sorted metadata hashes, preserving duplicate result identities. + result_identities: tuple[str, ...] | None = None + + def apply(self, row: 'Row') -> 'Row': + return replace( + row, + evaluation_id=self.evaluation_id, + retrieved_timestamp=self.retrieved_timestamp, + cron_adapter=self.cron_adapter, + cron_run_date=self.cron_run_date, + result_count=self.result_count, + result_identities=self.result_identities, + ids_recorded=True, + ) + + +@dataclass(frozen=True) +class Supersession: + """Rows with supersession applied, plus what the rules declined.""" + + rows: tuple['Row', ...] + #: Series keys left unmarked because two captures share a run date. + ambiguous_series: tuple[str, ...] = () + #: Series left unmarked because the newest capture does not cover + #: every older result identity, or identities cannot be read. + regressed_series: tuple[str, ...] = () + #: Collections skipped by the automatic rule while ids are pending. + pending_collections: tuple[str, ...] = () + #: Retire-list entries naming a uuid absent from the index. + stale_retired: tuple[str, ...] = () + #: Retire-list entries whose ``replaced_by`` is absent from the index. + dangling_replaced_by: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RetiredEntry: + """One ``flat/retired.jsonl`` line: a record a maintainer retired.""" + + object_uuid: str + reason: str + replaced_by: str | None = None + + @dataclass(frozen=True) class ManifestInfo: """One snapshot directory under ``flat/manifests/``.""" @@ -233,6 +354,19 @@ class RebuildReport: manifest_core_sha256: str | None = None verified_files: int = 0 verified_reserialized: int = 0 + ids_backfilled: int = 0 + ids_pending: int = 0 + bulk_revision: str | None = None + bulk_pack_count: int = 0 + bulk_lfs_download_count: int = 0 + bulk_lfs_download_paths: tuple[str, ...] = () + superseded_captures: int = 0 + superseded_retired: int = 0 + ambiguous_series: tuple[str, ...] = () + regressed_series: tuple[str, ...] = () + pending_collections: tuple[str, ...] = () + stale_retired: tuple[str, ...] = () + dangling_replaced_by: tuple[str, ...] = () commits: int = 0 uploads: int = 0 duration_seconds: float = 0.0 @@ -672,6 +806,7 @@ def build_rows( schema_version = schema_version_of(data, path, errors) if schema_version is None: continue + ids = record_ids(data) sha256 = sha256_bytes(data) old = old_by_uuid.get(object_uuid) if old is not None: @@ -698,6 +833,8 @@ def build_rows( excluded.add(samples_path_for(path)) if row is None: continue + if not row.ids_recorded: + row = ids.apply(row) if ( row.instance_level_available and not old.instance_level_available @@ -706,14 +843,16 @@ def build_rows( upload_samples.add(object_uuid) rows[object_uuid] = row continue - row = Row( - object_uuid=object_uuid, - object_path=object_path_for(object_uuid, suffix='.json'), - sha256=sha256, - size_bytes=len(data), - legacy_path=path, - benchmark=benchmark, - eval_schema_version=schema_version, + row = ids.apply( + Row( + object_uuid=object_uuid, + object_path=object_path_for(object_uuid, suffix='.json'), + sha256=sha256, + size_bytes=len(data), + legacy_path=path, + benchmark=benchmark, + eval_schema_version=schema_version, + ) ) if listing is not None and row.object_path in listing: old_object = _fetch_existing( @@ -890,6 +1029,271 @@ def schema_version_of(data: bytes, path: str, errors: list[str]) -> str | None: return version +def result_identities(results: Any) -> tuple[str, ...] | None: + """Hash result metadata, excluding scores and capture-local identifiers. + + Duplicate multiplicity is preserved; result order is ignored. Malformed + result lists have no usable identity coverage. + """ + if not isinstance(results, list): + return None + identities = [] + for result in results: + if not isinstance(result, dict) or not isinstance( + result.get('evaluation_name'), str + ): + return None + metadata = { + key: value + for key, value in result.items() + if key + not in { + 'score_details', + 'evaluation_timestamp', + 'evaluation_result_id', + } + } + identities.append(sha256_bytes(stable_json_bytes(metadata))) + return tuple(sorted(identities)) + + +def record_ids(data: bytes) -> RecordIds: + """Read the fields a row copies from its record; absent ones are None.""" + try: + loaded = json.loads(data) + except json.JSONDecodeError: + return RecordIds() + if not isinstance(loaded, dict): + return RecordIds() + + def text(mapping: Any, key: str) -> str | None: + value = mapping.get(key) if isinstance(mapping, dict) else None + return value if isinstance(value, str) else None + + source = loaded.get('source_metadata') + details = ( + source.get('additional_details') if isinstance(source, dict) else None + ) + cron = text(details, 'type_of_addition') == 'cron' + results = loaded.get('evaluation_results') + return RecordIds( + evaluation_id=text(loaded, 'evaluation_id'), + retrieved_timestamp=text(loaded, 'retrieved_timestamp'), + cron_adapter=text(details, 'cron_adapter') if cron else None, + cron_run_date=text(details, 'cron_run_date') if cron else None, + result_count=len(results) if isinstance(results, list) else None, + result_identities=result_identities(results), + ) + + +def backfill_ids( + rows: Sequence[Row], + fetch: Callable[[Sequence[str]], dict[str, bytes]], + *, + chunk: int = BACKFILL_CHUNK, +) -> tuple[list[Row], int, int]: + """Fill record ids of inherited rows from their published objects. + + All missing rows are read in ``legacy_path`` order, ``chunk`` objects at + a time. Returns ``(rows, filled, pending)``. + """ + missing = sorted( + (row for row in rows if not row.ids_recorded), + key=lambda row: row.legacy_path, + ) + filled: dict[str, Row] = {} + for start in range(0, len(missing), chunk): + batch = missing[start : start + chunk] + contents = fetch([row.object_path for row in batch]) + for row in batch: + data = contents[row.object_path] + if len(data) != row.size_bytes or sha256_bytes(data) != row.sha256: + raise FlatRebuildError( + f'{row.object_path}: published bytes differ from the ' + 'inherited snapshot; identity backfill cannot proceed' + ) + filled[row.object_uuid] = record_ids(data).apply(row) + return ( + [filled.get(row.object_uuid, row) for row in rows], + len(filled), + len(missing) - len(filled), + ) + + +def parse_retired(data: bytes) -> list[RetiredEntry]: + """Parse ``flat/retired.jsonl``; malformed lines raise.""" + entries: list[RetiredEntry] = [] + problems: list[str] = [] + seen: set[str] = set() + for number, line in enumerate(data.decode('utf-8').splitlines(), 1): + if not line.strip(): + continue + try: + loaded = json.loads(line) + except json.JSONDecodeError as exc: + problems.append(f'line {number}: invalid JSON ({exc})') + continue + if not isinstance(loaded, dict): + problems.append(f'line {number}: expected a JSON object') + continue + object_uuid = loaded.get('object_uuid') + reason = loaded.get('reason') + replaced_by = loaded.get('replaced_by') + if not isinstance(object_uuid, str) or not object_uuid: + problems.append(f'line {number}: object_uuid is required') + continue + if not isinstance(reason, str) or not reason.strip(): + problems.append(f'line {number}: reason is required') + continue + if replaced_by is not None and not isinstance(replaced_by, str): + problems.append(f'line {number}: replaced_by must be a uuid') + continue + if replaced_by == object_uuid: + problems.append(f'line {number}: {object_uuid} replaces itself') + continue + if object_uuid in seen: + problems.append(f'line {number}: {object_uuid} is listed twice') + continue + seen.add(object_uuid) + entries.append(RetiredEntry(object_uuid, reason, replaced_by)) + if problems: + raise FlatRebuildError( + f'{RETIRED_LIST_PATH} is malformed:\n' + '\n'.join(problems) + ) + return entries + + +def _number(text: str | None) -> Decimal | None: + if text is None or not _NUMERIC.fullmatch(text): + return None + return Decimal(text) + + +def series_key(row: Row) -> tuple[str, str, str] | None: + """Return ``(collection, cron_adapter, id prefix)``, or None. + + Only cron-ingested rows with a run date and a result count form series, and only when the + ``evaluation_id``'s last ``/`` segment is a plain (possibly fractional) + epoch number equal to the record's ``retrieved_timestamp``. + """ + if ( + row.evaluation_id is None + or row.cron_adapter is None + or row.cron_run_date is None + or row.result_count is None + ): + return None + prefix, separator, tail = row.evaluation_id.rpartition('/') + number = _number(tail) + if ( + not separator + or not prefix + or number is None + or number != _number(row.retrieved_timestamp) + ): + return None + return row.benchmark, row.cron_adapter, prefix + + +def apply_supersession( + rows: Sequence[Row], retired: Sequence[RetiredEntry] = () +) -> Supersession: + """Recompute ``superseded_by`` for every row; order is preserved. + + Explicit rule first: each retired entry points at its ``replaced_by`` + or the literal ``'retired'``, with reason ``retired``; entries naming a + uuid absent from ``rows`` are skipped and reported as stale. + + Automatic rule: rows sharing a ``series_key`` form a series. Among the + series' rows that are not retired, the newest capture (by + ``retrieved_timestamp``, then uuid) is current and every other one + points at it with reason ``newer_capture``. A series in which two rows + share a ``cron_run_date`` is ambiguous and left unmarked, as is one + whose newest capture does not contain every older result identity, + including duplicate multiplicity (reported as regressed for a human to + settle with a retire entry). Unreadable identities also prevent automatic + supersession. A collection with rows whose ids are still pending is skipped. + """ + current = { + row.object_uuid: replace( + row, superseded_by=None, superseded_reason=None + ) + for row in rows + } + stale: list[str] = [] + dangling: list[str] = [] + for entry in retired: + if entry.object_uuid not in current: + stale.append(entry.object_uuid) + continue + if entry.replaced_by is not None and entry.replaced_by not in current: + dangling.append(f'{entry.object_uuid} -> {entry.replaced_by}') + current[entry.object_uuid] = replace( + current[entry.object_uuid], + superseded_by=entry.replaced_by or 'retired', + superseded_reason='retired', + ) + pending = {row.benchmark for row in rows if not row.ids_recorded} + series: dict[tuple[str, str, str], list[Row]] = defaultdict(list) + for row in current.values(): + key = series_key(row) + if ( + key is not None + and row.benchmark not in pending + and row.superseded_reason is None + ): + series[key].append(row) + ambiguous: list[str] = [] + regressed: list[str] = [] + for key, members in sorted(series.items()): + if len(members) < 2: + continue + run_dates = [row.cron_run_date for row in members] + if len(set(run_dates)) != len(run_dates): + ambiguous.append('/'.join(key)) + continue + newest = max( + members, + key=lambda row: (_number(row.retrieved_timestamp), row.object_uuid), + ) + older = [row for row in members if row is not newest] + if any(row.result_identities is None for row in members): + regressed.append(f'{"/".join(key)}: unreadable result identities') + continue + newest_identities = Counter(newest.result_identities) + missing = [ + ( + row.object_uuid, + Counter(row.result_identities) - newest_identities, + ) + for row in older + ] + lost = [ + (uuid, sum(counts.values())) for uuid, counts in missing if counts + ] + if lost: + regressed.append( + f'{"/".join(key)}: newest {newest.object_uuid} drops result ' + 'identities from ' + + ', '.join(f'{uuid} ({count})' for uuid, count in sorted(lost)) + ) + continue + for row in older: + current[row.object_uuid] = replace( + row, + superseded_by=newest.object_uuid, + superseded_reason='newer_capture', + ) + return Supersession( + rows=tuple(current[row.object_uuid] for row in rows), + ambiguous_series=tuple(ambiguous), + regressed_series=tuple(regressed), + pending_collections=tuple(sorted(pending)), + stale_retired=tuple(stale), + dangling_replaced_by=tuple(dangling), + ) + + def plan_retire( listing: dict[str, int], live_benchmarks: set[str], *, today: str ) -> list[tuple[str, str]]: @@ -1256,9 +1660,64 @@ def orchestrate( allow_bootstrap: bool = False, dry_run: bool = False, verify: bool = False, + bootstrap_ids: bool = False, now: datetime | None = None, ) -> RebuildReport: """Plan and, unless dry, publish one flat rebuild. Returns the report.""" + bulk: FlatSnapshotReader | None = None + try: + with ExitStack() as stack: + + def read_objects(paths: Sequence[str]) -> dict[str, bytes]: + nonlocal bulk + if bulk is None: + bulk = stack.enter_context(FlatSnapshotReader(api, repo_id)) + return bulk.read_many(paths) + + def read_retired() -> bytes: + if bulk is not None: + return bulk.read(RETIRED_LIST_PATH) + return download_bytes(api, repo_id, [RETIRED_LIST_PATH])[ + RETIRED_LIST_PATH + ] + + report = _orchestrate( + api, + repo_id, + read_objects=read_objects, + read_retired=read_retired, + retain_days=retain_days, + keep_newest=keep_newest, + scan_budget=scan_budget, + allow_bootstrap=allow_bootstrap, + dry_run=dry_run, + verify=verify, + now=now, + ) + if bulk is not None: + report.bulk_revision = bulk.revision + report.bulk_pack_count = bulk.pack_count + report.bulk_lfs_download_count = bulk.lfs_download_count + report.bulk_lfs_download_paths = tuple(bulk.lfs_download_paths) + return report + except FlatSnapshotError as exc: + raise FlatRebuildError(str(exc)) from exc + + +def _orchestrate( + api: HfApi, + repo_id: str, + *, + read_objects: Callable[[Sequence[str]], dict[str, bytes]], + read_retired: Callable[[], bytes], + retain_days: int, + keep_newest: int, + scan_budget: int, + allow_bootstrap: bool, + dry_run: bool, + verify: bool, + now: datetime | None, +) -> RebuildReport: if not repo_id.strip(): raise FlatRebuildError('repo_id must not be empty') started = time.monotonic() @@ -1293,6 +1752,27 @@ def orchestrate( } ) contents = download_bytes(api, repo_id, needed) + removed_uuids = {row.object_uuid for row in diff.removed_rows} + missing_objects = sorted( + row.object_path + for row in old_rows + if not row.ids_recorded + and row.object_uuid not in removed_uuids + and row.object_path not in listing + ) + if missing_objects: + raise FlatRebuildError( + f'{len(missing_objects)} inherited object(s) are missing, so ' + 'their ids cannot be backfilled:\n' + '\n'.join(missing_objects) + ) + inherited = [ + row for row in old_rows if row.object_uuid not in removed_uuids + ] + kept, report.ids_backfilled, report.ids_pending = backfill_ids( + inherited, read_objects + ) + backfilled = {row.object_uuid: row for row in kept} + build_input = [backfilled.get(row.object_uuid, row) for row in old_rows] object_cache: dict[str, bytes] = {} def existing_object(object_path: str) -> bytes | None: @@ -1300,10 +1780,15 @@ def existing_object(object_path: str) -> bytes | None: if object_path not in listing: return None if object_path not in object_cache: - object_cache.update(download_bytes(api, repo_id, [object_path])) + if object_path.startswith('flat/objects/') and object_path.endswith( + '.json' + ): + object_cache.update(read_objects([object_path])) + else: + object_cache.update(download_bytes(api, repo_id, [object_path])) return object_cache[object_path] - build = build_rows(old_rows, diff, contents, existing_object, listing) + build = build_rows(build_input, diff, contents, existing_object, listing) if build.errors: raise FlatRebuildError( f'{len(build.errors)} record(s) cannot be flattened:\n' @@ -1311,7 +1796,22 @@ def existing_object(object_path: str) -> bytes | None: ) report.conflicts = build.conflicts report.excluded_paths = build.excluded_paths - rows = build.rows + retired: list[RetiredEntry] = [] + if RETIRED_LIST_PATH in listing: + retired = parse_retired(read_retired()) + supersession = apply_supersession(build.rows, retired) + rows = supersession.rows + report.ambiguous_series = supersession.ambiguous_series + report.regressed_series = supersession.regressed_series + report.pending_collections = supersession.pending_collections + report.stale_retired = supersession.stale_retired + report.dangling_replaced_by = supersession.dangling_replaced_by + report.superseded_captures = sum( + 1 for row in rows if row.superseded_reason == 'newer_capture' + ) + report.superseded_retired = sum( + 1 for row in rows if row.superseded_reason == 'retired' + ) if verify: pending_objects = frozenset( [ @@ -1510,8 +2010,48 @@ def summary_lines( f'{len(report.manifests_trimmed)} trimmed, ' f'{len(report.manifests_pinned)} pinned, ' f'{len(report.manifests_unscanned)} unscanned', + f'- superseded: {report.superseded_captures} by a newer capture, ' + f'{report.superseded_retired} retired', f'- manifest core sha256: `{report.manifest_core_sha256}`', ] + if report.ids_backfilled or report.ids_pending: + lines.append( + f'- evaluation_id backfill: {report.ids_backfilled} filled, ' + f'{report.ids_pending} pending' + ) + if report.bulk_revision is not None: + lines.append( + f'- published aggregate snapshot: `{report.bulk_revision}`; ' + f'{report.bulk_pack_count} Git pack transfer(s), ' + f'{report.bulk_lfs_download_count} required LFS payload(s)' + ) + if report.pending_collections: + lines.append( + f'- automatic supersession skipped in ' + f'{len(report.pending_collections)} collection(s) with ids ' + 'pending' + ) + if report.ambiguous_series: + lines.append( + f'- **ambiguous series: {len(report.ambiguous_series)}** ' + '(two captures share a cron run date; left unmarked)' + ) + if report.regressed_series: + lines.append( + f'- **regressed series: {len(report.regressed_series)}** ' + '(newest capture has fewer results than an older one; left ' + 'unmarked)' + ) + if report.stale_retired: + lines.append( + f'- **stale retire entries: {len(report.stale_retired)}** ' + f'uuid(s) in {RETIRED_LIST_PATH} are not in the index' + ) + if report.dangling_replaced_by: + lines.append( + f'- **dangling replaced_by: {len(report.dangling_replaced_by)}** ' + f'{RETIRED_LIST_PATH} entries point at uuids not in the index' + ) if report.verified_files: lines.append( f'- deep verification: {report.verified_files} data file(s) ' @@ -1550,7 +2090,13 @@ def summary_lines( ) for label, paths in ( ('Excluded records', report.conflicts), + ('Required LFS payloads', report.bulk_lfs_download_paths), ('Stale acknowledgements', stale), + ('Ambiguous series', report.ambiguous_series), + ('Regressed series', report.regressed_series), + ('Collections with ids pending', report.pending_collections), + ('Stale retire entries', report.stale_retired), + ('Dangling replaced_by', report.dangling_replaced_by), ('Retired indexes', report.indexes_retired), ('Trimmed snapshots', report.manifests_trimmed), ('Pinned snapshots', report.manifests_pinned), @@ -1622,6 +2168,14 @@ def main(argv: list[str] | None = None) -> int: 'loudly. Slow; intended for a periodic sweep, not every run.' ), ) + parser.add_argument( + '--bootstrap-ids', + action='store_true', + help=( + 'Compatibility alias: every run fills all missing inherited ' + 'ids from a pinned bulk read of published aggregate objects.' + ), + ) parser.add_argument( '--acknowledged-conflicts', type=Path, @@ -1644,6 +2198,7 @@ def main(argv: list[str] | None = None) -> int: allow_bootstrap=args.allow_bootstrap, dry_run=args.dry_run, verify=args.verify, + bootstrap_ids=args.bootstrap_ids, ) except FlatRebuildError as exc: if os.environ.get('GITHUB_ACTIONS'): diff --git a/every_eval_ever/cron/flat_snapshot.py b/every_eval_ever/cron/flat_snapshot.py new file mode 100644 index 000000000..3b48ff2c4 --- /dev/null +++ b/every_eval_ever/cron/flat_snapshot.py @@ -0,0 +1,256 @@ +"""Read published aggregate objects from one pinned, shallow Git fetch.""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess +import tempfile +from pathlib import Path, PurePosixPath +from typing import Iterable +from urllib.parse import urlsplit + +from huggingface_hub import HfApi +from huggingface_hub.utils import build_hf_headers + +_COMMIT = re.compile(r'[0-9a-fA-F]{40}(?:[0-9a-fA-F]{24})?\Z') +_LFS_VERSION = b'version https://git-lfs.github.com/spec/' +_RETIRED_PATH = 'flat/retired.jsonl' + + +class FlatSnapshotError(RuntimeError): + """The pinned published objects could not be read safely.""" + + +def _pointer(data: bytes, path: str) -> tuple[str, int] | None: + if not data.startswith(_LFS_VERSION): + return None + try: + lines = data.decode('ascii').splitlines() + except UnicodeDecodeError as exc: + raise FlatSnapshotError(f'Malformed Git LFS pointer: {path}') from exc + oid = [ + line.removeprefix('oid sha256:') + for line in lines + if line.startswith('oid sha256:') + ] + size = [ + line.removeprefix('size ') for line in lines if line.startswith('size ') + ] + if ( + lines[0] != 'version https://git-lfs.github.com/spec/v1' + or len(oid) != 1 + or re.fullmatch(r'[0-9a-f]{64}', oid[0]) is None + or len(size) != 1 + or re.fullmatch(r'[0-9]+', size[0]) is None + or any( + not line.startswith(('version ', 'oid sha256:', 'size ', 'ext-')) + for line in lines + ) + ): + raise FlatSnapshotError(f'Malformed Git LFS pointer: {path}') + return oid[0], int(size[0]) + + +class FlatSnapshotReader: + """A temporary checkout of aggregate objects and the retirement list. + + Resolve the dataset's current commit once, or supply an immutable ``revision``. + Git transfers plain blobs in one pack with LFS smudging disabled. Only a read + of an LFS-backed path downloads its payload, at the same pinned commit. + Sample sidecars are outside this reader's scope. + """ + + def __init__( + self, + api: HfApi, + repo_id: str, + *, + revision: str | None = None, + remote_url: str | None = None, + ) -> None: + self.api = api + self.repo_id = repo_id + self.revision = revision + endpoint = getattr(api, 'endpoint', 'https://huggingface.co').rstrip( + '/' + ) + self.remote_url = remote_url or f'{endpoint}/datasets/{repo_id}' + parsed = urlsplit(self.remote_url) + if parsed.username is not None or parsed.password is not None: + raise ValueError('Git remote URLs must not contain credentials') + self.pack_count = 0 + self.lfs_download_paths: list[str] = [] + self._payloads: dict[str, Path] = {} + self._temporary: tempfile.TemporaryDirectory | None = None + self._directory: Path | None = None + self._env = os.environ.copy() + self._env.update(GIT_LFS_SKIP_SMUDGE='1', GIT_TERMINAL_PROMPT='0') + config = { + 'filter.lfs.smudge': '', + 'filter.lfs.process': '', + 'filter.lfs.required': 'false', + } + self._authorization = None + if self.remote_url.startswith(endpoint + '/'): + self._authorization = build_hf_headers( + token=getattr(api, 'token', None) + ).get('authorization') + if self._authorization: + config[f'http.{endpoint}/.extraHeader'] = ( + f'Authorization: {self._authorization}' + ) + count = int(self._env.get('GIT_CONFIG_COUNT', '0')) + for key, value in config.items(): + self._env[f'GIT_CONFIG_KEY_{count}'] = key + self._env[f'GIT_CONFIG_VALUE_{count}'] = value + count += 1 + self._env['GIT_CONFIG_COUNT'] = str(count) + + @property + def lfs_download_count(self) -> int: + """Number of named LFS payload downloads requested by this reader.""" + return len(self.lfs_download_paths) + + def _git(self, *args: str) -> str: + try: + result = subprocess.run( + ['git', *args], + cwd=self._directory, + env=self._env, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise FlatSnapshotError( + 'Git is required to read published flat objects in bulk' + ) from exc + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or '').strip() + if self._authorization: + detail = detail.replace(self._authorization, '[redacted]') + raise FlatSnapshotError( + f'Git {args[0]} failed while reading flat objects: {detail}' + ) from exc + return result.stdout.strip() + + def __enter__(self) -> FlatSnapshotReader: + if self._temporary is not None or self.pack_count: + raise FlatSnapshotError('A flat snapshot reader cannot be reused') + if self.revision is None: + self.revision = self.api.repo_info( + repo_id=self.repo_id, repo_type='dataset' + ).sha + if not isinstance(self.revision, str) or not _COMMIT.fullmatch( + self.revision + ): + raise FlatSnapshotError( + 'The datastore revision must be a commit SHA' + ) + self.revision = self.revision.lower() + self._temporary = tempfile.TemporaryDirectory( + prefix='eee-flat-objects-' + ) + self._directory = Path(self._temporary.name).resolve() + try: + self._git('init', '--quiet') + self._git('fetch', '--depth', '1', self.remote_url, self.revision) + self.pack_count += 1 + fetched = self._git('rev-parse', 'FETCH_HEAD^{commit}') + if fetched != self.revision: + raise FlatSnapshotError( + f'Fetched datastore commit {fetched} differs from pin ' + f'{self.revision}' + ) + self._git('sparse-checkout', 'init', '--no-cone') + self._git( + 'sparse-checkout', + 'set', + '--no-cone', + '/flat/objects/**/*.json', + f'/{_RETIRED_PATH}', + ) + self._git('checkout', '--detach', self.revision) + if self._git('rev-parse', 'HEAD') != self.revision: + raise FlatSnapshotError( + 'Checked-out datastore commit differs from pin' + ) + except BaseException: + self.close() + raise + return self + + def close(self) -> None: + """Remove the temporary checkout; downloaded Hub cache entries remain.""" + if self._temporary is not None: + self._temporary.cleanup() + self._temporary = None + self._directory = None + self._payloads.clear() + + def __exit__(self, *_exc: object) -> None: + self.close() + + def _path(self, path: str) -> Path: + parts = PurePosixPath(path).parts + allowed = path == _RETIRED_PATH or ( + len(parts) >= 3 + and parts[:2] == ('flat', 'objects') + and parts[-1].endswith('.json') + ) + if ( + not allowed + or '..' in parts + or '\\' in path + or PurePosixPath(path).as_posix() != path + ): + raise ValueError( + f'Not a published aggregate or retirement list: {path}' + ) + if self._directory is None: + raise FlatSnapshotError( + 'Open the flat snapshot reader before reading' + ) + local = self._directory / path + if local.is_symlink() or self._directory not in local.resolve().parents: + raise FlatSnapshotError( + f'Published object must be a regular file: {path}' + ) + return local + + def exists(self, path: str) -> bool: + """Return whether a permitted path exists in the pinned checkout.""" + return self._path(path).is_file() + + def read(self, path: str) -> bytes: + """Read a published aggregate or retirement list, resolving LFS explicitly.""" + data = self._path(path).read_bytes() + pointer = _pointer(data, path) + if pointer is None: + return data + if path not in self._payloads: + self._payloads[path] = Path( + self.api.hf_hub_download( + repo_id=self.repo_id, + filename=path, + repo_type='dataset', + revision=self.revision, + ) + ) + self.lfs_download_paths.append(path) + payload = self._payloads[path].read_bytes() + digest, size = pointer + if ( + len(payload) != size + or hashlib.sha256(payload).hexdigest() != digest + ): + raise FlatSnapshotError( + f'Git LFS payload failed SHA256/size check: {path}' + ) + return payload + + def read_many(self, paths: Iterable[str]) -> dict[str, bytes]: + """Read each requested aggregate without retaining an extra byte cache.""" + return {path: self.read(path) for path in paths} diff --git a/every_eval_ever/eval_types.py b/every_eval_ever/eval_types.py index c9e0946d6..f44bcf3eb 100644 --- a/every_eval_ever/eval_types.py +++ b/every_eval_ever/eval_types.py @@ -364,7 +364,8 @@ class SourceDataHf(BaseModel): ) hf_split: str | None = Field(None, description='One of train, val or test.') samples_number: int | None = Field( - None, description='Number of samples in the dataset' + None, + description='Number of items the reported score covers (the evaluated subset when it differs from the dataset size)', ) sample_ids: list[str] | None = Field( None, description='Array of sample ids used for evaluation' diff --git a/every_eval_ever/schemas/eval.schema.json b/every_eval_ever/schemas/eval.schema.json index ed4276611..f4cdf57cb 100644 --- a/every_eval_ever/schemas/eval.schema.json +++ b/every_eval_ever/schemas/eval.schema.json @@ -752,7 +752,7 @@ }, "samples_number": { "type": "integer", - "description": "Number of samples in the dataset" + "description": "Number of items the reported score covers (the evaluated subset when it differs from the dataset size)" }, "sample_ids": { "type": "array", diff --git a/tests/data/arc_agi/history/v2.json b/tests/data/arc_agi/history/v2.json new file mode 100644 index 000000000..15bf4442a --- /dev/null +++ b/tests/data/arc_agi/history/v2.json @@ -0,0 +1,126 @@ +{ + "revision": "026789c1c12a4c34580a32e84dcaf5630d7e8f31", + "paths": { + "claude-opus-4-5-20251101-thinking-16k/0934a4d8.json": 1, + "claude-opus-4-5-20251101-thinking-16k/135a2760.json": 1, + "claude-opus-4-5-20251101-thinking-16k/136b0064.json": 1, + "claude-opus-4-5-20251101-thinking-16k/13e47133.json": 1, + "claude-opus-4-5-20251101-thinking-16k/142ca369.json": 1, + "claude-opus-4-5-20251101-thinking-16k/16b78196.json": 1, + "claude-opus-4-5-20251101-thinking-16k/16de56c4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/1818057f.json": 1, + "claude-opus-4-5-20251101-thinking-16k/195c6913.json": 1, + "claude-opus-4-5-20251101-thinking-16k/1ae2feb7.json": 1, + "claude-opus-4-5-20251101-thinking-16k/20270e3b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/20a9e565.json": 1, + "claude-opus-4-5-20251101-thinking-16k/21897d95.json": 1, + "claude-opus-4-5-20251101-thinking-16k/221dfab4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/247ef758.json": 1, + "claude-opus-4-5-20251101-thinking-16k/269e22fb.json": 1, + "claude-opus-4-5-20251101-thinking-16k/271d71e2.json": 1, + "claude-opus-4-5-20251101-thinking-16k/28a6681f.json": 1, + "claude-opus-4-5-20251101-thinking-16k/291dc1e1.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2b83f449.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2ba387bc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2c181942.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2d0172a1.json": 1, + "claude-opus-4-5-20251101-thinking-16k/31f7f899.json": 1, + "claude-opus-4-5-20251101-thinking-16k/332f06d7.json": 1, + "claude-opus-4-5-20251101-thinking-16k/35ab12c3.json": 1, + "claude-opus-4-5-20251101-thinking-16k/36a08778.json": 1, + "claude-opus-4-5-20251101-thinking-16k/38007db0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/3a25b0d8.json": 1, + "claude-opus-4-5-20251101-thinking-16k/3dc255db.json": 1, + "claude-opus-4-5-20251101-thinking-16k/3e6067c3.json": 1, + "claude-opus-4-5-20251101-thinking-16k/409aa875.json": 1, + "claude-opus-4-5-20251101-thinking-16k/446ef5d2.json": 1, + "claude-opus-4-5-20251101-thinking-16k/45a5af55.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4a21e3da.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4c3d4a41.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4c416de3.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4c7dc4dd.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4e34c42c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/53fb4810.json": 1, + "claude-opus-4-5-20251101-thinking-16k/5545f144.json": 1, + "claude-opus-4-5-20251101-thinking-16k/581f7754.json": 1, + "claude-opus-4-5-20251101-thinking-16k/58490d8a.json": 1, + "claude-opus-4-5-20251101-thinking-16k/58f5dbd5.json": 1, + "claude-opus-4-5-20251101-thinking-16k/5961cc34.json": 1, + "claude-opus-4-5-20251101-thinking-16k/5dbc8537.json": 1, + "claude-opus-4-5-20251101-thinking-16k/62593bfd.json": 1, + "claude-opus-4-5-20251101-thinking-16k/64efde09.json": 1, + "claude-opus-4-5-20251101-thinking-16k/65b59efc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/67e490f4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/6e453dd6.json": 1, + "claude-opus-4-5-20251101-thinking-16k/6e4f6532.json": 1, + "claude-opus-4-5-20251101-thinking-16k/6ffbe589.json": 1, + "claude-opus-4-5-20251101-thinking-16k/71e489b6.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7491f3cf.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7666fa5d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/78332cb0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b0280bc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b3084d4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b5033c1.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b80bb43.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7c66cb00.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7ed72f31.json": 1, + "claude-opus-4-5-20251101-thinking-16k/800d221b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/80a900e0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8698868d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/88bcf3b4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/88e364bc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/89565ca0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/898e7135.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8b7bacbf.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8b9c3697.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8e5c0c38.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8f215267.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8f3a5a89.json": 1, + "claude-opus-4-5-20251101-thinking-16k/9385bd28.json": 1, + "claude-opus-4-5-20251101-thinking-16k/97d7923e.json": 1, + "claude-opus-4-5-20251101-thinking-16k/981571dc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/9aaea919.json": 1, + "claude-opus-4-5-20251101-thinking-16k/9bbf930d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a251c730.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a25697e4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a32d8b75.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a395ee82.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a47bf94d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a6f40cea.json": 1, + "claude-opus-4-5-20251101-thinking-16k/aa4ec2a5.json": 1, + "claude-opus-4-5-20251101-thinking-16k/abc82100.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b0039139.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b10624e5.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b5ca7ac4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b6f77b65.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b99e7126.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b9e38dc0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/bf45cf4b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/c4d067a0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/c7f57c3e.json": 1, + "claude-opus-4-5-20251101-thinking-16k/cb2d8a2c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/cbebaa4b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/d35bdbdc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/d59b0160.json": 1, + "claude-opus-4-5-20251101-thinking-16k/d8e07eb2.json": 1, + "claude-opus-4-5-20251101-thinking-16k/da515329.json": 1, + "claude-opus-4-5-20251101-thinking-16k/db0c5428.json": 1, + "claude-opus-4-5-20251101-thinking-16k/db695cfb.json": 1, + "claude-opus-4-5-20251101-thinking-16k/dbff022c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/dd6b8c4b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/de809cff.json": 1, + "claude-opus-4-5-20251101-thinking-16k/dfadab01.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e12f9a14.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e3721c99.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e376de54.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e8686506.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e87109e9.json": 1, + "claude-opus-4-5-20251101-thinking-16k/edb79dae.json": 1, + "claude-opus-4-5-20251101-thinking-16k/eee78d87.json": 1, + "claude-opus-4-5-20251101-thinking-16k/f560132c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/f931b4a8.json": 1, + "claude-opus-4-5-20251101-thinking-16k/faa9f03d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/fc7cae8d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/results.json": 1 + } +} diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/28a6681f.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/28a6681f.json new file mode 100644 index 000000000..8d0b4b003 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/28a6681f.json @@ -0,0 +1,320 @@ +[ + { + "attempt_1": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5, + 0, + 5 + ], + [ + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5 + ], + [ + 3, + 3, + 0, + 1, + 1, + 1, + 5, + 5, + 0, + 5 + ], + [ + 3, + 3, + 3, + 1, + 1, + 5, + 5, + 5, + 0, + 5 + ], + [ + 3, + 3, + 3, + 3, + 5, + 5, + 5, + 5, + 0, + 5 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T16:13:14.383973+00:00", + "end_timestamp": "2025-11-21T16:15:31.704129+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2297, + "completion_tokens": 11646, + "total_tokens": 13943, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 11646, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.011485, + "completion_cost": 0.29115, + "reasoning_cost": 0.0, + "total_cost": 0.30263500000000004 + }, + "task_id": "28a6681f", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": false + }, + "attempt_2": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5, + 1, + 5 + ], + [ + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5 + ], + [ + 3, + 3, + 0, + 0, + 0, + 0, + 5, + 5, + 1, + 5 + ], + [ + 3, + 3, + 3, + 0, + 0, + 5, + 5, + 5, + 1, + 5 + ], + [ + 3, + 3, + 3, + 3, + 5, + 5, + 5, + 5, + 1, + 5 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T16:15:31.705226+00:00", + "end_timestamp": "2025-11-21T16:17:25.495164+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2297, + "completion_tokens": 10390, + "total_tokens": 12687, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 10390, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.011485, + "completion_cost": 0.25975000000000004, + "reasoning_cost": 0.0, + "total_cost": 0.27123500000000006 + }, + "task_id": "28a6681f", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": false + } + } +] diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/58490d8a.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/58490d8a.json new file mode 100644 index 000000000..c866e2bb5 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/58490d8a.json @@ -0,0 +1,314 @@ +[ + { + "attempt_1": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 2, + 0, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:14:14.600064+00:00", + "end_timestamp": "2025-11-21T17:15:27.470540+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 6488, + "completion_tokens": 7239, + "total_tokens": 13727, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7239, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.032440000000000004, + "completion_cost": 0.180975, + "reasoning_cost": 0.0, + "total_cost": 0.213415 + }, + "task_id": "58490d8a", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": true + }, + "attempt_2": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 2, + 0, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:15:27.474466+00:00", + "end_timestamp": "2025-11-21T17:16:52.086340+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 6488, + "completion_tokens": 7541, + "total_tokens": 14029, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7541, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.032440000000000004, + "completion_cost": 0.188525, + "reasoning_cost": 0.0, + "total_cost": 0.220965 + }, + "task_id": "58490d8a", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": true + } + } +] diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/7b5033c1.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/7b5033c1.json new file mode 100644 index 000000000..3d81da5a5 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/7b5033c1.json @@ -0,0 +1,230 @@ +[ + { + "attempt_1": { + "answer": [ + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 3 + ], + [ + 3 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:32:14.686501+00:00", + "end_timestamp": "2025-11-21T17:33:34.203070+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 7453, + "total_tokens": 10074, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7453, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.013105, + "completion_cost": 0.18632500000000002, + "reasoning_cost": 0.0, + "total_cost": 0.19943000000000002 + }, + "task_id": "7b5033c1", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": true + }, + "attempt_2": { + "answer": [ + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:33:34.208119+00:00", + "end_timestamp": "2025-11-21T17:34:59.171003+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 7616, + "total_tokens": 10237, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7616, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.013105, + "completion_cost": 0.1904, + "reasoning_cost": 0.0, + "total_cost": 0.20350500000000002 + }, + "task_id": "7b5033c1", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": false + } + } +] diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/results.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/results.json new file mode 100644 index 000000000..5acc494d3 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/results.json @@ -0,0 +1,41 @@ +{ + "score": 2.0, + "total_tasks": 3, + "total_cost": 1.4111850000000001, + "total_attempts": 6, + "avg_cost_per_task": 0.47039500000000006, + "avg_cost_per_attempt": 0.23519750000000003, + "avg_output_tokens_per_task": 17295.0, + "avg_total_tokens_per_task": 24899.0, + "avg_duration_per_task": 191.02396566666667, + "task_results": { + "28a6681f": { + "score": 0.0, + "cost": 0.5738700000000001, + "attempts": 2, + "output_tokens": 22036, + "total_tokens": 26630, + "duration": 251.110094, + "num_attempts_with_empty_list": 0 + }, + "58490d8a": { + "score": 1.0, + "cost": 0.43438, + "attempts": 2, + "output_tokens": 14780, + "total_tokens": 27756, + "duration": 157.48235, + "num_attempts_with_empty_list": 0 + }, + "7b5033c1": { + "score": 1.0, + "cost": 0.40293500000000004, + "attempts": 2, + "output_tokens": 15069, + "total_tokens": 20311, + "duration": 164.479453, + "num_attempts_with_empty_list": 0 + } + }, + "num_attempts_with_empty_list": 0 +} diff --git a/tests/data/hfopenllm_v2/contents/data/train-00000-of-00001.parquet b/tests/data/hfopenllm_v2/contents/data/train-00000-of-00001.parquet new file mode 100644 index 000000000..4622cc87b Binary files /dev/null and b/tests/data/hfopenllm_v2/contents/data/train-00000-of-00001.parquet differ diff --git a/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-01-16T02-47-58.653318.json b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-01-16T02-47-58.653318.json new file mode 100644 index 000000000..1995549ff --- /dev/null +++ b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-01-16T02-47-58.653318.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": null, + "model_args": "pretrained=Daemontatox/DocumentCogito,revision=23dcfc6bf91d84db1c977b151fd0923270d3e3ef,trust_remote_code=False,dtype=bfloat16,parallelize=False", + "model_dtype": "torch.bfloat16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a781a6b", + "model_name": "Daemontatox/DocumentCogito", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 187, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 146, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 178, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 250, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 198, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 546, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 448, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 541, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 307, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 123, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 132, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 280, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 154, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 193, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 135, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 12032, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 250, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 256, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 250, + "original": 250 + } + }, + "transformers_version": "4.48.0" +} diff --git a/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-02-13T18-27-04.338360.json b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-02-13T18-27-04.338360.json new file mode 100644 index 000000000..1995549ff --- /dev/null +++ b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-02-13T18-27-04.338360.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": null, + "model_args": "pretrained=Daemontatox/DocumentCogito,revision=23dcfc6bf91d84db1c977b151fd0923270d3e3ef,trust_remote_code=False,dtype=bfloat16,parallelize=False", + "model_dtype": "torch.bfloat16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a781a6b", + "model_name": "Daemontatox/DocumentCogito", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 187, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 146, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 178, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 250, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 198, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 546, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 448, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 541, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 307, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 123, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 132, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 280, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 154, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 193, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 135, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 12032, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 250, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 256, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 250, + "original": 250 + } + }, + "transformers_version": "4.48.0" +} diff --git a/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-03-09T04-35-49.828129.json b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-03-09T04-35-49.828129.json new file mode 100644 index 000000000..c5be6dfb3 --- /dev/null +++ b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-03-09T04-35-49.828129.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": null, + "model_args": "pretrained=Daemontatox/DocumentCogito,revision=9bdbfd8f330754c4103822ce180e0e3e3ce0973e,trust_remote_code=False,dtype=float16,parallelize=False", + "model_dtype": "torch.float16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a781a6b", + "model_name": "Daemontatox/DocumentCogito", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 187, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 146, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 178, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 250, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 198, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 546, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 448, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 541, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 307, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 123, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 132, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 280, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 154, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 193, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 135, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 12032, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 250, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 256, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 250, + "original": 250 + } + }, + "transformers_version": "4.49.0" +} diff --git a/tests/data/hfopenllm_v2/results/databricks/dbrx-base/results_2025-02-13T18-27-04.338360.json b/tests/data/hfopenllm_v2/results/databricks/dbrx-base/results_2025-02-13T18-27-04.338360.json new file mode 100644 index 000000000..74fd8b78e --- /dev/null +++ b/tests/data/hfopenllm_v2/results/databricks/dbrx-base/results_2025-02-13T18-27-04.338360.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": 100.0, + "model_args": "pretrained=databricks/dbrx-base,revision=main,dtype=float16,trust_remote_code=False,parallelize=True", + "model_dtype": "torch.float16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a4d01d5f", + "model_name": "databricks/dbrx-base", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 100, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 100, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 100, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 100, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 100, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 100, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 100, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 100, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 100, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 100, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 100, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 100, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 100, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 100, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 100, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 100, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 100, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 100, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 100, + "original": 250 + } + }, + "transformers_version": "4.40.2" +} diff --git a/tests/data/vals_ai/mmmu_page.html b/tests/data/vals_ai/mmmu_page.html new file mode 100644 index 000000000..1920bea00 --- /dev/null +++ b/tests/data/vals_ai/mmmu_page.html @@ -0,0 +1,38 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

We adhered closely to the official MMMU Pro evaluation protocol with the following implementation details:

+
    +
  1. Prompt Structure: We used the chain-of-thought prompt from the original MMMU repository. Each question-answer set followed this format:
    2. +
+
Which of the following best explains the overall trend shown in the <image>?
+A. Migrations to areas of Central Asia for resettlement
+B. The spread of pathogens across the Silk Road
+C. Invasions by Mongol tribes
+D. Large-scale famine due to crop failures
+Answer the preceding multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering.
+
+
    +
  1. +

    Image Processing: All <image> tags were replaced with actual image bytes during model inference, replicating the original methodology exactly.

    +
    2. +
  2. +

    Standardization: We established a consistent baseline by using a default configuration of 8192 maximum output tokens for all models to ensure that outputs were not truncated. All models were ran with a temperature of 0.

    +
    3. +
  3. +

    Parsing Adaptation: We modified the answer extraction regex to handle markdown output from some models, ensuring reliable parsing across all responses.

    +
    4. +
  4. +

    Statistical Validity: This was a pass@1 evaluation (one attempt per question), with the large dataset size (1,700 questions). We found a standard deviation approximately 1% (calculated using the methodology from Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations)

    +
    5. +
  5. +

    Thinking Models Configuration: For models with “thinking” capabilities (Claude 3.7 Sonnet Thinking, o1, etc.), we set the maximum output token limit to 16,384 to accommodate extended reasoning.

    +
    6. +

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/reverse_eng_page.html b/tests/data/vals_ai/reverse_eng_page.html new file mode 100644 index 000000000..5d2f23523 --- /dev/null +++ b/tests/data/vals_ai/reverse_eng_page.html @@ -0,0 +1,15 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

SRE Bench pairs 19 in-house programs (averaging over 16,915.8 lines of code, spanning network protocols, firmware, games, file-format recovery, and malware) with in-house protection settings from a 27K-LoC anti-analysis suite of more than ten state-of-the-art protections, over half of which have no public implementations.

+SRE Bench development and evaluation pipeline: source code across languages and RE domains is compiled with protection layers into binaries, which agents reverse engineer in a workspace with auxiliary material and a deterministic grading server, with adversarial reward-hacking audits feeding back into development +

This yields 262 contamination-free instances with realistic difficulty. Each instance defines six scenario-specific, deterministically verifiable tasks that cover the core capabilities required for program understanding, for a total of 1,572 RE tasks.

+

Compared with existing RE-adjacent benchmarks — which rely on existing CTF challenges or toy programs with easy or textbook protection schemes — SRE Bench is RE-centric, contamination-controlled, and built from real-world-scale software with real-world protection complexity.

+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/sage_page.html b/tests/data/vals_ai/sage_page.html new file mode 100644 index 000000000..c193dbf77 --- /dev/null +++ b/tests/data/vals_ai/sage_page.html @@ -0,0 +1,23 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

The models are given student work samples and corresponding rubrics, expressed as a set of potential point deductions. We ask the models to determine which deductions apply to the given work.

+

All student work in the dataset consists of handwritten mathematical solutions, which adds an additional layer of complexity as models must be able to parse handwritten text and mathematical notation.

+

The evaluation process requires models to:

+
    +
  1. Parse and understand the rubric criteria
    2. +
  2. Analyze the student’s mathematical work and reasoning
    3. +
  3. Apply appropriate scoring based on the rubric guidelines
    4. +
  4. Provide clear justification for their scoring decisions
    5. +
+

This methodology tests not only the models’ mathematical understanding but also their ability to apply consistent evaluation standards—a critical skill for educational applications.

+

All models are evaluated with temperature 1, and produce at most 30K tokens (the length of a short book - more than enough to adequately grade student work).

+
+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/sage_page_statement_removed.html b/tests/data/vals_ai/sage_page_statement_removed.html new file mode 100644 index 000000000..fd9dd178d --- /dev/null +++ b/tests/data/vals_ai/sage_page_statement_removed.html @@ -0,0 +1,23 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

The models are given student work samples and corresponding rubrics, expressed as a set of potential point deductions. We ask the models to determine which deductions apply to the given work.

+

All student work in the dataset consists of handwritten mathematical solutions, which adds an additional layer of complexity as models must be able to parse handwritten text and mathematical notation.

+

The evaluation process requires models to:

+
    +
  1. Parse and understand the rubric criteria
    2. +
  2. Analyze the student’s mathematical work and reasoning
    3. +
  3. Apply appropriate scoring based on the rubric guidelines
    4. +
  4. Provide clear justification for their scoring decisions
    5. +
+

This methodology tests not only the models’ mathematical understanding but also their ability to apply consistent evaluation standards—a critical skill for educational applications.

+

All models are evaluated with default settings (the length of a short book - more than enough to adequately grade student work).

+
+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/saved_pages/README.md b/tests/data/vals_ai/saved_pages/README.md new file mode 100644 index 000000000..5a0cccdbe --- /dev/null +++ b/tests/data/vals_ai/saved_pages/README.md @@ -0,0 +1,12 @@ +# Saved Vals methodology pages + +Losslessly compressed HTML supplied for offline verification. Each file is an exact +copy of the corresponding page saved on 2026-10-21; runtime code does not read these +fixtures. The methodology tests verify the source sentences, not snapshot row counts. + +| Page | SHA-256 of decompressed HTML | +| --- | --- | +| https://www.vals.ai/benchmarks/sage | `fb3f238f596cc1402e066d95d95a2cb71936fb2f0350ba3a3376dba78714ca6b` | +| https://www.vals.ai/benchmarks/medcode | `e64e0ee3a4fdac087e0580855eace09192332b5e91b82771e5429cb84a649852` | +| https://www.vals.ai/benchmarks/medscribe | `e4250edb7f804703dfb56720f014433112fc4e583a2e4b6032c8b12b3c9d3c66` | +| https://www.vals.ai/benchmarks/mmmu | `403785fcf9a50f899cb9f1d75026bada929f9d37e2b9296dcd1331494631ed87` | diff --git a/tests/data/vals_ai/saved_pages/medcode.html.gz b/tests/data/vals_ai/saved_pages/medcode.html.gz new file mode 100644 index 000000000..d63e79d16 Binary files /dev/null and b/tests/data/vals_ai/saved_pages/medcode.html.gz differ diff --git a/tests/data/vals_ai/saved_pages/medscribe.html.gz b/tests/data/vals_ai/saved_pages/medscribe.html.gz new file mode 100644 index 000000000..eb105c881 Binary files /dev/null and b/tests/data/vals_ai/saved_pages/medscribe.html.gz differ diff --git a/tests/data/vals_ai/saved_pages/mmmu.html.gz b/tests/data/vals_ai/saved_pages/mmmu.html.gz new file mode 100644 index 000000000..1c805653e Binary files /dev/null and b/tests/data/vals_ai/saved_pages/mmmu.html.gz differ diff --git a/tests/data/vals_ai/saved_pages/sage.html.gz b/tests/data/vals_ai/saved_pages/sage.html.gz new file mode 100644 index 000000000..7085d9fc2 Binary files /dev/null and b/tests/data/vals_ai/saved_pages/sage.html.gz differ diff --git a/tests/data/vals_ai/srebench_page.html b/tests/data/vals_ai/srebench_page.html new file mode 100644 index 000000000..5d2f23523 --- /dev/null +++ b/tests/data/vals_ai/srebench_page.html @@ -0,0 +1,15 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

SRE Bench pairs 19 in-house programs (averaging over 16,915.8 lines of code, spanning network protocols, firmware, games, file-format recovery, and malware) with in-house protection settings from a 27K-LoC anti-analysis suite of more than ten state-of-the-art protections, over half of which have no public implementations.

+SRE Bench development and evaluation pipeline: source code across languages and RE domains is compiled with protection layers into binaries, which agents reverse engineer in a workspace with auxiliary material and a deterministic grading server, with adversarial reward-hacking audits feeding back into development +

This yields 262 contamination-free instances with realistic difficulty. Each instance defines six scenario-specific, deterministically verifiable tasks that cover the core capabilities required for program understanding, for a total of 1,572 RE tasks.

+

Compared with existing RE-adjacent benchmarks — which rely on existing CTF challenges or toy programs with easy or textbook protection schemes — SRE Bench is RE-centric, contamination-controlled, and built from real-world-scale software with real-world protection complexity.

+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/web_search_page.html b/tests/data/vals_ai/web_search_page.html new file mode 100644 index 000000000..fda3ce6bf --- /dev/null +++ b/tests/data/vals_ai/web_search_page.html @@ -0,0 +1,55 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

Agents run on the same harness across both domains, with the web-search tool (native/Exa) as the variable under test. Two things distinguish these runs from the original FAB v2 and Legal Research Benchmark evaluations: the tool is swapped (native vs. Exa), and the domain-specific search tools those benchmarks provide (SEC EDGAR for finance, CourtListener for legal, and the like) are removed, leaving the web-search tool as the agent’s only means of gathering external information. Runs are orchestrated through Valkyrie with model calls routed through model-library.

+

Grading

+
    +
  • Legal Research: each task in the Legal Research Benchmark carries a rubric of required elements written by legal experts; rubrics range from 1 to 22 items per task (mean 9.47). All-pass is our primary metric: a task counts as correct only if every rubric item is satisfied. Grading is performed by a single LLM judge (GPT-5.4).
    • +
  • Finance Analysis: tasks use FAB v2’s dealbreaker-gated Partial Credit as the primary metric, category-balanced across the dataset’s nine question categories. A subset of checks in each question is flagged as dealbreakers—load-bearing facts or numbers required for a satisfactory answer—and failing any dealbreaker means the response scores 0% for that question regardless of everything else. All-pass (100% only if every check passes) is reported as a secondary metric. Grading uses the same three-judge jury as FAB v2: GPT-5.4, Gemini-3.1-Pro, and Claude Sonnet 4.6.
    • +
+

For the full rubrics, jury setup, and scoring rules behind any specific grading decision, refer to the original FAB v2 and Legal Research Benchmark methodologies.

+

Dataset

+
    +
  • Legal: 208 expert-authored legal research tasks, each tagged by the reasoning skill it tests (a task can test more than one): Statutory Interpretation, Application of Precedent to a Fact Pattern, Doctrinal Test Identification, Regulatory Framework Interpretation, Interaction of Multiple Legal Regimes, Cross-Jurisdiction Comparison, Timeline/Doctrinal Evolution, Validity of Precedent, Resolving Conflicting Precedent, and Consensus vs. Disagreement Across Courts.
    • +
  • Finance: 450 finance-analyst questions across nine categories, scored with FAB v2’s category-balanced methodology and reflecting real equity-research workflows: General Qualitative Analysis, General Quantitative Analysis, Market Analysis, Comparables, Precedents, Adjustments, Earnings Analysis, Disclosure Analysis, and Financial Modeling.
    • +
+

Statistical Analysis

+

To establish whether the two search tools differ statistically, we fit a mixed-effects model per domain, with search tool as a fixed effect (native as the reference level) and a random intercept per task absorbing task-level difficulty: a logistic model for legal’s All-pass metric (all_pass ~ search_tool + (1 | task_id)) and a linear model for finance’s primary metric (score ~ search_tool + (1 | task_id)). Each model pools the per-task, per-model observations underlying the Results charts above. The domain-level significance cited in the Key Takeaways comes from these models.

+

The fixed effect of search tool (Exa relative to native) in each model:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DomainOutcome (model)Exa effectStd. errorStatisticp
FinanceWeighted score, 0-100 (linear)+6.51 points0.98t = 6.62< 0.001
LegalAll-pass, 0/1 (logistic)−0.02 log-odds0.13z = −0.170.86

Results

+

Not methodology.

+
+ diff --git a/tests/test_arc_agi_adapter.py b/tests/test_arc_agi_adapter.py index 806377207..8626b7227 100644 --- a/tests/test_arc_agi_adapter.py +++ b/tests/test_arc_agi_adapter.py @@ -1,12 +1,29 @@ -"""Unit tests for the ARC Prize leaderboard adapter.""" +"""Unit tests for the ARC Prize leaderboard adapter. + +The run-log fixture under ``tests/data/arc_agi/v2/`` is three task files +(``7b5033c1``, ``58490d8a``, ``28a6681f``) from the directory +``claude-opus-4-5-20251101-thinking-16k`` of the Hugging Face dataset +``arcprize/arc_agi_v2_public_eval`` at commit +``026789c1c12a4c34580a32e84dcaf5630d7e8f31``, with each attempt's +``choices`` and ``reasoning_summary`` removed. Its ``results.json`` keeps +those three ``task_results`` entries verbatim and recomputes the run totals +from them (score 2.0 of 3 tasks, average cost per task 0.470395). +``tests/data/arc_agi/history/v2.json`` is that directory's path history at +the same commit (``git log --name-only --no-renames`` over a blobless clone, +restricted to the directory): every path committed once. +""" from __future__ import annotations import json +import shutil +import subprocess +from pathlib import Path import pytest -from every_eval_ever.adapters.arc_agi import adapter +from every_eval_ever.adapters.arc_agi import adapter, run_logs +from every_eval_ever.adapters.arc_agi.run_logs import StatedFloat from every_eval_ever.eval_types import EvaluationLog @@ -258,6 +275,7 @@ def test_run_writes_records_and_replays_offline(tmp_path): str(payload_path), '--output-dir', str(out_dir), + '--no-run-logs', ] ) written = adapter.run(args) @@ -267,3 +285,556 @@ def test_run_writes_records_and_replays_offline(tmp_path): record = json.loads(files[0].read_text(encoding='utf-8')) assert record['schema_version'] assert record['evaluation_id'].startswith('arc-agi/') + + +RUN_FIXTURE = Path(__file__).parent / 'data' / 'arc_agi' +RUN_ID = 'claude-opus-4-5-20251101-thinking-16k' +ALIAS_ID = 'anthropic-claude-opus-4-5-20251101-thinking-16k' +V2 = run_logs.RUN_DATASETS['v2_Public_Eval'] +RETRIEVED = '1790000000.0' +FIXTURE_HISTORY = run_logs.load_history_file( + RUN_FIXTURE / 'history' / 'v2.json', V2.revision +) + + +def run_log_payload(**v2_row: object) -> dict: + row = { + 'datasetId': 'v2_Public_Eval', + 'modelId': RUN_ID, + 'score': 0.6667, + 'costPerTask': 0.47, + 'display': True, + } + row.update(v2_row) + return { + 'datasets': [], + 'providers': [{'id': 'Anthropic', 'displayName': 'Anthropic'}], + 'models': [ + {'id': model_id, 'providerId': 'Anthropic'} + for model_id in (RUN_ID, ALIAS_ID) + ], + 'evaluations': [ + row, + { + 'datasetId': 'v1_Public_Eval', + 'modelId': RUN_ID, + 'score': 0.6667, + 'costPerTask': 0.47, + 'display': True, + }, + { + 'datasetId': 'v2_Semi_Private', + 'modelId': RUN_ID, + 'score': 0.6667, + 'costPerTask': 0.47, + 'display': True, + }, + ], + } + + +@pytest.fixture +def runs_dir(tmp_path: Path) -> Path: + root = tmp_path / 'runs' + shutil.copytree(RUN_FIXTURE, root) + return root + + +def run_dir(root: Path, name: str = RUN_ID) -> Path: + return root / 'v2' / name + + +def edit_attempts(directory: Path, edit, task_id: str | None = None) -> None: + """Apply ``edit(metadata, task_id, attempt_key)`` to every recorded attempt.""" + for path in sorted(directory.glob('*.json')): + if path.name == 'results.json' or task_id not in (None, path.stem): + continue + pairs = json.loads(path.read_text(encoding='utf-8')) + for pair in pairs: + for key, attempt in pair.items(): + if attempt is not None: + edit(attempt['metadata'], path.stem, key) + path.write_text(json.dumps(pairs), encoding='utf-8') + + +def convert_with_logs( + payload: dict, + root: Path, + history: dict[str, int] = FIXTURE_HISTORY, + retrieved: str = RETRIEVED, +) -> tuple[EvaluationLog, run_logs.RunLogs]: + logs = run_logs.RunLogs( + { + dataset_id: run_logs.RunLogSource( + dataset, root / dataset.subdir, dataset.revision, history + ) + for dataset_id, dataset in run_logs.RUN_DATASETS.items() + } + ) + result = adapter.convert_logs( + payload, retrieved_timestamp=retrieved, run_logs=logs + ) + result.raise_if_incomplete() + [(log, _, _)] = result.records + return log, logs + + +def result_by_id(log: EvaluationLog, result_id: str): + return next( + r for r in log.evaluation_results if r.evaluation_result_id == result_id + ) + + +def test_accepted_run_fills_the_score_result(runs_dir): + log, logs = convert_with_logs(run_log_payload(), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + config = score.generation_config + assert config.generation_args.max_tokens == 64000 + assert config.generation_args.temperature is None + assert config.additional_details == { + 'cap_key': 'max_tokens', + 'request_log': f'{V2.repo_id}@{V2.revision}/{RUN_ID}', + } + assert score.source_data.additional_details['scored_tasks'] == '3' + assert score.score_details.uncertainty.num_samples == 3 + assert [(o.dataset_id, o.rejection) for o in logs.outcomes] == [ + ('v2_Public_Eval', None) + ] + + +def test_cost_result_and_other_sets_are_untouched(runs_dir): + log, _ = convert_with_logs(run_log_payload(), runs_dir) + for result_id in ( + 'v2_Public_Eval::cost_per_task', + 'v1_Public_Eval::score', + 'v1_Public_Eval::cost_per_task', + 'v2_Semi_Private::score', + 'v2_Semi_Private::cost_per_task', + ): + result = result_by_id(log, result_id) + assert result.generation_config is None, result_id + assert 'scored_tasks' not in result.source_data.additional_details + assert result.score_details.uncertainty is None + + +def test_v2_run_never_fills_v1_even_when_v1_has_no_directory(runs_dir): + (runs_dir / 'v1').mkdir() + log, logs = convert_with_logs(run_log_payload(), runs_dir) + assert result_by_id(log, 'v1_Public_Eval::score').generation_config is None + assert {o.dataset_id for o in logs.outcomes} == {'v2_Public_Eval'} + assert logs.unjoined == [('v1_Public_Eval', RUN_ID)] + assert adapter.run_log_report(logs)['gate_counts'] == { + 'filled': 1, + 'no_run_directory': 1, + } + + +def _set_cap(metadata, task_id, key): + if (task_id, key) == ('28a6681f', 'attempt_2'): + metadata['kwargs']['max_tokens'] = 32000 + + +def _mix_cap_key(metadata, task_id, key): + if (task_id, key) == ('28a6681f', 'attempt_2'): + metadata['kwargs']['max_output_tokens'] = metadata['kwargs'].pop( + 'max_tokens' + ) + + +def _wrong_test_id(metadata, task_id, key): + if (task_id, key) == ('58490d8a', 'attempt_1'): + metadata['test_id'] = 'some-other-run' + + +@pytest.mark.parametrize( + ('row', 'mutate', 'gate'), + [ + ({'score': 0.6}, None, 'score_mismatch'), + ({'costPerTask': 0.48}, None, 'cost_mismatch'), + ({}, lambda d: edit_attempts(d, _set_cap), 'cap_conflict'), + ({}, lambda d: edit_attempts(d, _mix_cap_key), 'cap_key_mixed'), + ({}, lambda d: (d / 'results.json').unlink(), 'no_results_file'), + ({}, lambda d: edit_attempts(d, _wrong_test_id), 'test_id_mismatch'), + ], + ids=[ + 'score', + 'cost', + 'cap_conflict', + 'mixed_cap_keys', + 'no_results', + 'test_id', + ], +) +def test_failed_gate_leaves_the_result_unfilled(runs_dir, row, mutate, gate): + if mutate is not None: + mutate(run_dir(runs_dir)) + log, _ = convert_with_logs(run_log_payload(**row), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config.generation_args is None + assert score.generation_config.additional_details == { + 'request_args_unavailable': gate, + 'request_log': f'{V2.repo_id}@{V2.revision}/{RUN_ID}', + } + assert 'scored_tasks' not in score.source_data.additional_details + cost = result_by_id(log, 'v2_Public_Eval::cost_per_task') + assert cost.generation_config is None + + +def test_score_is_compared_at_the_row_precision(runs_dir): + log, _ = convert_with_logs(run_log_payload(score=0.667), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config.generation_args.max_tokens == 64000 + + +def test_run_ending_after_retrieval_is_rejected(runs_dir): + log, _ = convert_with_logs( + run_log_payload(), runs_dir, retrieved='1700000000.0' + ) + details = result_by_id( + log, 'v2_Public_Eval::score' + ).generation_config.additional_details + assert details['request_args_unavailable'] == 'run_ended_after_retrieval' + + +def test_rewritten_path_in_history_is_rejected(runs_dir): + history = {f'{RUN_ID}/results.json': 1, f'{RUN_ID}/7b5033c1.json': 2} + log, _ = convert_with_logs(run_log_payload(), runs_dir, history=history) + details = result_by_id( + log, 'v2_Public_Eval::score' + ).generation_config.additional_details + assert details['request_args_unavailable'] == 'run_rewritten' + + +def test_temperature_only_when_every_attempt_states_it(runs_dir): + def set_temperature(metadata, task_id, key): + metadata['kwargs']['temperature'] = 1.0 + + edit_attempts(run_dir(runs_dir), set_temperature) + log, _ = convert_with_logs(run_log_payload(), runs_dir) + args = result_by_id(log, 'v2_Public_Eval::score').generation_config + assert args.generation_args.temperature == 1.0 + + def drop_one(metadata, task_id, key): + if key == 'attempt_2': + metadata['kwargs'].pop('temperature') + + edit_attempts(run_dir(runs_dir), drop_one, task_id='28a6681f') + log, _ = convert_with_logs(run_log_payload(), runs_dir) + args = result_by_id(log, 'v2_Public_Eval::score').generation_config + assert args.generation_args.temperature is None + assert args.generation_args.max_tokens == 64000 + + +def test_unrecorded_attempt_rejects_the_run(runs_dir): + path = run_dir(runs_dir) / '28a6681f.json' + pairs = json.loads(path.read_text(encoding='utf-8')) + pairs[0]['attempt_2'] = None + path.write_text(json.dumps(pairs), encoding='utf-8') + log, _ = convert_with_logs(run_log_payload(), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config.generation_args is None + assert ( + score.generation_config.additional_details['request_args_unavailable'] + == 'attempt_not_recorded' + ) + assert 'scored_tasks' not in score.source_data.additional_details + + +@pytest.mark.parametrize( + ('stated', 'filled'), [('0.470', True), ('0.4700', False)] +) +def test_cost_is_compared_at_its_written_precision(runs_dir, stated, filled): + results_path = run_dir(runs_dir) / 'results.json' + results = json.loads(results_path.read_text(encoding='utf-8')) + results['avg_cost_per_task'] = 0.4704 + results_path.write_text(json.dumps(results), encoding='utf-8') + log, _ = convert_with_logs( + run_log_payload(costPerTask=StatedFloat(stated)), runs_dir + ) + config = result_by_id(log, 'v2_Public_Eval::score').generation_config + if filled: + assert config.generation_args.max_tokens == 64000 + else: + assert config.generation_args is None + assert config.additional_details['request_args_unavailable'] == ( + 'cost_mismatch' + ) + + +def test_payload_parsing_keeps_written_precision(tmp_path): + texts = { + 'evaluations': '[{"modelId": "m", "score": 0.4700, "costPerTask": 1}]', + 'models': '[]', + 'providers': '[]', + 'datasets': '[]', + } + path = tmp_path / 'raw.json' + path.write_text( + adapter.combined_payload_text(texts, 'https://example.test', '1.0'), + encoding='utf-8', + ) + [row] = adapter.load_payload_file(path)['evaluations'] + assert row['score'].text == '0.4700' + assert adapter.parse_payload_texts(texts)['evaluations'][0][ + 'score' + ].text == ('0.4700') + + +def test_join_uses_the_chosen_rows_model_id(runs_dir): + payload = run_log_payload() + alias_row = dict(payload['evaluations'][0], modelId=ALIAS_ID) + payload['evaluations'].append(alias_row) + log, _ = convert_with_logs(payload, runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.score_details.details['raw_model_id'] == RUN_ID + assert score.generation_config.generation_args.max_tokens == 64000 + + run_dir(runs_dir).rename(run_dir(runs_dir, ALIAS_ID)) + edit_attempts( + run_dir(runs_dir, ALIAS_ID), + lambda metadata, task_id, key: metadata.update(test_id=ALIAS_ID), + ) + log, logs = convert_with_logs(payload, runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config is None + assert logs.outcomes == [] + + +def test_output_is_deterministic_for_a_fixed_directory(runs_dir): + first, _ = convert_with_logs(run_log_payload(), runs_dir) + second, _ = convert_with_logs(run_log_payload(), runs_dir) + assert first.model_dump_json() == second.model_dump_json() + + +def test_run_reads_a_local_runs_dir(tmp_path, runs_dir): + payload_path = tmp_path / 'payload.json' + payload_path.write_text(json.dumps(run_log_payload()), encoding='utf-8') + out_dir = tmp_path / 'data' / 'arc-agi' + args = adapter.parse_args( + [ + '--input-json', + str(payload_path), + '--output-dir', + str(out_dir), + '--arc-runs-dir', + str(runs_dir), + '--arc-history-dir', + str(runs_dir / 'history'), + ] + ) + assert adapter.run(args) == 1 + report = json.loads( + (tmp_path / 'adapter_reports' / 'arc-agi_failures.json').read_text( + encoding='utf-8' + ) + ) + assert report['failed_record_count'] == 0 + evidence = report['run_log_evidence'] + assert evidence['sources'] == { + 'v2_Public_Eval': {'dataset': V2.repo_id, 'revision': V2.revision} + } + assert evidence['gate_counts'] == {'filled': 1} + assert evidence['regressions'] == [] + assert evidence['unavailable'] == [] + assert evidence['filled'] == [ + { + 'model_id': RUN_ID, + 'dataset_id': 'v2_Public_Eval', + 'directory': RUN_ID, + 'cap_key': 'max_tokens', + 'max_tokens': 64000, + 'temperature': None, + 'scored_tasks': 3, + } + ] + [path] = out_dir.glob('*/*/*.json') + record = json.loads(path.read_text(encoding='utf-8')) + [score] = [ + r + for r in record['evaluation_results'] + if r['evaluation_result_id'] == 'v2_Public_Eval::score' + ] + assert score['generation_config']['generation_args'] == { + 'max_tokens': 64000 + } + + +def test_public_model_ids_cover_only_sets_with_run_logs(): + payload = run_log_payload() + payload['evaluations'].append( + {'datasetId': 'v2_Public_Eval', 'modelId': 'hidden', 'display': False} + ) + assert adapter.public_model_ids(payload) == { + 'v1_Public_Eval': {RUN_ID}, + 'v2_Public_Eval': {RUN_ID}, + } + + +def _git(*args: str, cwd: Path) -> str: + return subprocess.run( + [ + 'git', + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.com', + *args, + ], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +@pytest.fixture +def dataset_repo(tmp_path: Path) -> Path: + if shutil.which('git') is None: + pytest.skip('git not available') + repo = tmp_path / 'origin' + shutil.copytree(RUN_FIXTURE / 'v2', repo) + other = repo / 'unrelated-run' + other.mkdir() + (other / 'results.json').write_text('{}', encoding='utf-8') + _git('init', '--quiet', cwd=repo) + _git('add', '.', cwd=repo) + _git('commit', '--quiet', '-m', 'publish', cwd=repo) + return repo + + +def test_fetch_clones_and_checks_out_wanted_dirs_at_the_pinned_revision( + tmp_path, dataset_repo, monkeypatch +): + revision = _git('rev-parse', 'HEAD', cwd=dataset_repo) + path = dataset_repo / RUN_ID / '7b5033c1.json' + path.write_text(path.read_text(encoding='utf-8') + '\n', encoding='utf-8') + (dataset_repo / 'late.txt').write_text('later', encoding='utf-8') + _git('add', '.', cwd=dataset_repo) + _git('commit', '--quiet', '-m', 'later', cwd=dataset_repo) + monkeypatch.setattr( + run_logs.RunDataset, + 'url', + property(lambda self: dataset_repo.as_uri()), + ) + dataset = run_logs.RunDataset('v2_Public_Eval', 'test/v2', revision, 'v2') + logs = run_logs.fetch_run_logs( + {'v2_Public_Eval': {RUN_ID, 'not-published'}}, + tmp_path / 'work', + datasets={'v2_Public_Eval': dataset}, + ) + source = logs.sources['v2_Public_Eval'] + assert source.revision == revision + assert _git('rev-parse', 'HEAD', cwd=source.root) == revision + assert (source.root / RUN_ID / 'results.json').is_file() + assert not (source.root / 'unrelated-run').exists() + assert not (source.root / 'late.txt').exists() + assert source.history == { + f'{RUN_ID}/{name}': 1 + for name in sorted(p.name for p in (dataset_repo / RUN_ID).iterdir()) + } | {'unrelated-run/results.json': 1} + + +def test_git_failure_carries_stderr(tmp_path): + if shutil.which('git') is None: + pytest.skip('git not available') + with pytest.raises(RuntimeError, match='not a git repository|fatal'): + run_logs.read_history(tmp_path, 'HEAD') + + +def test_local_git_checkout_supplies_revision_and_history( + tmp_path, dataset_repo +): + runs = tmp_path / 'local' + runs.mkdir() + dataset_repo.rename(runs / 'v2') + path = runs / 'v2' / RUN_ID / '7b5033c1.json' + path.write_text(path.read_text(encoding='utf-8') + '\n', encoding='utf-8') + _git('commit', '--quiet', '-am', 'rewrite', cwd=runs / 'v2') + logs = run_logs.local_run_logs(runs) + source = logs.sources['v2_Public_Eval'] + assert source.revision == _git('rev-parse', 'HEAD', cwd=runs / 'v2') + assert source.history[f'{RUN_ID}/7b5033c1.json'] == 2 + result = adapter.convert_logs( + run_log_payload(), retrieved_timestamp=RETRIEVED, run_logs=logs + ) + [(log, _, _)] = result.records + details = result_by_id( + log, 'v2_Public_Eval::score' + ).generation_config.additional_details + assert details['request_args_unavailable'] == 'run_rewritten' + + +def test_gate_rejection_is_reported_without_failing_the_run( + tmp_path, runs_dir, capsys +): + payload_path = tmp_path / 'payload.json' + payload_path.write_text( + json.dumps(run_log_payload(score=0.6)), encoding='utf-8' + ) + args = adapter.parse_args( + [ + '--input-json', + str(payload_path), + '--output-dir', + str(tmp_path / 'data' / 'arc-agi'), + '--arc-runs-dir', + str(runs_dir), + '--arc-history-dir', + str(runs_dir / 'history'), + ] + ) + assert adapter.run(args) == 1 + evidence = json.loads( + (tmp_path / 'adapter_reports' / 'arc-agi_failures.json').read_text( + encoding='utf-8' + ) + )['run_log_evidence'] + assert evidence['gate_counts'] == {'score_mismatch': 1} + assert evidence['unavailable'] == [] + assert 'ARC run-log regression: ' in capsys.readouterr().err + assert evidence['regressions'] == [ + { + 'model_id': RUN_ID, + 'dataset_id': 'v2_Public_Eval', + 'directory': RUN_ID, + 'gate': 'score_mismatch', + 'detail': '', + } + ] + assert evidence['filled'] == [] + + +def test_plain_runs_dir_without_history_fails(tmp_path, runs_dir): + payload_path = tmp_path / 'payload.json' + payload_path.write_text(json.dumps(run_log_payload()), encoding='utf-8') + args = adapter.parse_args( + [ + '--input-json', + str(payload_path), + '--output-dir', + str(tmp_path / 'data' / 'arc-agi'), + '--arc-runs-dir', + str(runs_dir), + ] + ) + with pytest.raises(SystemExit, match='--arc-history-dir'): + adapter.run(args) + assert not (tmp_path / 'data').exists() + + +def test_history_file_must_match_the_revision(tmp_path): + path = tmp_path / 'v2.json' + path.write_text( + json.dumps({'revision': 'other', 'paths': {}}), encoding='utf-8' + ) + with pytest.raises(run_logs.RunLogsUnavailable, match='revision'): + run_logs.load_history_file(path, V2.revision) + + +def test_permanent_rejections_are_listed_as_unavailable(tmp_path, runs_dir): + (run_dir(runs_dir) / 'results.json').unlink() + log, logs = convert_with_logs(run_log_payload(), runs_dir) + report = adapter.run_log_report(logs) + assert list(report)[2:4] == ['regressions', 'unavailable'] + assert report['regressions'] == [] + assert [e['gate'] for e in report['unavailable']] == ['no_results_file'] diff --git a/tests/test_flat_rebuild.py b/tests/test_flat_rebuild.py index 0b1452a4d..263fed459 100644 --- a/tests/test_flat_rebuild.py +++ b/tests/test_flat_rebuild.py @@ -3,9 +3,12 @@ from __future__ import annotations import json -from dataclasses import dataclass +import shutil +import subprocess +from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from pathlib import Path, PurePosixPath +from types import SimpleNamespace from typing import Any from uuid import uuid4 @@ -14,7 +17,7 @@ from huggingface_hub.errors import EntryNotFoundError from every_eval_ever.cron import flat_rebuild as fr -from every_eval_ever.cron import store +from every_eval_ever.cron import flat_snapshot, store NOW = datetime(2026, 9, 6, 12, 0, 0, tzinfo=UTC) OLD = NOW - timedelta(days=100) @@ -98,6 +101,54 @@ def create_commit( self.files.pop(operation.path_in_repo, None) +class FakeFlatSnapshotReader: + """Explicit offline substitute for the published aggregate transport.""" + + def __init__(self, api, repo_id): + self.files = dict(api.files) + self.revision = 'a' * 40 + self.pack_count = 0 + self.lfs_download_count = 0 + self.lfs_download_paths = [] + self.reads = [] + self.closed = False + + def __enter__(self): + self.pack_count += 1 + return self + + def __exit__(self, *exc): + self.closed = True + + def read(self, path): + assert path == fr.RETIRED_LIST_PATH or ( + path.startswith('flat/objects/') and path.endswith('.json') + ) + self.reads.append(path) + if path not in self.files: + raise fr.FlatSnapshotError(f'Missing published object: {path}') + return self.files[path] + + def read_many(self, paths): + return {path: self.read(path) for path in paths} + + def exists(self, path): + return path in self.files + + +@pytest.fixture(autouse=True) +def bulk_readers(monkeypatch): + readers = [] + + def create(api, repo_id): + reader = FakeFlatSnapshotReader(api, repo_id) + readers.append(reader) + return reader + + monkeypatch.setattr(fr, 'FlatSnapshotReader', create) + return readers + + def committed_paths(api: FakeApi) -> set[str]: return { operation.path_in_repo @@ -1213,7 +1264,9 @@ def test_moved_companion_conflict_excludes_samples_path(): path: record_bytes(row.object_uuid, 'gsm8k'), sample_path: b'{"x":2000}\n', }, - lambda object_path: published if object_path == row.instance_level_path else None, + lambda object_path: ( + published if object_path == row.instance_level_path else None + ), ) assert not result.errors assert len(result.conflicts) == 1 @@ -1318,3 +1371,783 @@ def test_verify_bootstrap_allows_planned_new_objects(tmp_path): assert not report.conflicts assert report.verified_files == 0 assert row.object_path in api.files + + +# -- supersession ------------------------------------------------------------ + + +def captured_record( + uuid: str, + evaluation_id: str | None, + retrieved: str | None, + *, + adapter: str | None = 'hle', + run_date: str | None = None, +) -> bytes: + payload = json.loads(record_bytes(uuid, 'x')) + payload.pop('evaluation_id') + if evaluation_id is not None: + payload['evaluation_id'] = evaluation_id + if retrieved is not None: + payload['retrieved_timestamp'] = retrieved + if adapter is not None: + payload['source_metadata'] = { + 'additional_details': { + 'type_of_addition': 'cron', + 'cron_adapter': adapter, + 'cron_run_date': run_date or f'day-{retrieved}', + } + } + return json.dumps(payload).encode() + + +def captured_row( + uuid: str, + benchmark: str, + evaluation_id: str | None, + retrieved: str | None = None, + *, + adapter: str | None = 'hle', + run_date: str | None = None, + result_count: int = 1, + result_identities: tuple[str, ...] | None = None, +) -> fr.Row: + return replace( + make_row(uuid, benchmark), + evaluation_id=evaluation_id, + result_count=result_count, + result_identities=( + result_identities + if result_identities is not None + else tuple(f'result-{i}' for i in range(result_count)) + ), + retrieved_timestamp=retrieved, + cron_adapter=adapter, + cron_run_date=(run_date or f'day-{retrieved}') if adapter else None, + ) + + +def capture(uuid: str, stamp: str, **kwargs: Any) -> fr.Row: + return captured_row(uuid, 'hle', f'hle/m/{stamp}', stamp, **kwargs) + + +def supersede(rows, retired=()) -> dict[str, fr.Row]: + return by_uuid(fr.apply_supersession(rows, retired).rows) + + +def by_uuid(rows) -> dict[str, fr.Row]: + return {row.object_uuid: row for row in rows} + + +def test_older_capture_is_superseded_by_newest() -> None: + rows = supersede([capture('a', '100.5'), capture('b', '200')]) + assert rows['a'].superseded_by == 'b' + assert rows['a'].superseded_reason == 'newer_capture' + assert rows['b'].superseded_by is None + assert rows['b'].to_dict()['superseded_by'] is None + + +def test_timestamp_tie_breaks_on_uuid() -> None: + rows = supersede( + [capture('c', '500'), capture('a', '500.0', run_date='d2')] + ) + assert rows['c'].superseded_by is None + assert rows['a'].superseded_by == 'c' + + +def test_record_ids_reads_cron_provenance_only_for_cron_records() -> None: + ids = fr.record_ids(captured_record('a', 'hle/m/1', '1', run_date='d')) + assert ids == fr.RecordIds( + 'hle/m/1', + '1', + 'hle', + 'd', + 1, + fr.result_identities( + json.loads(captured_record('a', 'hle/m/1', '1'))[ + 'evaluation_results' + ] + ), + ) + manual = json.loads(captured_record('a', 'hle/m/1', '1')) + manual['source_metadata']['additional_details']['type_of_addition'] = ( + 'manual' + ) + ids = fr.record_ids(json.dumps(manual).encode()) + assert (ids.cron_adapter, ids.cron_run_date) == (None, None) + + +def test_rows_without_cron_provenance_are_never_grouped() -> None: + rows = fr.apply_supersession( + [capture('a', '100', adapter=None), capture('b', '200', adapter=None)] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_series_are_keyed_by_adapter() -> None: + rows = fr.apply_supersession( + [capture('a', '100'), capture('b', '200', adapter='other')] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_tail_must_equal_retrieved_timestamp() -> None: + rows = fr.apply_supersession( + [ + captured_row('a', 'hle', 'hle/m/100', '150'), + captured_row('b', 'hle', 'hle/m/200', '200'), + ] + ).rows + assert all(row.superseded_by is None for row in rows) + + +@pytest.mark.parametrize( + 'evaluation_id', ['hle/m/run-a', 'hle/m/1e9', '12345', None] +) +def test_ids_without_timestamp_segment_form_no_series(evaluation_id) -> None: + tail = (evaluation_id or '').rpartition('/')[2] or None + rows = fr.apply_supersession( + [ + captured_row('a', 'hle', evaluation_id, tail, run_date='d1'), + captured_row('b', 'hle', evaluation_id, tail, run_date='d2'), + ] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_series_do_not_cross_collections() -> None: + rows = fr.apply_supersession( + [ + captured_row('a', 'hle', 'shared/m/100', '100'), + captured_row('b', 'mmlu_pro', 'shared/m/200', '200'), + ] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_same_run_date_makes_series_ambiguous() -> None: + result = fr.apply_supersession( + [ + capture('a', '100', run_date='2026-08-13'), + capture('b', '200', run_date='2026-08-13'), + capture('c', '300', run_date='2026-08-14'), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert result.ambiguous_series == ('hle/hle/hle/m',) + + +def test_newest_with_fewer_results_leaves_series_unmarked() -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_count=5), + capture('b', '200', result_count=3), + capture('c', '300', result_count=2), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert result.regressed_series == ( + 'hle/hle/hle/m: newest c drops result identities from a (3), b (1)', + ) + report = fr.RebuildReport( + repo_id='org/ds', regressed_series=result.regressed_series + ) + lines = fr.summary_lines(report) + assert any('regressed series: 1' in line for line in lines) + + +@pytest.mark.parametrize('newest_count', [3, 4]) +def test_newest_with_equal_or_more_results_supersedes(newest_count) -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_count=3), + capture('b', '200', result_count=newest_count), + ] + ) + assert by_uuid(result.rows)['a'].superseded_by == 'b' + assert result.regressed_series == () + + +@pytest.mark.parametrize( + 'newer', [('bench-a', 'bench-c'), ('bench-a', 'bench-c', 'bench-d')] +) +def test_equal_or_larger_capture_cannot_drop_result_identity(newer) -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_identities=('bench-a', 'bench-b')), + capture('b', '200', result_identities=newer), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert len(result.regressed_series) == 1 + + +def test_duplicate_result_multiplicity_is_preserved() -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_identities=('x', 'x')), + capture('b', '200', result_identities=('x', 'y', 'z')), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert len(result.regressed_series) == 1 + + +def test_result_identity_order_and_score_changes_do_not_block_supersession() -> ( + None +): + old = json.loads(captured_record('a', 'hle/m/100', '100')) + old['evaluation_results'].append( + { + 'evaluation_name': 'other', + 'metric_config': {'metric_id': 'accuracy'}, + 'score_details': {'score': 0.5}, + } + ) + new = json.loads(captured_record('b', 'hle/m/200', '200')) + new['evaluation_results'] = list(reversed(old['evaluation_results'])) + for result in new['evaluation_results']: + result['score_details'] = { + 'score': 0.7, + 'uncertainty': {'num_samples': 20}, + } + result['evaluation_result_id'] = 'new-capture-result' + result['evaluation_timestamp'] = '200' + rows = supersede( + [ + fr.record_ids(json.dumps(old).encode()).apply(make_row('a', 'hle')), + fr.record_ids(json.dumps(new).encode()).apply(make_row('b', 'hle')), + ] + ) + assert rows['a'].superseded_by == 'b' + + +@pytest.mark.parametrize( + 'field,value', + [ + ('evaluation_name', 'new-benchmark-version'), + ('metric_config', {'metric_id': 'f1'}), + ( + 'source_data', + {'source_type': 'hf', 'dataset_name': 'bench', 'split': 'test'}, + ), + ('generation_config', {'temperature': 0.5}), + ], +) +def test_result_identity_distinguishes_published_metadata(field, value) -> None: + old = json.loads(captured_record('a', 'hle/m/100', '100')) + new = json.loads(captured_record('b', 'hle/m/200', '200')) + new['evaluation_results'][0][field] = value + result = fr.apply_supersession( + [ + fr.record_ids(json.dumps(old).encode()).apply(make_row('a', 'hle')), + fr.record_ids(json.dumps(new).encode()).apply(make_row('b', 'hle')), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert len(result.regressed_series) == 1 + + +@pytest.mark.parametrize('results', [None, {}, [None], [{}]]) +def test_unreadable_result_identity_cannot_establish_coverage(results) -> None: + assert fr.result_identities(results) is None + result = fr.apply_supersession( + [ + capture('a', '100'), + replace(capture('b', '200'), result_identities=None), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert result.regressed_series == ( + 'hle/hle/hle/m: unreadable result identities', + ) + + +def test_count_only_index_rows_are_backfilled_before_supersession() -> None: + old = capture('a', '100').to_dict() + del old['result_identities'] + data = captured_record('a', 'hle/m/100', '100') + row = replace( + fr.Row.from_dict(old), + sha256=fr.sha256_bytes(data), + size_bytes=len(data), + ) + assert not row.ids_recorded + calls = [] + + def fetch(paths): + calls.extend(paths) + return {row.object_path: data} + + result = fr.apply_supersession([row, capture('b', '200')]) + assert result.pending_collections == ('hle',) + assert all(row.superseded_by is None for row in result.rows) + filled, count, pending = fr.backfill_ids([row], fetch) + assert calls == [row.object_path] + assert (count, pending) == (1, 0) + assert filled[0].ids_recorded + assert filled[0].result_identities is not None + + +def test_identity_backfill_rejects_published_object_drift(): + data = captured_record('a', 'hle/m/100', '100') + row = replace( + capture('a', '100'), + ids_recorded=False, + sha256=fr.sha256_bytes(data), + size_bytes=len(data), + ) + changed = data.replace(b'100', b'200') + assert len(data) == len(changed) + with pytest.raises(fr.FlatRebuildError, match='published bytes differ'): + fr.backfill_ids([row], lambda paths: {row.object_path: changed}) + + +def test_retired_fuller_row_does_not_block_series() -> None: + rows = supersede( + [ + capture('a', '100', result_count=9), + capture('b', '200', result_count=2), + capture('c', '300', result_count=2), + ], + [fr.RetiredEntry('a', 'extra rows were wrong')], + ) + assert rows['b'].superseded_by == 'c' + + +def test_collection_with_pending_ids_is_skipped() -> None: + pending = replace(make_row('p', 'hle'), ids_recorded=False) + result = fr.apply_supersession( + [ + capture('a', '100'), + capture('b', '200'), + pending, + captured_row('c', 'gsm8k', 'gsm8k/m/1', '1'), + captured_row('d', 'gsm8k', 'gsm8k/m/2', '2'), + ] + ) + rows = by_uuid(result.rows) + assert rows['a'].superseded_by is None + assert rows['c'].superseded_by == 'd' + assert result.pending_collections == ('hle',) + + +def test_retire_list_marks_rows_with_or_without_replacement() -> None: + rows = supersede( + [ + captured_row('c', 'hle', 'hle/other/1', '1'), + capture('b', '200'), + captured_row('d', 'hle', None, adapter=None), + ], + [ + fr.RetiredEntry('b', 'wrong temperature', replaced_by='c'), + fr.RetiredEntry('d', 'pre-cron, no successor'), + ], + ) + assert (rows['b'].superseded_by, rows['b'].superseded_reason) == ( + 'c', + 'retired', + ) + assert (rows['d'].superseded_by, rows['d'].superseded_reason) == ( + 'retired', + 'retired', + ) + assert rows['c'].superseded_by is None + + +def test_retiring_newest_capture_restores_previous() -> None: + rows = supersede( + [capture('a', '100'), capture('b', '200'), capture('c', '50')], + [fr.RetiredEntry('b', 'wrong temperature')], + ) + assert rows['a'].superseded_by is None + assert rows['b'].superseded_reason == 'retired' + assert rows['c'].superseded_by == 'a' + + +def test_retiring_older_capture_leaves_newest_current() -> None: + rows = supersede( + [capture('a', '100'), capture('b', '200')], + [fr.RetiredEntry('a', 'bad capture', replaced_by='b')], + ) + assert rows['b'].superseded_by is None + assert (rows['a'].superseded_by, rows['a'].superseded_reason) == ( + 'b', + 'retired', + ) + + +def test_fully_retired_series_has_no_current_row() -> None: + rows = fr.apply_supersession( + [capture('a', '100'), capture('b', '200')], + [fr.RetiredEntry('a', 'x'), fr.RetiredEntry('b', 'y')], + ).rows + assert all(row.superseded_reason == 'retired' for row in rows) + + +def test_stale_retire_entry_is_reported_and_skipped() -> None: + result = fr.apply_supersession( + [capture('a', '100')], + [ + fr.RetiredEntry('zzz', 'gone'), + fr.RetiredEntry('a', 'successor left', replaced_by='yyy'), + ], + ) + assert result.stale_retired == ('zzz',) + assert result.rows[0].superseded_by == 'yyy' + assert result.dangling_replaced_by == ('a -> yyy',) + + +def test_dangling_replaced_by_is_listed_in_summary() -> None: + report = fr.RebuildReport( + repo_id='org/ds', dangling_replaced_by=('a -> yyy',) + ) + lines = fr.summary_lines(report) + assert any('dangling replaced_by: 1' in line for line in lines) + assert '- `a -> yyy`' in lines + + +def test_supersession_is_recomputed_not_inherited() -> None: + stale = replace( + captured_row('a', 'hle', None), + superseded_by='retired', + superseded_reason='retired', + ) + assert fr.apply_supersession([stale]).rows[0].superseded_by is None + + +def test_unknown_superseded_reason_is_a_rebuild_error() -> None: + row = capture('a', '1').to_dict() + row.update(superseded_by='b', superseded_reason='bogus') + with pytest.raises(fr.FlatRebuildError, match='bogus'): + fr.Row.from_dict(row) + + +@pytest.mark.parametrize( + 'text', + [ + b'not json\n', + b'{"reason": "no uuid"}\n', + b'{"object_uuid": "a"}\n', + b'{"object_uuid": "a", "reason": "x", "replaced_by": "a"}\n', + b'{"object_uuid": "a", "reason": "x"}\n{"object_uuid": "a", "reason": "y"}\n', + ], +) +def test_malformed_retire_list_errors(text) -> None: + with pytest.raises(fr.FlatRebuildError, match='malformed'): + fr.parse_retired(text) + + +def test_row_ids_round_trip_and_legacy_rows_are_unrecorded() -> None: + row = replace( + capture('a', '1'), + superseded_by='b', + superseded_reason='newer_capture', + ) + assert fr.Row.from_dict(row.to_dict()) == row + legacy = row.to_dict() + for key in ( + 'evaluation_id', + 'retrieved_timestamp', + 'cron_adapter', + 'cron_run_date', + 'result_count', + 'result_identities', + ): + del legacy[key] + restored = fr.Row.from_dict(legacy) + assert not restored.ids_recorded + assert 'evaluation_id' not in restored.to_dict() + + +def seed_captures(files: dict[str, bytes], benchmark: str, stamps) -> list: + uuids = [] + for stamp in stamps: + uuid = str(uuid4()) + files[f'data/{benchmark}/dev/model/{uuid}.json'] = captured_record( + uuid, f'{benchmark}/dev/model/{stamp}', stamp + ) + uuids.append(uuid) + return uuids + + +def published_rows(api: FakeApi, path: str) -> dict[str, dict[str, Any]]: + return { + row['object_uuid']: row + for row in map(json.loads, api.files[path].decode().splitlines()) + } + + +def test_orchestrate_marks_supersession_and_stays_idempotent( + tmp_path: Path, +) -> None: + files: dict[str, bytes] = {} + old, new = seed_captures(files, 'hle', ['100', '200']) + api = FakeApi(files, tmp_path) + fr.orchestrate(api, 'org/ds', allow_bootstrap=True, now=NOW) + index = published_rows(api, f'{fr.INDEXES_PREFIX}/hle.jsonl') + assert index[old]['superseded_by'] == new + assert index[old]['evaluation_id'] == 'hle/dev/model/100' + assert index[old]['cron_adapter'] == 'hle' + assert index[old]['result_count'] == 1 + assert index[new]['superseded_by'] is None + latest = json.loads(api.files[fr.LATEST_MANIFEST_PATH]) + assert published_rows(api, latest['entries_path']) == index + + commits = len(api.commits) + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert report.noop + assert report.superseded_captures == 1 + assert len(api.commits) == commits + + +def test_orchestrate_applies_retire_list_and_republishes( + tmp_path: Path, +) -> None: + files: dict[str, bytes] = {} + (only,) = seed_captures(files, 'hle', ['100']) + api = FakeApi(files, tmp_path) + fr.orchestrate(api, 'org/ds', allow_bootstrap=True, now=NOW) + core_before = json.loads(api.files[fr.LATEST_MANIFEST_PATH])[ + 'manifest_core_sha256' + ] + + api.files[fr.RETIRED_LIST_PATH] = ( + json.dumps({'object_uuid': only, 'reason': 'wrong prompt label'}) + '\n' + ).encode() + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert not report.noop + assert report.superseded_retired == 1 + latest = json.loads(api.files[fr.LATEST_MANIFEST_PATH]) + assert latest['manifest_core_sha256'] != core_before + row = published_rows(api, f'{fr.INDEXES_PREFIX}/hle.jsonl')[only] + assert (row['superseded_by'], row['superseded_reason']) == ( + 'retired', + 'retired', + ) + assert f'data/hle/dev/model/{only}.json' in api.files + assert fr.orchestrate(api, 'org/ds', now=NOW).noop + + +def test_orchestrate_reports_stale_retire_entry(tmp_path: Path) -> None: + files: dict[str, bytes] = {} + seed_captures(files, 'hle', ['100']) + api = FakeApi(files, tmp_path) + fr.orchestrate(api, 'org/ds', allow_bootstrap=True, now=NOW) + api.files[fr.RETIRED_LIST_PATH] = ( + b'{"object_uuid": "0000", "reason": "typo"}\n' + ) + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert report.noop + assert report.stale_retired == ('0000',) + assert any( + 'stale retire entries: 1' in line for line in fr.summary_lines(report) + ) + + +def seed_legacy_snapshot(files: dict[str, bytes], uuids) -> None: + """Publish rows for ``uuids`` in the shape written before ids existed.""" + legacy_rows = [] + for uuid in uuids: + data = files[f'data/hle/dev/model/{uuid}.json'] + row = replace( + make_row(uuid, 'hle'), + sha256=fr.sha256_bytes(data), + size_bytes=len(data), + ) + files[row.object_path] = data + legacy = row.to_dict() + for key in ( + 'evaluation_id', + 'retrieved_timestamp', + 'cron_adapter', + 'cron_run_date', + 'result_count', + 'result_identities', + 'superseded_by', + 'superseded_reason', + ): + legacy.pop(key, None) + legacy_rows.append(legacy) + manifest = fr.manifest_for( + [fr.Row.from_dict(row) for row in legacy_rows], + created_at=RECENT.isoformat(), + ) + files[manifest['entries_path']] = fr.jsonl_text(legacy_rows).encode() + seed_pointer(files, manifest) + + +def test_orchestrate_backfills_all_ids(tmp_path: Path, bulk_readers) -> None: + files: dict[str, bytes] = {} + uuids = seed_captures(files, 'hle', ['100', '200', '300']) + seed_legacy_snapshot(files, uuids) + api = FakeApi(files, tmp_path) + + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert (report.ids_backfilled, report.ids_pending) == (3, 0) + assert report.pending_collections == () + assert report.superseded_captures == 2 + assert len(bulk_readers) == 1 + assert bulk_readers[0].closed + assert report.bulk_pack_count == 1 + index = published_rows(api, f'{fr.INDEXES_PREFIX}/hle.jsonl') + assert {row['superseded_by'] for row in index.values()} == { + uuids[2], + None, + } + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert report.noop + assert report.ids_backfilled == 0 + assert len(bulk_readers) == 1 + assert report.bulk_pack_count == 0 + + +def test_bootstrap_ids_is_compatibility_alias( + tmp_path: Path, bulk_readers +) -> None: + files: dict[str, bytes] = {} + uuids = seed_captures(files, 'hle', ['100', '200']) + seed_legacy_snapshot(files, uuids) + normal = FakeApi(files, tmp_path / 'normal') + alias = FakeApi(files, tmp_path / 'alias') + normal_report = fr.orchestrate(normal, 'org/ds', now=NOW) + alias_report = fr.orchestrate(alias, 'org/ds', now=NOW, bootstrap_ids=True) + assert normal.files == alias.files + assert replace(normal_report, duration_seconds=0) == replace( + alias_report, duration_seconds=0 + ) + assert len(bulk_readers) == 2 + assert all( + reader.closed and reader.pack_count == 1 for reader in bulk_readers + ) + + +def test_bulk_reader_closes_and_translates_errors( + tmp_path, monkeypatch, bulk_readers +): + files = {} + uuids = seed_captures(files, 'hle', ['100']) + seed_legacy_snapshot(files, uuids) + + def unreadable(self, paths): + raise fr.FlatSnapshotError('Cannot read pinned object') + + monkeypatch.setattr(FakeFlatSnapshotReader, 'read_many', unreadable) + with pytest.raises(fr.FlatRebuildError, match='Cannot read pinned object'): + fr.orchestrate(FakeApi(files, tmp_path), 'org/ds', now=NOW) + assert len(bulk_readers) == 1 + assert bulk_readers[0].closed + + +def test_all_id_backfill_and_semantic_comparison_share_real_git_pack( + tmp_path, monkeypatch +): + if shutil.which('git') is None: + pytest.skip('Git is required for the local bulk-reader fixture') + count = fr.BACKFILL_CHUNK + 1 + files = {} + uuids = seed_captures(files, 'hle', [str(i) for i in range(count)]) + seed_legacy_snapshot(files, uuids) + touched = fr.object_path_for(uuids[0], suffix='.json') + unused = 'flat/objects/unused/unused.json' + sample = 'flat/objects/unused/sample.jsonl' + files[unused] = b'{"unused": true}' + files[sample] = b'{"sample": true}\n' + files[fr.RETIRED_LIST_PATH] = b'' + changed = f'data/hle/dev/model/{uuids[0]}.json' + files[changed] = json.dumps(json.loads(files[changed]), indent=2).encode() + + def pointer(payload): + return ( + 'version https://git-lfs.github.com/spec/v1\n' + f'oid sha256:{fr.sha256_bytes(payload)}\n' + f'size {len(payload)}\n' + ).encode() + + work = tmp_path / 'source repository' + work.mkdir() + + def git(*args, cwd=work): + return subprocess.run( + ['git', *args], cwd=cwd, capture_output=True, text=True, check=True + ).stdout.strip() + + git('init', '--initial-branch=fixture') + git('config', 'user.name', 'Fixture') + git('config', 'user.email', 'fixture@example.invalid') + for path, payload in files.items(): + target = work / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes( + pointer(payload) if path in (touched, unused, sample) else payload + ) + git('add', '.') + git('commit', '-m', 'Published aggregate fixture') + pinned = git('rev-parse', 'HEAD') + remote = tmp_path / 'remote repository.git' + git('clone', '--bare', str(work), str(remote)) + + class PinnedApi(FakeApi): + def __init__(self, root): + super().__init__(files, root) + self.info_calls = 0 + self.pinned_downloads = [] + + def repo_info(self, *, repo_id, repo_type): + assert (repo_id, repo_type) == ('org/ds', 'dataset') + self.info_calls += 1 + return SimpleNamespace(sha=pinned) + + def hf_hub_download(self, repo_id, repo_type, filename, revision=None): + if filename.startswith('flat/objects/'): + assert filename == touched + assert revision == pinned + self.pinned_downloads.append((filename, revision)) + else: + assert filename != fr.RETIRED_LIST_PATH + return super().hf_hub_download(repo_id, repo_type, filename) + + readers = [] + + class ObservedReader(flat_snapshot.FlatSnapshotReader): + def __enter__(self): + super().__enter__() + self.checkout_path = self._directory + assert not (self.checkout_path / sample).exists() + return self + + def create_reader(api, repo_id): + reader = ObservedReader(api, repo_id, remote_url=remote.as_uri()) + readers.append(reader) + return reader + + monkeypatch.setattr(fr, 'FlatSnapshotReader', create_reader) + normal = PinnedApi(tmp_path / 'normal') + alias = PinnedApi(tmp_path / 'alias') + reports = [ + fr.orchestrate(normal, 'org/ds', now=NOW), + fr.orchestrate(alias, 'org/ds', now=NOW, bootstrap_ids=True), + ] + assert normal.files == alias.files + for api, report in zip((normal, alias), reports, strict=True): + assert report.ids_backfilled == count + assert report.ids_pending == 0 + assert report.superseded_captures == count - 1 + assert report.bulk_revision == pinned + assert report.bulk_pack_count == 1 + assert report.bulk_lfs_download_count == 1 + assert report.bulk_lfs_download_paths == (touched,) + assert api.info_calls == 1 + assert api.pinned_downloads == [(touched, pinned)] + assert any(pinned in line for line in fr.summary_lines(report)) + assert any(touched in line for line in fr.summary_lines(report)) + assert len(readers) == 2 + assert all(not reader.checkout_path.exists() for reader in readers) + + # The unchanged manifest still reads the reserialized data's old object. + noop = fr.orchestrate(normal, 'org/ds', now=NOW) + assert noop.noop + assert noop.ids_backfilled == 0 + assert noop.bulk_pack_count == 1 + assert len(readers) == 3 + assert not readers[-1].checkout_path.exists() diff --git a/tests/test_flat_snapshot.py b/tests/test_flat_snapshot.py new file mode 100644 index 000000000..db0ce99a6 --- /dev/null +++ b/tests/test_flat_snapshot.py @@ -0,0 +1,343 @@ +"""Offline Git and LFS fixtures for published aggregate identity reads.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +from types import SimpleNamespace + +import pytest + +from every_eval_ever.cron import flat_snapshot as fs + +PLAIN = 'flat/objects/aa/plain.json' +LFS = 'flat/objects/bb/large.json' +UNTOUCHED_LFS = 'flat/objects/cc/untouched.json' +RETIRED = 'flat/retired.jsonl' + + +def git(directory, *args): + env = os.environ.copy() + env.update( + GIT_CONFIG_NOSYSTEM='1', + GIT_CONFIG_GLOBAL=os.devnull, + GIT_LFS_SKIP_SMUDGE='1', + GIT_ALLOW_PROTOCOL='file', + GIT_CONFIG_COUNT='3', + GIT_CONFIG_KEY_0='filter.lfs.process', + GIT_CONFIG_VALUE_0='', + GIT_CONFIG_KEY_1='filter.lfs.smudge', + GIT_CONFIG_VALUE_1='', + GIT_CONFIG_KEY_2='filter.lfs.required', + GIT_CONFIG_VALUE_2='false', + ) + return subprocess.run( + ['git', *args], + cwd=directory, + env=env, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def pointer(payload): + return ( + 'version https://git-lfs.github.com/spec/v1\n' + f'oid sha256:{hashlib.sha256(payload).hexdigest()}\n' + f'size {len(payload)}\n' + ).encode() + + +@pytest.fixture +def repository(tmp_path): + source = tmp_path / 'source' + source.mkdir() + git(source, 'init', '--quiet') + git(source, 'config', 'user.email', 'fixture@example.invalid') + git(source, 'config', 'user.name', 'Offline fixture') + plain = b'{"evaluation_results": [{"name": "fixture metric"}]}\n' + payload = b'{"evaluation_results": [{"name": "larger fixture metric"}]}\n' + files = { + PLAIN: plain, + 'flat/objects/direct.json': b'{}\n', + LFS: pointer(payload), + UNTOUCHED_LFS: pointer(b'{"untouched": true}\n'), + RETIRED: b'{"fixture": "retired"}\n', + 'data/unrelated/record.json': b'{"not_checked_out": true}\n', + } + for relative, data in files.items(): + path = source / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + git(source, 'add', '.') + git(source, 'commit', '--quiet', '-m', 'Pinned fixture') + pin = git(source, 'rev-parse', 'HEAD') + (source / PLAIN).write_bytes(b'{"later": true}\n') + git(source, 'add', '.') + git(source, 'commit', '--quiet', '-m', 'Later fixture') + remote = tmp_path / 'remote.git' + git(tmp_path, 'clone', '--bare', str(source), str(remote)) + cache = tmp_path / 'payload.json' + cache.write_bytes(payload) + return SimpleNamespace( + source=source, + remote=remote, + pin=pin, + tip=git(source, 'rev-parse', 'HEAD'), + plain=plain, + payload=payload, + cache=cache, + ) + + +class FakeApi: + endpoint = 'https://example.invalid' + token = False + + def __init__(self, repository): + self.repository = repository + self.info_calls = [] + self.downloads = [] + + def repo_info(self, **kwargs): + self.info_calls.append(kwargs) + return SimpleNamespace(sha=self.repository.pin) + + def hf_hub_download(self, **kwargs): + self.downloads.append(kwargs) + assert kwargs == { + 'repo_id': 'fixture/datastore', + 'repo_type': 'dataset', + 'filename': LFS, + 'revision': self.repository.pin, + } + return str(self.repository.cache) + + +def reader(repository, api=None, **kwargs): + return fs.FlatSnapshotReader( + api or FakeApi(repository), + 'fixture/datastore', + remote_url=str(repository.remote), + **kwargs, + ) + + +def test_one_shallow_pack_at_resolved_pin_and_sparse_checkout( + repository, monkeypatch +): + api = FakeApi(repository) + calls = [] + run = subprocess.run + + def recording_run(command, **kwargs): + calls.append((command, kwargs)) + return run(command, **kwargs) + + monkeypatch.setattr(fs.subprocess, 'run', recording_run) + snapshot = reader(repository, api) + with snapshot: + assert snapshot.revision == repository.pin != repository.tip + assert snapshot.pack_count == 1 + assert snapshot.read_many([PLAIN, RETIRED]) == { + PLAIN: repository.plain, + RETIRED: b'{"fixture": "retired"}\n', + } + assert snapshot.read('flat/objects/direct.json') == b'{}\n' + assert snapshot.exists(LFS) + assert not snapshot.exists('flat/objects/missing.json') + checkout = snapshot._directory + assert checkout is not None + assert not (checkout / 'data').exists() + assert (checkout / '.git/shallow').read_text().strip() == repository.pin + assert api.downloads == [] + assert snapshot.lfs_download_count == 0 + assert not checkout.exists() + assert api.info_calls == [ + {'repo_id': 'fixture/datastore', 'repo_type': 'dataset'} + ] + fetches = [command for command, _ in calls if command[1] == 'fetch'] + assert fetches == [ + ['git', 'fetch', '--depth', '1', str(repository.remote), repository.pin] + ] + assert all('--filter' not in ' '.join(command) for command, _ in calls) + assert all( + kwargs['env']['GIT_LFS_SKIP_SMUDGE'] == '1' for _, kwargs in calls + ) + assert all( + kwargs['env']['GIT_TERMINAL_PROMPT'] == '0' for _, kwargs in calls + ) + + +def test_only_touched_lfs_payload_downloaded_once_at_pin(repository): + api = FakeApi(repository) + with reader(repository, api) as snapshot: + assert snapshot.read(PLAIN) == repository.plain + assert api.downloads == [] + assert snapshot.read(LFS) == repository.payload + assert snapshot.read(LFS) == repository.payload + assert len(api.downloads) == snapshot.lfs_download_count == 1 + assert snapshot.lfs_download_paths == [LFS] + assert UNTOUCHED_LFS not in snapshot.lfs_download_paths + + +@pytest.mark.parametrize('corruption', ['wrong size', 'wrong hash']) +def test_lfs_payload_hash_and_size_verified(repository, corruption): + repository.cache.write_bytes( + b'bad' if corruption == 'wrong size' else b'x' * len(repository.payload) + ) + with reader(repository) as snapshot: + with pytest.raises(fs.FlatSnapshotError, match='SHA256/size'): + snapshot.read(LFS) + + +def test_supplied_pin_skips_revision_resolution(repository): + api = FakeApi(repository) + with reader(repository, api, revision=repository.pin) as snapshot: + assert snapshot.read(PLAIN) == repository.plain + assert api.info_calls == [] + + +@pytest.mark.parametrize( + 'path', + [ + 'flat/objects/aa/private_samples.jsonl', + 'data/source/private_samples.jsonl', + 'flat/objects/../retired.json', + '/flat/objects/x.json', + 'flat/objects/./x.json', + 'flat/objects\\x.json', + 'flat/manifests/example/entries.jsonl', + ], +) +def test_identity_reader_rejects_samples_and_other_paths(repository, path): + api = FakeApi(repository) + with reader(repository, api) as snapshot: + with pytest.raises(ValueError, match='Not a published aggregate'): + snapshot.read(path) + assert api.downloads == [] + + +def test_cleanup_after_consumer_error(repository): + snapshot = reader(repository) + with pytest.raises(RuntimeError, match='consumer error'): + with snapshot: + checkout = snapshot._directory + raise RuntimeError('consumer error') + assert not checkout.exists() + + +def test_missing_git_fails_loudly_and_cleans_up(repository, monkeypatch): + def missing(*_args, **_kwargs): + raise FileNotFoundError('git') + + monkeypatch.setattr(fs.subprocess, 'run', missing) + snapshot = reader(repository) + with pytest.raises(fs.FlatSnapshotError, match='Git is required'): + with snapshot: + pass + assert snapshot._directory is None + assert snapshot._temporary is None + + +def test_fetch_failure_cleans_up(repository, monkeypatch): + run = subprocess.run + + def failing(command, **kwargs): + if command[1] == 'fetch': + raise subprocess.CalledProcessError( + 1, command, stderr='fixture failure' + ) + return run(command, **kwargs) + + monkeypatch.setattr(fs.subprocess, 'run', failing) + snapshot = reader(repository) + with pytest.raises( + fs.FlatSnapshotError, match='Git fetch failed.*fixture failure' + ): + with snapshot: + pass + assert snapshot._directory is None + + +@pytest.mark.parametrize('checked_ref', ['FETCH_HEAD^{commit}', 'HEAD']) +def test_fetched_and_checked_out_commit_must_match_pin( + repository, monkeypatch, checked_ref +): + original = fs.FlatSnapshotReader._git + + def wrong_pin(self, *args): + if args == ('rev-parse', checked_ref): + return repository.tip + return original(self, *args) + + monkeypatch.setattr(fs.FlatSnapshotReader, '_git', wrong_pin) + snapshot = reader(repository) + with pytest.raises(fs.FlatSnapshotError, match='differs from pin'): + with snapshot: + pass + assert snapshot._directory is None + + +def test_non_commit_revision_rejected_before_git(repository, monkeypatch): + def unexpected(*_args, **_kwargs): + pytest.fail('Git must not run for a mutable revision') + + monkeypatch.setattr(fs.subprocess, 'run', unexpected) + with pytest.raises(fs.FlatSnapshotError, match='must be a commit SHA'): + with reader(repository, revision='main'): + pass + + +def test_lfs_filters_disabled_without_needing_git_lfs(repository, monkeypatch): + # An inherited filter that fails must never run during the sparse checkout. + monkeypatch.setenv('GIT_CONFIG_COUNT', '2') + monkeypatch.setenv('GIT_CONFIG_KEY_0', 'filter.lfs.process') + monkeypatch.setenv('GIT_CONFIG_VALUE_0', 'a-nonexistent-lfs-process') + monkeypatch.setenv('GIT_CONFIG_KEY_1', 'filter.lfs.required') + monkeypatch.setenv('GIT_CONFIG_VALUE_1', 'true') + (repository.source / '.gitattributes').write_text('*.json filter=lfs\n') + # Stage attributes only: existing pointer blobs remain ordinary Git blobs. + git(repository.source, 'add', '.gitattributes') + git(repository.source, 'commit', '--quiet', '-m', 'LFS attributes') + pin = git(repository.source, 'rev-parse', 'HEAD') + git(repository.remote, 'fetch', str(repository.source), pin) + repository.pin = pin + with reader(repository, revision=pin) as snapshot: + assert snapshot.read(LFS) == repository.payload + + +def test_auth_header_uses_environment_and_does_not_enter_git_arguments( + repository, +): + api = FakeApi(repository) + api.token = 'hf_fixture_secret' + snapshot = fs.FlatSnapshotReader(api, 'fixture/datastore') + assert ( + snapshot.remote_url + == 'https://example.invalid/datasets/fixture/datastore' + ) + config = { + snapshot._env[f'GIT_CONFIG_KEY_{i}']: snapshot._env[ + f'GIT_CONFIG_VALUE_{i}' + ] + for i in range(int(snapshot._env['GIT_CONFIG_COUNT'])) + } + assert config['http.https://example.invalid/.extraHeader'] == ( + 'Authorization: Bearer hf_fixture_secret' + ) + + +@pytest.mark.parametrize( + 'data', + [ + b'version https://git-lfs.github.com/spec/v2\n', + b'version https://git-lfs.github.com/spec/v1\noid sha256:bad\nsize 2\n', + b'version https://git-lfs.github.com/spec/v1\n\xff', + ], +) +def test_malformed_lfs_pointer_fails_explicitly(data): + with pytest.raises(fs.FlatSnapshotError, match='Malformed Git LFS pointer'): + fs._pointer(data, LFS) diff --git a/tests/test_helm_adapter.py b/tests/test_helm_adapter.py index 802901fcf..68a5b138d 100644 --- a/tests/test_helm_adapter.py +++ b/tests/test_helm_adapter.py @@ -551,3 +551,65 @@ def test_missing_model_deployment_falls_back_to_model(): assert converted_eval.model_info.id == 'openai/gpt2' assert converted_eval.model_info.developer == 'openai' assert converted_eval.model_info.inference_platform == 'unknown' + + +MMLU_RUN = ( + 'tests/data/helm/' + 'mmlu-subject=philosophy,method=multiple_choice_joint,model=openai_gpt2' +) +NARRATIVE_QA_RUN = 'tests/data/helm/narrative_qa-model=openai_gpt2' +_METADATA_ARGS = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, +} + + +@pytest.mark.parametrize( + ('run', 'method'), + [ + (HELLASWAG_RUN, 'multiple_choice_joint'), + (NARRATIVE_QA_RUN, 'generation'), + ], +) +def test_adapter_method_is_recorded_verbatim(run, method): + converted_eval = _load_eval(HELMAdapter(), run, _METADATA_ARGS) + + assert { + result.generation_config.additional_details['adapter_method'] + for result in converted_eval.evaluation_results + } == {method} + + +def test_hf_split_is_the_split_each_score_was_computed_on(): + """HELM's `valid` split is the schema's `val`.""" + converted_eval = _load_eval(HELMAdapter(), MMLU_RUN, _METADATA_ARGS) + + splits = { + (result.score_details.details['split'], result.source_data.hf_split) + for result in converted_eval.evaluation_results + } + assert splits == {('test', 'test'), ('valid', 'val')} + + +def test_sample_ids_count_each_instance_once(tmp_path): + """A run with two train trials requests every instance twice.""" + import copy + import json + import shutil + + destination = tmp_path / 'run' + shutil.copytree(Path(HELLASWAG_RUN), destination) + state_path = destination / 'scenario_state.json' + scenario_state = json.loads(state_path.read_text()) + second_trial = copy.deepcopy(scenario_state['request_states']) + for state in second_trial: + state['train_trial_index'] = 1 + scenario_state['request_states'] += second_trial + state_path.write_text(json.dumps(scenario_state), encoding='utf-8') + + converted_eval = _load_eval(HELMAdapter(), destination, _METADATA_ARGS) + + source_data = converted_eval.evaluation_results[0].source_data + assert source_data.samples_number == 10 + assert len(source_data.sample_ids) == 10 + assert len(set(source_data.sample_ids)) == 10 diff --git a/tests/test_hfopenllm_v2_adapter.py b/tests/test_hfopenllm_v2_adapter.py index 73e7e2fe5..ac0f51993 100644 --- a/tests/test_hfopenllm_v2_adapter.py +++ b/tests/test_hfopenllm_v2_adapter.py @@ -1,9 +1,30 @@ import json +import shutil +import subprocess +import sys +from pathlib import Path +import pytest + +from every_eval_ever.adapters.hfopenllm_v2 import adapter from every_eval_ever.adapters.hfopenllm_v2.adapter import ( + HARNESS_FORK, + contents_parquet_revision, convert_model, convert_models, + download_results, process_models, + results_dir_revision, +) +from every_eval_ever.adapters.hfopenllm_v2.contents import ( + CONTENTS_FILE, + CONTENTS_REVISION, + contents_row, +) +from every_eval_ever.adapters.hfopenllm_v2.dump_index import ( + RESULTS_REVISION, + build_dump_index, + dump_identity, ) from every_eval_ever.helpers.io import SourceRecordsError from every_eval_ever.validate import validate_file @@ -33,7 +54,9 @@ def test_single_model_conversion_remains_strict(): except ValueError as exc: assert "Evaluation 'bbh' could not be converted" in str(exc) else: - raise AssertionError('expected strict conversion to reject missing score') + raise AssertionError( + 'expected strict conversion to reject missing score' + ) def test_batch_keeps_valid_metrics_and_records_missing_metric(): @@ -82,10 +105,774 @@ def test_process_models_writes_valid_output_and_external_failure_report( assert len(outputs) == 1 assert validate_file(outputs[0]).valid - report_path = ( - tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' - ) + report_path = tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' report = json.loads(report_path.read_text()) assert report['converted_records'] == 1 assert len(report['failed_records']) == 1 assert not report_path.is_relative_to(output_dir) + + +RESULTS = Path(__file__).parent / 'data' / 'hfopenllm_v2' / 'results' +COGITO = 'Daemontatox/DocumentCogito' +DBRX = 'databricks/dbrx-base' + + +def _row(name, precision, keys=('ifeval', 'bbh', 'math', 'gpqa', 'musr')): + names = { + 'ifeval': 'IFEval', + 'bbh': 'BBH', + 'math': 'MATH Level 5', + 'gpqa': 'GPQA', + 'musr': 'MUSR', + 'mmlu_pro': 'MMLU-PRO', + } + return { + 'model': {'name': name, 'precision': precision}, + 'metadata': {}, + 'evaluations': { + key: {'name': names[key], 'value': 0.5} + for key in keys + ('mmlu_pro',) + }, + } + + +def _results(log): + return {r.evaluation_name: r for r in log.evaluation_results} + + +def _copy_results(tmp_path, *paths): + root = tmp_path / 'results' + for rel in paths: + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(RESULTS / rel, target) + return root + + +def _convert(rows, results_dir=RESULTS): + index = build_dump_index(results_dir) + result = convert_models(rows, retrieved_timestamp='1234', dump_index=index) + assert not result.failures + return [record.eval_log for record in result.records] + + +def test_record_matches_datastore_shape_without_dumps(): + (log,) = _convert([_row('example/model', 'bfloat16')], RESULTS / 'none') + + assert log.evaluation_id == 'hfopenllm_v2/example_model/bfloat16/1234' + ifeval = _results(log)['IFEval'] + assert ifeval.evaluation_result_id == ( + 'hfopenllm_v2/example_model/bfloat16/1234#ifeval#accuracy' + ) + assert _results(log)['MATH Level 5'].evaluation_result_id.endswith( + '#math_level_5#exact_match' + ) + metric = _results(log)['MMLU-PRO'].metric_config + assert ( + metric.metric_id, + metric.metric_name, + metric.metric_kind, + metric.metric_unit, + ) == ('accuracy', 'Accuracy', 'accuracy', 'proportion') + assert ifeval.generation_config is None + assert ifeval.source_data.hf_repo == 'google/IFEval' + assert ifeval.source_data.hf_split is None + assert ifeval.source_data.samples_number is None + assert log.eval_library.additional_details == { + 'fork': HARNESS_FORK, + } + + +def test_two_precisions_get_distinct_ids_and_their_own_dumps(): + bf16, fp16 = _convert([_row(COGITO, 'bfloat16'), _row(COGITO, 'float16')]) + + assert bf16.evaluation_id != fp16.evaluation_id + assert bf16.evaluation_id.endswith('/bfloat16/1234') + assert fp16.evaluation_id.endswith('/float16/1234') + assert bf16.eval_library.additional_details['transformers_version'] == ( + '4.48.0' + ) + assert fp16.eval_library.additional_details['transformers_version'] == ( + '4.49.0' + ) + assert bf16.eval_library.additional_details['git_hash'] == 'a781a6b' + + +def test_generative_groups_get_decoding_and_multiple_choice_do_not(): + (log,) = _convert([_row(COGITO, 'bfloat16')]) + results = _results(log) + + ifeval = results['IFEval'].generation_config + assert ifeval.generation_args.temperature == 0.0 + assert ifeval.generation_args.max_tokens == 1280 + assert ifeval.additional_details == { + 'output_type': 'generate_until', + 'num_fewshot': '0', + 'do_sample': 'false', + } + math = results['MATH Level 5'].generation_config + assert math.generation_args.max_tokens == 1024 + assert math.additional_details['num_fewshot'] == '4' + + bbh = results['BBH'].generation_config + assert bbh.generation_args is None + assert bbh.additional_details == { + 'output_type': 'multiple_choice', + 'num_fewshot': '3', + } + for name in ('GPQA', 'MUSR', 'MMLU-PRO'): + config = results[name].generation_config + assert config.generation_args is None + assert config.additional_details['output_type'] == 'multiple_choice' + + +def test_dataset_split_and_counts_come_from_the_dumps(): + (log,) = _convert([_row(COGITO, 'bfloat16')]) + results = _results(log) + for result in results.values(): + assert result.score_details.uncertainty.num_samples == ( + result.source_data.samples_number + ) + + ifeval = results['IFEval'].source_data + assert ifeval.hf_repo == 'google/IFEval' + assert ifeval.additional_details == { + 'harness_dataset_path': 'wis-k/instruction-following-eval' + } + assert ifeval.hf_split == 'train' + assert ifeval.samples_number == 541 + math = results['MATH Level 5'].source_data + assert math.hf_repo == 'DigitalLearningGmbH/MATH-lighteval' + assert math.additional_details == { + 'harness_dataset_path': 'lighteval/MATH-Hard' + } + assert math.hf_split == 'test' + assert math.samples_number == 1324 + assert results['BBH'].source_data.samples_number == 5761 + assert results['MMLU-PRO'].source_data.samples_number == 12032 + + gpqa = results['GPQA'].source_data + assert gpqa.samples_number == 1192 + assert gpqa.hf_split is None + musr = results['MUSR'].source_data + assert musr.hf_repo == 'TAUR-Lab/MuSR' + assert musr.additional_details is None + assert results['BBH'].source_data.additional_details is None + assert musr.samples_number == 756 + assert musr.hf_split is None + + +def test_limited_run_counts_scored_items_and_reports_dataset_size(): + (log,) = _convert([_row(DBRX, 'float16')]) + results = _results(log) + + math = results['MATH Level 5'] + assert math.generation_config.generation_args.max_tokens == 512 + assert math.source_data.samples_number == 700 + assert math.source_data.additional_details == { + 'harness_dataset_path': 'lighteval/MATH-Hard', + 'dataset_size': '1324', + } + bbh = results['BBH'].source_data + assert bbh.samples_number == 2400 + assert bbh.additional_details == {'dataset_size': '5761'} + + +def test_cross_dump_disagreement_leaves_field_null(tmp_path): + first, second = ( + f'{COGITO}/results_2025-01-16T02-47-58.653318.json', + f'{COGITO}/results_2025-02-13T18-27-04.338360.json', + ) + root = _copy_results(tmp_path, first, second) + path = root / second + dump = json.loads(path.read_text()) + for task, config in dump['configs'].items(): + if task.startswith('leaderboard_math_'): + config['generation_kwargs']['max_gen_toks'] = 512 + dump['n-samples']['leaderboard_ifeval']['effective'] = 100 + path.write_text(json.dumps(dump)) + + (log,) = _convert([_row(COGITO, 'bfloat16')], root) + results = _results(log) + + assert results['MATH Level 5'].generation_config is None + assert results['MATH Level 5'].source_data.samples_number == 1324 + assert results['IFEval'].source_data.samples_number is None + assert results['IFEval'].generation_config.generation_args.max_tokens == ( + 1280 + ) + index = build_dump_index(root) + gaps = index.lookup(COGITO, 'bfloat16').groups + assert gaps['math'].gaps['max_tokens'] == 'cross_dump_conflict' + assert gaps['ifeval'].gaps['samples_number'] == 'cross_dump_conflict' + + +def test_subtask_disagreement_within_a_dump_leaves_group_null(tmp_path): + rel = f'{DBRX}/results_2025-02-13T18-27-04.338360.json' + root = _copy_results(tmp_path, rel) + dump = json.loads((root / rel).read_text()) + dump['configs']['leaderboard_math_algebra_hard']['generation_kwargs'][ + 'max_gen_toks' + ] = 1024 + (root / rel).write_text(json.dumps(dump)) + + (log,) = _convert([_row(DBRX, 'float16')], root) + + math = _results(log)['MATH Level 5'] + assert math.generation_config is None + assert math.source_data.samples_number == 700 + + +def test_dump_contradicting_its_model_args_is_rejected(tmp_path): + rel = f'{DBRX}/results_2025-02-13T18-27-04.338360.json' + root = _copy_results(tmp_path, rel) + dump = json.loads((root / rel).read_text()) + dump['config']['model_args'] = dump['config']['model_args'].replace( + 'pretrained=databricks/dbrx-base', 'pretrained=databricks/dbrx-instruct' + ) + (root / rel).write_text(json.dumps(dump)) + + index = build_dump_index(root) + + assert index.entries == {} + assert index.rejected[0]['reason'] == 'identity_contradicts_model_args' + (log,) = _convert([_row(DBRX, 'float16')], root) + assert _results(log)['BBH'].generation_config is None + + +def test_adapter_dump_is_named_after_the_adapter(): + assert dump_identity( + { + 'model_name': 'org/adapter', + 'config': {'model_args': 'pretrained=org/base,peft=org/adapter'}, + } + ) == ('org/adapter', None) + assert dump_identity( + { + 'model_name': 'org/base', + 'config': {'model_args': 'pretrained=org/base,peft=org/adapter'}, + } + ) == (None, 'identity_contradicts_model_args') + + +def test_precision_without_a_dump_converts_with_nulls(): + (log,) = _convert([_row(DBRX, 'bfloat16')]) + + for result in log.evaluation_results: + assert result.generation_config is None + assert result.source_data.samples_number is None + assert result.source_data.hf_split is None + assert set(log.eval_library.additional_details) == {'fork'} + + +def test_process_models_output_validates_and_reports_gaps(tmp_path): + output_dir = tmp_path / 'data' / 'hfopenllm_v2' + rows = [ + _row(COGITO, 'bfloat16'), + _row(COGITO, 'float16'), + _row(DBRX, 'float16'), + _row(DBRX, 'bfloat16'), + ] + + count = process_models(rows, output_dir, build_dump_index(RESULTS)) + + assert count == 4 + outputs = sorted(output_dir.glob('*/*/*.json')) + assert len(outputs) == 4 + for path in outputs: + report = validate_file(path) + assert report.valid, report + cli = subprocess.run( + [sys.executable, '-m', 'every_eval_ever', 'validate', str(path)], + capture_output=True, + text=True, + ) + assert cli.returncode == 0, cli.stdout + cli.stderr + + report = json.loads( + ( + tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' + ).read_text() + ) + assert report['failed_record_count'] == 0 + counts = report['harness_evidence']['unfilled_counts'] + assert counts['gpqa.hf_split: not_stated'] == 3 + assert counts['musr.hf_split: subtask_conflict'] == 3 + assert counts['ifeval.all: no_dump'] == 1 + assert report['harness_evidence']['dumps_read'] == 4 + + +OTHER_SHA = 'b' * 40 + + +def _snapshot(tmp_path, name): + root = tmp_path / 'snapshots' / name + shutil.copytree(RESULTS, root) + return root + + +def _run_main(monkeypatch, tmp_path, results_dir, *extra): + monkeypatch.setattr( + adapter, 'fetch_json', lambda url: [_row(DBRX, 'float16')] + ) + output_dir = tmp_path / 'data' / 'hfopenllm_v2' + adapter.main( + [ + '--output-dir', + str(output_dir), + '--source-api', + '--results-dir', + str(results_dir), + *extra, + ] + ) + report = tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' + return json.loads(report.read_text())['harness_evidence'] + + +def test_results_dir_at_the_pinned_revision_is_accepted(monkeypatch, tmp_path): + root = _snapshot(tmp_path, RESULTS_REVISION) + + assert results_dir_revision(root) == (RESULTS_REVISION, False) + evidence = _run_main(monkeypatch, tmp_path, root) + assert evidence['results_revision'] == RESULTS_REVISION + assert evidence['results_dir_unpinned'] is False + + +def test_results_dir_at_another_revision_fails_naming_both( + monkeypatch, tmp_path +): + root = _snapshot(tmp_path, OTHER_SHA) + monkeypatch.setattr( + adapter, + 'fetch_json', + lambda url: pytest.fail('fetched before checking the results dir'), + ) + + with pytest.raises(SystemExit) as exc: + adapter.main(['--results-dir', str(root)]) + assert OTHER_SHA in str(exc.value) + assert RESULTS_REVISION in str(exc.value) + with pytest.raises(ValueError, match='no commit sha'): + results_dir_revision(RESULTS) + + +def test_unpinned_results_dir_is_recorded_and_reported(monkeypatch, tmp_path): + recorded = [] + monkeypatch.setattr( + adapter.raw_capture, + 'record_hf_dataset', + lambda repo, **kwargs: recorded.append(kwargs['revision']), + ) + root = _snapshot(tmp_path, OTHER_SHA) + + evidence = _run_main( + monkeypatch, tmp_path, root, '--allow-unpinned-results-dir' + ) + + assert recorded == [OTHER_SHA] + assert evidence['results_revision'] == OTHER_SHA + assert evidence['results_dir_unpinned'] is True + assert results_dir_revision(RESULTS, allow_unpinned=True) == ( + 'local', + True, + ) + + +@pytest.fixture +def results_git_remote(tmp_path, monkeypatch, request): + if shutil.which('git') is None: + pytest.skip('Git is required for the local bulk-fetch fixture') + + def git(*args, cwd=None): + return subprocess.run( + ['git', *args], cwd=cwd, capture_output=True, text=True, check=True + ).stdout.strip() + + work = tmp_path / 'source repository' + work.mkdir() + git('init', '--initial-branch=fixture', cwd=work) + git('config', 'user.name', 'Fixture', cwd=work) + git('config', 'user.email', 'fixture@example.invalid', cwd=work) + (work / 'README.md').write_text('First commit, without result dumps.\n') + git('add', '.', cwd=work) + git('commit', '-m', 'Initial fixture', cwd=work) + + cached = tmp_path / 'cached snapshot' + shutil.copytree(RESULTS, cached) + spaced = cached / 'extra organization' / 'model with spaces' + spaced.mkdir(parents=True) + shutil.copy( + next((RESULTS / DBRX).glob('results_*.json')), + spaced / 'results_extra.json', + ) + for relative in ( + 'single/results_shallow.json', + 'one/two/three/results_deep.json', + ): + target = cached / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(spaced / 'results_extra.json', target) + if getattr(request, 'param', None) == 'lfs_pointer': + (spaced / 'results_future.json').write_text( + 'version https://git-lfs.github.com/spec/v1\n' + f'oid sha256:{"a" * 64}\n' + 'size 1234\n' + ) + shutil.copytree(cached, work, dirs_exist_ok=True) + shutil.copy(spaced / 'results_extra.json', work / 'results_root.json') + git('add', '.', cwd=work) + git('commit', '-m', 'Pinned fixture results', cwd=work) + pinned = git('rev-parse', 'HEAD', cwd=work) + + dump = next((work / DBRX).glob('results_*.json')) + dump.unlink() + (work / 'README.md').write_text('Later commit, with a result removed.\n') + git('add', '.', cwd=work) + git('commit', '-m', 'Later fixture', cwd=work) + latest = git('rev-parse', 'HEAD', cwd=work) + remote = tmp_path / 'remote with spaces.git' + git('clone', '--bare', str(work), str(remote)) + + monkeypatch.setattr(adapter, 'RESULTS_URL', remote.as_uri()) + monkeypatch.setattr(adapter, 'RESULTS_REVISION', pinned) + return cached, pinned, latest + + +def test_results_bulk_fetch_matches_cached_index_and_conversion( + results_git_remote, tmp_path, monkeypatch +): + cached, pinned, latest = results_git_remote + commands = [] + actual_run = subprocess.run + + def record_command(args, **kwargs): + commands.append(args) + return actual_run(args, **kwargs) + + monkeypatch.setattr(adapter.subprocess, 'run', record_command) + monkeypatch.setattr( + adapter.raw_capture, + 'record_hf_dataset', + lambda *args, **kwargs: pytest.fail('unexpected per-file Hub path'), + ) + captured = [] + monkeypatch.setattr( + adapter.raw_capture, + 'record_git_checkout', + lambda url, checkout, **kwargs: captured.append( + (url, checkout, kwargs['ref']) + ), + ) + + root = download_results(tmp_path / 'download with spaces') + + assert results_dir_revision(root) == (pinned, False) + assert pinned != latest + assert (root / '.git' / 'shallow').read_text().splitlines() == [pinned] + assert not (root / 'README.md').exists() + assert not (root / 'results_root.json').exists() + assert not list((root / '.git' / 'objects' / 'pack').glob('*.promisor')) + fetches = [command for command in commands if command[1] == 'fetch'] + assert fetches == [ + ['git', 'fetch', '--depth', '1', adapter.RESULTS_URL, pinned] + ] + assert all( + '--filter' not in argument + for command in commands + for argument in command + ) + assert captured == [(adapter.RESULTS_URL, root, pinned)] + assert build_dump_index(root) == build_dump_index(cached) + rows = [_row(COGITO, 'bfloat16'), _row(DBRX, 'float16')] + assert _convert(rows, root) == _convert(rows, cached) + for expected in cached.rglob('results_*.json'): + assert ( + root / expected.relative_to(cached) + ).read_bytes() == expected.read_bytes() + + +@pytest.mark.parametrize('results_git_remote', ['lfs_pointer'], indirect=True) +def test_results_lfs_pointer_fails_loudly_after_bulk_fetch( + results_git_remote, tmp_path, monkeypatch +): + cached, _, _ = results_git_remote + root = download_results(tmp_path / 'results') + monkeypatch.setattr( + adapter.subprocess, + 'run', + lambda *args, **kwargs: pytest.fail('unexpected extra Git transfer'), + ) + for results_dir in (cached, root): + with pytest.raises( + RuntimeError, match='results_future.json is a Git LFS pointer' + ): + build_dump_index(results_dir) + + +def test_results_git_checkout_does_not_trust_directory_name( + results_git_remote, tmp_path, monkeypatch +): + _, pinned, latest = results_git_remote + root = download_results(tmp_path / latest) + assert results_dir_revision(root) == (pinned, False) + + monkeypatch.setattr(adapter, 'RESULTS_REVISION', latest) + with pytest.raises(ValueError, match=pinned): + results_dir_revision(root) + assert results_dir_revision(root, allow_unpinned=True) == (pinned, True) + + +def test_results_fetch_requires_git(tmp_path, monkeypatch): + def missing_git(*args, **kwargs): + raise FileNotFoundError('git') + + monkeypatch.setattr(adapter.subprocess, 'run', missing_git) + with pytest.raises(RuntimeError, match='Git is required'): + download_results(tmp_path / 'results') + + +def test_results_fetch_does_not_modify_nonempty_destination(tmp_path): + existing = tmp_path / 'keep.txt' + existing.write_text('caller data') + with pytest.raises(ValueError, match='must be empty'): + download_results(tmp_path) + assert existing.read_text() == 'caller data' + assert not (tmp_path / '.git').exists() + + +def test_results_fetch_capture_records_verified_checkout( + results_git_remote, tmp_path, monkeypatch +): + _, pinned, _ = results_git_remote + sink = adapter.raw_capture.RawSink(tmp_path / 'capture') + monkeypatch.setattr(adapter.raw_capture, 'active_sink', lambda: sink) + monkeypatch.setattr( + adapter.raw_capture, + 'record_hf_dataset', + lambda *args, **kwargs: pytest.fail('unexpected Hub lookup'), + ) + + download_results(tmp_path / 'results') + + (entry,) = sink.entries() + assert entry['kind'] == 'pointer' + assert entry['pointer_kind'] == 'git' + assert entry['reference'] == adapter.RESULTS_URL + assert entry['revision'] == pinned + assert not sink.degraded + + +@pytest.mark.parametrize('wrong_step', ['fetch', 'checkout']) +def test_results_fetch_rejects_wrong_commit( + results_git_remote, tmp_path, monkeypatch, wrong_step +): + _, _, latest = results_git_remote + actual_git = adapter._git + + def wrong_revision(*args, **kwargs): + result = actual_git(*args, **kwargs) + target = ( + 'FETCH_HEAD^{commit}' if wrong_step == 'fetch' else 'HEAD^{commit}' + ) + return latest if args == ('rev-parse', target) else result + + monkeypatch.setattr(adapter, '_git', wrong_revision) + with pytest.raises(RuntimeError, match=f'commit {latest}, expected'): + download_results(tmp_path / 'results') + + +def test_results_fetch_failure_has_no_hub_fallback(tmp_path, monkeypatch): + commands = [] + + def failed_fetch(args, **kwargs): + commands.append(args) + if args[1] == 'fetch': + raise subprocess.CalledProcessError( + 128, args, stderr='Pinned commit is unavailable' + ) + return subprocess.CompletedProcess(args, 0, stdout='') + + monkeypatch.setattr(adapter.subprocess, 'run', failed_fetch) + with pytest.raises(RuntimeError, match='Pinned commit is unavailable'): + download_results(tmp_path / 'results') + assert [command[1] for command in commands] == ['init', 'fetch'] + + +@pytest.mark.parametrize('fail_indexing', [False, True]) +def test_main_cleans_temporary_results_checkout( + results_git_remote, tmp_path, monkeypatch, fail_indexing +): + workspaces = [] + actual_download = download_results + actual_index = build_dump_index + + def downloaded(destination): + workspaces.append(Path(destination)) + return actual_download(destination) + + def index_results(root): + if fail_indexing: + raise ValueError('Cannot index results') + return actual_index(root) + + monkeypatch.setattr(adapter, 'download_results', downloaded) + monkeypatch.setattr(adapter, 'build_dump_index', index_results) + monkeypatch.setattr(adapter, 'fetch_json', lambda url: []) + monkeypatch.setattr(adapter, 'process_models', lambda *args: 0) + argv = ['--source-api', '--output-dir', str(tmp_path / 'outputs')] + if fail_indexing: + with pytest.raises(ValueError, match='Cannot index results'): + adapter.main(argv) + else: + assert adapter.main(argv) == 0 + assert len(workspaces) == 1 + assert not workspaces[0].exists() + + +def test_stated_model_details_are_kept_with_the_library_version(): + source = _row('example/model', 'float16') + source['model']['architecture'] = 'LlamaForCausalLM' + source['metadata']['params_billions'] = 4.65 + + log = convert_model(source, '1234') + + details = log.model_info.additional_details + assert details['precision'] == 'float16' + assert details['architecture'] == 'LlamaForCausalLM' + assert details['params_billions'] == '4.65' + assert log.eval_library.name == 'lm-evaluation-harness' + assert log.eval_library.version == '0.4.0' + + +def test_split_does_not_depend_on_the_harness_dataset_path(tmp_path): + rel = f'{DBRX}/results_2025-02-13T18-27-04.338360.json' + root = _copy_results(tmp_path, rel) + dump = json.loads((root / rel).read_text()) + dump['configs']['leaderboard_math_algebra_hard']['dataset_path'] = 'x/y' + (root / rel).write_text(json.dumps(dump)) + + (log,) = _convert([_row(DBRX, 'float16')], root) + + math = _results(log)['MATH Level 5'].source_data + assert math.hf_repo == 'DigitalLearningGmbH/MATH-lighteval' + assert math.hf_split == 'test' + assert math.additional_details == {'dataset_size': '1324'} + + +CONTENTS = ( + Path(__file__).parent / 'data' / 'hfopenllm_v2' / 'contents' / CONTENTS_FILE +) + + +def test_contents_row_has_the_api_row_shape(): + row = contents_row( + { + 'fullname': 'org/model', + 'Precision': 'float16', + 'Architecture': 'LlamaForCausalLM', + '#Params (B)': float('nan'), + 'IFEval Raw': 0.5, + 'IFEval': 50.0, + 'MATH Lvl 5 Raw': None, + } + ) + + assert row['model'] == { + 'name': 'org/model', + 'precision': 'float16', + 'architecture': 'LlamaForCausalLM', + } + assert row['metadata'] == {} + assert row['evaluations']['ifeval'] == {'name': 'IFEval', 'value': 0.5} + assert row['evaluations']['math'] == { + 'name': 'MATH Level 5', + 'value': None, + } + assert set(row['evaluations']) == { + 'ifeval', + 'bbh', + 'math', + 'gpqa', + 'musr', + 'mmlu_pro', + } + + +def test_contents_parquet_rows_convert(): + pytest.importorskip('pyarrow') + from every_eval_ever.adapters.hfopenllm_v2.contents import load_contents + + rows = load_contents(CONTENTS) + + by_key = {(r['model']['name'], r['model']['precision']): r for r in rows} + assert set(by_key) == { + (COGITO, 'bfloat16'), + (COGITO, 'float16'), + (DBRX, 'float16'), + ('gpt2', 'float16'), + } + dbrx = by_key[(DBRX, 'float16')] + assert dbrx['model']['architecture'] == 'Unknown' + assert dbrx['metadata']['params_billions'] == 0.0 + assert 0 <= dbrx['evaluations']['math']['value'] <= 1 + + +def test_main_reads_pinned_contents_and_results(tmp_path): + pytest.importorskip('pyarrow') + contents = tmp_path / 'contents' / CONTENTS_REVISION / CONTENTS_FILE + contents.parent.mkdir(parents=True) + shutil.copy(CONTENTS, contents) + results = _snapshot(tmp_path, RESULTS_REVISION) + output_dir = tmp_path / 'data' / 'hfopenllm_v2' + + with pytest.raises(SourceRecordsError, match='gpt2'): + adapter.main( + [ + '--output-dir', + str(output_dir), + '--contents-parquet', + str(contents), + '--results-dir', + str(results), + ] + ) + + outputs = sorted(output_dir.glob('*/*/*.json')) + assert len(outputs) == 3 + for path in outputs: + assert validate_file(path).valid + report = json.loads( + ( + tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' + ).read_text() + ) + assert report['leaderboard_input'] == { + 'dataset': 'open-llm-leaderboard/contents', + 'revision': CONTENTS_REVISION, + 'unpinned': False, + } + assert report['failed_record_count'] == 1 + + +def test_contents_parquet_pinning(tmp_path): + other = tmp_path / OTHER_SHA / CONTENTS_FILE + + assert contents_parquet_revision( + tmp_path / CONTENTS_REVISION / CONTENTS_FILE + ) == (CONTENTS_REVISION, False) + with pytest.raises(ValueError) as exc: + contents_parquet_revision(other) + assert OTHER_SHA in str(exc.value) + assert CONTENTS_REVISION in str(exc.value) + assert '--allow-unpinned-contents-parquet' in str(exc.value) + assert contents_parquet_revision(other, allow_unpinned=True) == ( + OTHER_SHA, + True, + ) + assert contents_parquet_revision(CONTENTS, allow_unpinned=True) == ( + 'local', + True, + ) + with pytest.raises(SystemExit): + adapter.main(['--contents-parquet', str(CONTENTS)]) diff --git a/tests/test_hle_adapter.py b/tests/test_hle_adapter.py index 7d33e7a78..b85091cae 100644 --- a/tests/test_hle_adapter.py +++ b/tests/test_hle_adapter.py @@ -159,17 +159,102 @@ def test_metric_config_uses_percent_scale_with_judge(): assert judge.model_info.id == adapter.JUDGE_MODEL_ID -def test_generation_config_carries_model_temperature(): - """Per reviewer feedback on HF datastore PR #125: model-under-eval - temperature belongs on generation_config.generation_args.temperature - (schema L400), not in source/metric additional_details. - """ +def _log_for_note(note: str | None) -> EvaluationLog: + row = { + 'model': 'test-model', + 'rank': 1, + 'score': 10.0, + 'company': 'openai', + 'calibrationError': 40, + } + if note is not None: + row['contaminationMessage'] = note + log = adapter.make_logs([row], retrieved_timestamp='123.0')[0][0] + assert len(log.evaluation_results) == 2 + return log + + +def test_no_row_note_leaves_temperature_null_and_records_policy(): + log = _log_for_note(None) + for result in log.evaluation_results: + config = result.generation_config + assert config is None or config.generation_args is None + assert ( + log.source_metadata.additional_details['source_generation_policy'] + == adapter.SOURCE_GENERATION_POLICY + ) + assert 'temperature 0.0 when configurable' in ( + adapter.SOURCE_GENERATION_POLICY + ) + assert 'source_row_note' not in log.model_info.additional_details + + +def test_row_note_with_temperature_and_top_p_is_parsed(): + note = 'Sampled at temperature = 1.0, top_p = 0.95.' + log = _log_for_note(note) + for result in log.evaluation_results: + args = result.generation_config.generation_args + assert args.temperature == 1.0 + assert args.top_p == 0.95 + assert ( + result.generation_config.additional_details['source_row_note'] + == note + ) + assert log.model_info.additional_details['source_row_note'] == note + + +def test_row_note_with_temperature_only_is_parsed(): + log = _log_for_note(' Sampled at temperature 0.7 ') + for result in log.evaluation_results: + args = result.generation_config.generation_args + assert args.temperature == 0.7 + assert args.top_p is None + + +def test_default_temperature_note_parses_nothing(): + note = 'Sampled at 32K Tokens, temp = null (default temp)' + log = _log_for_note(note) + for result in log.evaluation_results: + assert result.generation_config.generation_args is None + assert ( + result.generation_config.additional_details['source_row_note'] + == note + ) + + +def test_budget_and_effort_notes_are_kept_but_not_parsed(): + for note in ( + 'Thinking budget: 16,000 tokens.', + "Sampled at reasoning_effort: 'high'.", + ): + log = _log_for_note(note) + for result in log.evaluation_results: + assert result.generation_config.generation_args is None + assert ( + result.generation_config.additional_details['source_row_note'] + == note + ) + + +def test_note_mentioning_temperature_in_other_prose_parses_nothing(): + for note in ( + 'Sampled at temperature 0.7 with a custom system prompt.', + 'Note: Sampled at temperature = 1.0, top_p = 0.95.', + 'Sampled at temperature = 1.0, top_p = 0.95', + 'The provider ignores temperature settings for this model.', + ): + log = _log_for_note(note) + for result in log.evaluation_results: + assert result.generation_config.generation_args is None + + +def test_judge_prompt_carries_no_temperature(): + assert 'temperature' not in adapter.JUDGE_PROMPT_DESCRIPTION.lower() bundles = adapter.make_logs(sample_rows(), retrieved_timestamp='123.0') for log, _, _ in bundles: for result in log.evaluation_results: - assert result.generation_config is not None - assert result.generation_config.generation_args is not None - assert result.generation_config.generation_args.temperature == 0.0 + prompt = result.metric_config.llm_scoring.input_prompt + assert 'temperature' not in prompt.lower() def test_judge_and_temperature_not_duplicated_in_additional_details(): diff --git a/tests/test_inspect_adapter.py b/tests/test_inspect_adapter.py index 27cbd418f..76529cc97 100644 --- a/tests/test_inspect_adapter.py +++ b/tests/test_inspect_adapter.py @@ -6,6 +6,7 @@ ) import contextlib +import json import logging import tempfile from pathlib import Path @@ -29,6 +30,7 @@ SourceDataUrl, SourceMetadata, ) +from tests.converter_cases import CASES TEST_UUID = '123e4567-e89b-42d3-a456-426614174000' OTHER_TEST_UUID = '123e4567-e89b-42d3-a456-426614174001' @@ -472,6 +474,280 @@ def test_humaneval_eval(): assert converted_eval.detailed_evaluation_results is not None +CYSE2_LOG = 'tests/data/inspect/data_cyse2_vuln_exploit_challenges.json' +GAIA_LOG = 'tests/data/inspect/2026-02-07T11-26-57+00-00_gaia_4V8zHbbRKpU5Yv2BMoBcjE.json' +_METADATA_ARGS = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, +} + + +def _load_rewritten_eval(tmp_path, fixture, rewrite): + """Convert a copy of an Inspect fixture with `rewrite` applied to its JSON.""" + log = json.loads(Path(fixture).read_text(encoding='utf-8')) + rewrite(log) + path = tmp_path / Path(fixture).name + path.write_text(json.dumps(log), encoding='utf-8') + return _load_eval(InspectAIAdapter(), path, _METADATA_ARGS) + + +def test_generation_args_come_from_the_plan_config(): + """Inspect runs every call under the model config merged with the plan config. + + This log's model config is empty and its task sets temperature and max_tokens, + which only the plan config carries. + """ + case = next(case for case in CASES if case.source == 'inspect') + converted_eval = _load_eval( + InspectAIAdapter(), case.log_path, _METADATA_ARGS + ) + + for result in converted_eval.evaluation_results: + generation_config = result.generation_config + args = generation_config.generation_args + assert args.temperature == 0.75 + assert args.max_tokens == 2048 + assert args.top_p is None + assert args.reasoning is None + details = generation_config.additional_details + assert details['generation_args_source'] == 'plan_config' + assert details['terminal_solver'] == 'generate' + assert details['temperature'] == '0.75' + assert details['max_tokens'] == '2048' + + +def test_max_tokens_is_the_plan_configs_per_call_cap(): + converted_eval = _load_eval( + InspectAIAdapter(), + 'tests/data/inspect/data_with_llm_scoring.json', + _METADATA_ARGS, + ) + + result = converted_eval.evaluation_results[0] + args = result.generation_config.generation_args + assert args.max_tokens == 128000 + assert args.temperature is None + + +def test_generation_args_from_the_model_config_alone_say_so(tmp_path): + def rewrite(log): + log['eval']['model_generate_config'] = {'temperature': 0.3} + log['plan']['config'] = {} + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + assert generation_config.generation_args.temperature == 0.3 + assert generation_config.generation_args.max_tokens is None + assert ( + generation_config.additional_details['generation_args_source'] + == 'model_config' + ) + + +@pytest.mark.parametrize( + ('step', 'temperature', 'max_tokens', 'overridden'), + [ + ( + {'solver': 'generate', 'params': {'temperature': 0.1}}, + None, + 2048, + 'temperature', + ), + ( + { + 'solver': 'generate', + 'params': {'kwargs': {'max_completion_tokens': 16}}, + }, + 0.75, + None, + 'max_tokens', + ), + ( + { + 'solver': 'my_agent', + 'params': {}, + 'params_passed': {'config': {'max_output_tokens': 16}}, + }, + 0.75, + None, + 'max_tokens', + ), + ( + {'solver': 'my_agent', 'params': {'model': 'openai/gpt-4o'}}, + None, + None, + 'max_tokens,temperature,top_k,top_p', + ), + ( + {'solver': 'my_agent', 'params': {'model': None}}, + 0.75, + 2048, + None, + ), + ], + ids=[ + 'step-temperature', + 'nested-max-tokens', + 'passed-only', + 'model', + 'model-default-none', + ], +) +def test_a_step_that_sets_its_own_generation_args_leaves_them_unstated( + tmp_path, step, temperature, max_tokens, overridden +): + """A solver passing its own value overrides the plan config for its calls.""" + + def rewrite(log): + log['plan']['steps'] = [step] + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + details = generation_config.additional_details + assert generation_config.generation_args.temperature == temperature + assert generation_config.generation_args.max_tokens == max_tokens + assert details.get('temperature') == ( + None if temperature is None else '0.75' + ) + assert details.get('max_tokens') == (None if max_tokens is None else '2048') + assert details.get('overridden_by_step') == overridden + assert details['terminal_solver'] == step['solver'] + if temperature is None and max_tokens is None: + assert 'generation_args_source' not in details + + +def test_generation_args_from_both_configs_are_mixed(tmp_path): + def rewrite(log): + log['eval']['model_generate_config'] = {'top_p': 0.9} + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + assert generation_config.generation_args.top_p == 0.9 + assert generation_config.generation_args.temperature == 0.75 + assert ( + generation_config.additional_details['generation_args_source'] + == 'mixed' + ) + + +def test_a_tool_argument_named_like_a_generation_arg_does_not_override( + tmp_path, +): + tool = { + 'type': 'tool', + 'name': 'sampler', + 'params': {'temperature': 0.1, 'max_tokens': 16}, + } + + def rewrite(log): + log['plan']['steps'] = [ + { + 'solver': 'use_tools', + 'params': {'tools': [[tool]], 'append': True}, + 'params_passed': {'tools': [[tool]]}, + }, + {'solver': 'generate', 'params': {}}, + ] + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + assert generation_config.generation_args.temperature == 0.75 + assert generation_config.generation_args.max_tokens == 2048 + + +@pytest.mark.parametrize( + ('reasoning_effort', 'reasoning'), + [('high', True), ('none', False)], +) +def test_reasoning_is_what_the_log_states( + tmp_path, reasoning_effort, reasoning +): + def rewrite(log): + log['plan']['config']['reasoning_effort'] = reasoning_effort + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + result = converted_eval.evaluation_results[0] + args = result.generation_config.generation_args + assert args.reasoning is reasoning + + +def test_gaia_states_its_split_and_how_many_samples_it_scored(): + converted_eval = _load_eval(InspectAIAdapter(), GAIA_LOG, _METADATA_ARGS) + + result = converted_eval.evaluation_results[0] + args = result.generation_config.generation_args + assert args.temperature == 0.5 + assert args.reasoning is None + assert result.generation_config.additional_details['temperature'] == '0.5' + assert ( + result.generation_config.additional_details['terminal_solver'] + == 'basic_agent_loop' + ) + + details = result.source_data.additional_details + assert details['samples_number'] == '2' + assert details['dataset_size'] == '165' + assert details['split'] == 'val' + + +@pytest.mark.parametrize( + ('split', 'hf_split'), + [('test', 'test'), ('validation', 'val'), ('dev', None)], +) +def test_hf_split_is_taken_only_from_a_schema_split(tmp_path, split, hf_split): + def rewrite(log): + log['eval']['task_args'] = {'split': split} + + converted_eval = _load_rewritten_eval( + tmp_path, 'tests/data/inspect/data_arc_qwen.json', rewrite + ) + + source_data = converted_eval.evaluation_results[0].source_data + assert source_data.hf_split == hf_split + assert source_data.samples_number == 3 + assert source_data.additional_details['dataset_size'] == '2376' + + +def test_a_complete_run_states_no_separate_dataset_size(): + converted_eval = _load_eval( + InspectAIAdapter(), + 'tests/data/inspect/data_with_llm_scoring.json', + _METADATA_ARGS, + ) + + source_data = converted_eval.evaluation_results[0].source_data + assert source_data.samples_number == 1000 + assert source_data.hf_split is None + assert 'dataset_size' not in source_data.additional_details + + +def test_library_version_is_inspect_ai_and_other_packages_are_details(): + converted_eval = _load_eval(InspectAIAdapter(), GAIA_LOG, _METADATA_ARGS) + + assert converted_eval.eval_library.name == 'inspect_ai' + assert converted_eval.eval_library.version == '0.3.171' + assert converted_eval.eval_library.additional_details == { + 'inspect_evals': '0.3.106' + } + + +def test_library_version_without_inspect_ai_package_is_unknown(tmp_path): + def rewrite(log): + log['eval']['packages'] = {'inspect_evals': '0.3.106'} + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + assert converted_eval.eval_library.version == 'unknown' + assert converted_eval.eval_library.additional_details == { + 'inspect_evals': '0.3.106' + } + + def test_extract_evaluation_results_one_scorer_with_two_metrics(): adapter = InspectAIAdapter() source_data = SourceDataHf( @@ -512,7 +788,8 @@ def test_extract_evaluation_results_one_scorer_with_two_metrics(): assert result_ids_by_scorer == {'choice': ['choice:accuracy', 'choice:f1']} -def test_each_scorer_counts_the_samples_it_could_score(): +@pytest.mark.parametrize('private', [False, True]) +def test_each_scorer_counts_the_samples_it_could_score(private): """Inspect computes a scorer's metrics over the samples that scorer scored, and says how many those were; the run-wide count includes the ones it skipped.""" adapter = InspectAIAdapter() @@ -525,12 +802,21 @@ def test_each_scorer_counts_the_samples_it_could_score(): _make_scorer('lenient', {'accuracy': _make_metric('accuracy', 0.9)}), ] + source = ( + SourceDataPrivate( + dataset_name='synthetic_ds', + source_type='other', + additional_details={'samples_number': '10'}, + ) + if private + else SourceDataHf( + dataset_name='synthetic_ds', source_type='hf_dataset', samples_number=10 + ) + ) results, _ = adapter._extract_evaluation_results( evaluation_task_name='synthetic/task', scores=scores, - source_data=SourceDataHf( - dataset_name='synthetic_ds', source_type='hf_dataset' - ), + source_data=source, generation_config=GenerationConfig(), num_samples=10, timestamp='1234567890', @@ -543,6 +829,35 @@ def test_each_scorer_counts_the_samples_it_could_score(): assert by_id['strict:accuracy'].num_samples == 7 # No count of its own, so the run's stands in. assert by_id['lenient:accuracy'].num_samples == 10 + counts = [ + int(result.source_data.additional_details['samples_number']) + if private + else result.source_data.samples_number + for result in results + ] + assert counts == [7, 10] + assert ( + int(source.additional_details['samples_number']) + if private + else source.samples_number + ) == 10 + + +def test_unverified_scaffold_cannot_supply_plan_generation_defaults(tmp_path): + def rewrite(log): + log['eval']['packages']['private_scaffold'] = '1.2.3' + log['eval']['model_generate_config'] = {'top_p': 0.9} + + converted = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + config = converted.evaluation_results[0].generation_config + assert config.generation_args.temperature is None + assert config.generation_args.max_tokens is None + assert config.generation_args.top_p == 0.9 + assert config.generation_args.eval_plan.config['max_tokens'] == '2048' + assert config.additional_details['generation_args_source'] == 'model_config' + assert config.additional_details['plan_config_unverified_packages'] == ( + 'private_scaffold' + ) def test_standard_error_says_how_inspect_computed_it(): @@ -802,7 +1117,10 @@ def test_supplemental_eval_details_fill_only_top_level_fields(): assert result.source_data.additional_details['subset'] == '{"name": "full"}' assert result.generation_config is not None - assert result.generation_config.additional_details == {'runner': 'inspect'} + assert result.generation_config.additional_details == { + 'runner': 'inspect', + 'terminal_solver': 'multiple_choice', + } assert result.generation_config.generation_args is not None assert ( result.generation_config.generation_args.agentic_eval_config is not None diff --git a/tests/test_lm_eval_adapter.py b/tests/test_lm_eval_adapter.py index cb94833f1..0d688c6e4 100644 --- a/tests/test_lm_eval_adapter.py +++ b/tests/test_lm_eval_adapter.py @@ -242,6 +242,59 @@ def test_transform_from_file_generation_config(): mode='json', exclude_none=True ) assert gen.additional_details['num_fewshot'] == '0' + assert gen.additional_details['output_type'] == 'generate_until' + + +def test_source_data_states_the_samples_scored(): + adapter = LMEvalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + + for log in logs: + source_data = log.evaluation_results[0].source_data + assert source_data.samples_number == 5000 + assert source_data.additional_details is None + + +def _multiple_choice_task(n_samples): + return { + 'results': {'mc_task': {'acc,none': 0.25, 'acc_stderr,none': 0.01}}, + 'configs': { + 'mc_task': { + 'task': 'mc_task', + 'dataset_path': 'org/mc_dataset', + 'test_split': 'test', + 'output_type': 'multiple_choice', + 'num_fewshot': 5, + } + }, + 'n-samples': {'mc_task': n_samples}, + } + + +def test_multiple_choice_task_states_its_scoring_mode_and_no_generation_args(): + adapter = LMEvalAdapter() + results = adapter._build_evaluation_results( + _multiple_choice_task({'original': 1000, 'effective': 1000}), 'mc_task' + ) + + gen = results[0].generation_config + assert gen.generation_args is None + assert gen.additional_details == { + 'output_type': 'multiple_choice', + 'num_fewshot': '5', + } + + +def test_a_limited_run_states_how_many_samples_it_scored(): + adapter = LMEvalAdapter() + results = adapter._build_evaluation_results( + _multiple_choice_task({'original': 1000, 'effective': 100}), 'mc_task' + ) + + source_data = results[0].source_data + assert source_data.samples_number == 100 + assert source_data.additional_details == {'dataset_size': '1000'} + assert results[0].score_details.uncertainty.num_samples == 100 def test_transform_from_file_eval_timestamp(): @@ -618,7 +671,8 @@ def test_directory_conversion_tracks_each_results_file_parent(tmp_path): second.write_text(json.dumps(second_source), encoding='utf-8') result = adapter.transform_from_directory_result( - tmp_path, {'parent_eval_output_dir': str(tmp_path)} + tmp_path, + _make_metadata_args(parent_eval_output_dir=str(tmp_path)), ) parents = { diff --git a/tests/test_mmlu_pro_adapter.py b/tests/test_mmlu_pro_adapter.py index 0c337126c..fcdc2bb7a 100644 --- a/tests/test_mmlu_pro_adapter.py +++ b/tests/test_mmlu_pro_adapter.py @@ -385,3 +385,104 @@ def test_source_data_is_hf_dataset_pointing_at_results_repo(): overall = log.evaluation_results[0] assert overall.source_data.source_type == 'hf_dataset' assert overall.source_data.hf_repo == adapter.RESULTS_HF_REPO + + +def _prompt_styles(log: EvaluationLog) -> set[str | None]: + styles: set[str | None] = set() + for result in log.evaluation_results: + styles.add( + (result.source_data.additional_details or {}).get('prompt_style') + ) + styles.add( + (result.metric_config.additional_details or {}).get('prompt_style') + ) + return styles + + +def test_rows_without_shot_marker_carry_no_prompt_style(): + bundles = adapter.make_logs(sample_rows(), retrieved_timestamp='123.0') + sources = { + (b[0].source_metadata.additional_details or {}).get( + 'leaderboard_data_source' + ) + for b in bundles + } + assert sources == {'TIGER-Lab', 'Self-Reported'} + for log, _, _ in bundles: + assert _prompt_styles(log) == {None} + + +def test_zero_shot_name_marker_sets_prompt_style(): + row = { + 'Models': 'Athene-V2-Chat (0-shot)', + 'Data Source': 'TIGER-Lab', + 'Model Size(B)': '72', + 'Overall': '0.7311', + 'Math': '0.8', + } + + [(log, _, _)] = adapter.make_logs([row], retrieved_timestamp='123.0') + + assert len(log.evaluation_results) == 2 + for result in log.evaluation_results: + assert result.source_data.additional_details['prompt_style'] == '0-shot' + assert ( + result.metric_config.additional_details['prompt_style'] == '0-shot' + ) + + +def test_descriptions_make_no_shot_claim(): + bundles = adapter.make_logs(sample_rows(), retrieved_timestamp='123.0') + descriptions = [ + result.metric_config.evaluation_description + for log, _, _ in bundles + for result in log.evaluation_results + ] + assert descriptions + for description in descriptions: + assert '5-shot' not in description + assert 'chain-of-thought' not in description + assert '5-shot' not in (adapter.__doc__ or '') + + +def test_blank_data_source_is_recorded_as_unknown(): + row = { + 'Models': 'GPT-4o (2024-05-13)', + 'Data Source': '', + 'Model Size(B)': 'unk', + 'Overall': '0.7255', + } + + [(log, _, _)] = adapter.make_logs([row], retrieved_timestamp='123.0') + + assert ( + log.source_metadata.additional_details['leaderboard_data_source'] + == 'unknown' + ) + assert 'leaderboard_data_source' not in ( + log.model_info.additional_details or {} + ) + + +def test_blank_data_source_keeps_evaluation_id_shape(): + row = { + 'Models': 'GPT-4o (2024-05-13)', + 'Data Source': '', + 'Model Size(B)': 'unk', + 'Overall': '0.7255', + } + + [(log, _, _)] = adapter.make_logs([row], retrieved_timestamp='123.0') + + assert log.evaluation_id == ( + 'mmlu-pro/openai_gpt-4o-2024-05-13/unknown/123.0' + ) + + +def test_evaluation_id_keeps_data_source_segment(): + [(log, _, _)] = adapter.make_logs( + [sample_rows()[0]], retrieved_timestamp='123.0' + ) + assert log.evaluation_id == ( + 'mmlu-pro/openai_gpt-4o-2024-05-13/tiger-lab/123.0' + ) diff --git a/tests/test_vals_ai_adapter.py b/tests/test_vals_ai_adapter.py index d9425ece9..9433b665c 100644 --- a/tests/test_vals_ai_adapter.py +++ b/tests/test_vals_ai_adapter.py @@ -1,16 +1,45 @@ from __future__ import annotations +import gzip +import json from pathlib import Path +import pytest + from every_eval_ever.adapters.vals_ai import adapter from every_eval_ever.eval_types import EvaluationLog from every_eval_ever.validate import validate_file -FIXTURE_PATH = ( - Path(__file__).parent / 'data' / 'vals_ai' / 'finance_agent_payload.json' +DATA_DIR = Path(__file__).parent / 'data' / 'vals_ai' +FIXTURE_PATH = DATA_DIR / 'finance_agent_payload.json' +SAGE_SENTENCE = ( + 'All models are evaluated with temperature 1, and produce at most 30K ' + 'tokens (the length of a short book - more than enough to adequately ' + 'grade student work).' ) +def page_benchmark(fixture: str, route: str) -> dict: + return adapter.normalize_benchmark_page( + (DATA_DIR / fixture).read_text(encoding='utf-8'), + f'https://www.vals.ai/benchmarks/{route}', + ) + + +def convert_pages(*benchmarks: dict): + return adapter.convert_logs( + {'benchmarks': list(benchmarks)}, + retrieved_timestamp='1234567890.0', + ) + + +def logs_by_raw_id(result) -> dict: + return { + bundle.log.model_info.additional_details['vals_model_id']: bundle.log + for bundle in result.records + } + + def sample_payload() -> dict: return { 'source_url': 'https://www.vals.ai/benchmarks', @@ -484,3 +513,406 @@ def test_export_paths_validate(tmp_path: Path): assert path.parent.parent.parent == output_dir report = validate_file(path) assert report.valid, report.errors + + +def test_sage_methodology_fills_null_row_values_and_names_the_source(): + result = convert_pages(page_benchmark('sage_page.html', 'sage')) + + assert not result.failures + logs = logs_by_raw_id(result) + assert set(logs) == { + 'anthropic/claude-sonnet-4-6', + 'openai/gpt-5.5', + 'openai/gpt-5.6-sol', + } + + full = logs['anthropic/claude-sonnet-4-6'].evaluation_results[0] + assert full.generation_config.generation_args.temperature == 1.0 + assert full.generation_config.generation_args.max_tokens == 30000 + assert full.generation_config.additional_details['temperature_source'] == ( + 'row' + ) + assert full.generation_config.additional_details['max_tokens_source'] == ( + 'row' + ) + + null_cap = logs['openai/gpt-5.5'].evaluation_results[0] + assert null_cap.score_details.details.get('max_output_tokens') is None + assert null_cap.generation_config.generation_args.max_tokens == 30000 + assert null_cap.generation_config.additional_details == { + 'reasoning_effort': 'xhigh', + 'temperature_source': 'row', + 'max_tokens_source': 'page_methodology', + } + + null_temperature = logs['openai/gpt-5.6-sol'].evaluation_results[0] + args = null_temperature.generation_config.generation_args + assert args.temperature == 1.0 + assert args.max_tokens == 30000 + details = null_temperature.generation_config.additional_details + assert details['temperature_source'] == 'page_methodology' + assert details['max_tokens_source'] == 'row' + + for log in logs.values(): + EvaluationLog.model_validate(log.model_dump()) + config = log.evaluation_results[0].generation_config + assert config.generation_args.eval_limits is None + + +def test_sage_methodology_is_linked_not_stored(): + benchmark = page_benchmark('sage_page.html', 'sage') + text = benchmark['methodology_text'] + + assert text.startswith( + 'The models are given student work samples and corresponding rubrics' + ) + assert text.endswith(SAGE_SENTENCE) + assert 'Parse and understand the rubric criteria Analyze' in text + assert 'Methodology' not in text + assert 'Trimmed fixture' not in text + assert 'Not methodology' not in text + + log = convert_pages(benchmark).records[0].log + source_details = log.source_metadata.additional_details + assert source_details['vals_methodology_url'] == ( + 'https://www.vals.ai/benchmarks/sage#methodology' + ) + assert 'vals_methodology_text' not in source_details + assert 'vals_methodology_sha256' not in source_details + assert source_details['benchmark_version'] == '1' + assert source_details['vals_runner'] == 'custom' + assert source_details['vals_mode'] == 'one-shot' + assert source_details['vals_family'] == 'sage' + assert source_details['vals_archived'] == 'false' + + +def test_missing_methodology_statement_fails_the_run(): + payload = sample_payload() + payload['benchmarks'].append( + page_benchmark('sage_page_statement_removed.html', 'sage') + ) + + with pytest.raises(adapter.ValsMethodologyStatementMissing) as excinfo: + adapter.convert_logs(payload, retrieved_timestamp='1234567890.0') + + message = str(excinfo.value) + assert "'sage'" in message + assert SAGE_SENTENCE in message + assert 'https://www.vals.ai/benchmarks/sage' in message + + +def test_statement_slug_replayed_without_methodology_fails_the_run(): + benchmark = page_benchmark('sage_page.html', 'sage') + del benchmark['methodology_text'] + + with pytest.raises(adapter.ValsMethodologyStatementMissing): + convert_pages(benchmark) + + +def test_mmmu_methodology_fills_missing_temperature_and_cap(): + benchmark = page_benchmark('mmmu_page.html', 'mmmu') + text = benchmark['methodology_text'] + assert 'All models were ran with a temperature of 0.' in text + benchmark['tasks']['overall']['openai/o1-2024-12-17'][ + 'max_output_tokens' + ] = None + + logs = logs_by_raw_id(convert_pages(benchmark)) + + gemini = logs['google/gemini-2.0-flash-001'].evaluation_results[0] + assert gemini.generation_config.generation_args.temperature == 0.0 + assert gemini.generation_config.generation_args.max_tokens == 8192 + assert gemini.generation_config.additional_details == { + 'temperature_source': 'page_methodology', + 'max_tokens_source': 'row', + } + + o1 = logs['openai/o1-2024-12-17'].evaluation_results[0] + assert o1.generation_config.generation_args.temperature == 0.0 + assert o1.generation_config.generation_args.max_tokens == 8192 + assert ( + o1.generation_config.additional_details['max_tokens_source'] + == 'page_methodology' + ) + + +@pytest.mark.parametrize('slug', sorted(adapter.PAGE_STATEMENTS)) +def test_page_statements_are_present_in_saved_source_pages(slug): + page_html = gzip.decompress( + (DATA_DIR / 'saved_pages' / f'{slug}.html.gz').read_bytes() + ).decode('utf-8') + text = adapter.extract_methodology_text(page_html) + assert text is not None + for statement in adapter.PAGE_STATEMENTS[slug]: + assert statement.sentence in text + + +def test_mmmu_missing_cap_sentence_fails_even_with_temperature_statement(): + benchmark = page_benchmark('mmmu_page.html', 'mmmu') + benchmark['methodology_text'] = 'All models were ran with a temperature of 0.' + with pytest.raises(adapter.ValsMethodologyStatementMissing, match='8192'): + convert_pages(benchmark) + + +def test_loader_fetches_full_rows_instead_of_graph_projections(monkeypatch): + page_html = ''' + + + + + ''' + view = { + 'metadata': {'slug': 'example'}, + 'tasks': { + 'overall': {'model': {'accuracy': 50, 'temperature': 0.7}}, + 'subtask': {'model': {'accuracy': 60, 'temperature': 0.7}}, + }, + } + responses = { + 'https://example.test/benchmarks': 'x', + 'https://example.test/benchmarks/example': page_html, + 'https://example.test/_astro/full-table.json': json.dumps(view), + } + calls = [] + + def fetch(url): + calls.append(url) + return responses[url] + + monkeypatch.setattr(adapter, 'fetch_text', fetch) + payload = adapter.extract_collection(base_url='https://example.test') + assert not payload['source_failures'] + assert payload['benchmarks'][0]['tasks'] == view['tasks'] + assert calls == list(responses) + + +def test_failed_loader_is_reported_without_using_partial_graph_data(monkeypatch): + def fetch(url): + if url.endswith('/benchmarks'): + return 'x' + if url.endswith('/example'): + return '''''' + raise RuntimeError('table unavailable') + + monkeypatch.setattr(adapter, 'fetch_text', fetch) + payload = adapter.extract_collection(base_url='https://example.test') + assert payload['benchmarks'] == [] + assert len(payload['source_failures']) == 1 + assert 'table unavailable' in payload['source_failures'][0]['reason'] + + +def test_loader_payload_without_table_fields_is_not_an_empty_benchmark(): + with pytest.raises(ValueError, match='lacks metadata/tasks'): + adapter.normalize_benchmark_page( + '', + 'https://example.test/benchmarks/example', + benchmark_view={'error': 'unavailable'}, + ) + + +def test_reasoning_is_typed_only_from_a_real_bool(): + def reasoning_of(row: dict): + config = adapter.make_generation_config(row) + if config is None or config.generation_args is None: + return None + return config.generation_args.reasoning + + assert reasoning_of({'reasoning': True}) is True + assert reasoning_of({'reasoning': False}) is False + assert reasoning_of({'reasoning': None, 'reasoning_effort': 'high'}) is None + assert reasoning_of({'reasoning': 'true', 'temperature': 1}) is None + + config = adapter.make_generation_config({'reasoning': True}) + assert config.additional_details is None + + +def test_harness_lands_in_agentic_eval_config(): + config = adapter.make_generation_config( + {'harness': 'OpenHands', 'temperature': 1} + ) + + args = config.generation_args + assert args.agentic_eval_config.additional_details == { + 'vals_harness': 'OpenHands' + } + assert args.eval_limits is None + + +def test_web_search_keeps_embedded_slug_and_page_harness_label(): + result = convert_pages(page_benchmark('web_search_page.html', 'web_search')) + + assert not result.failures + logs = logs_by_raw_id(result) + native = logs['anthropic/claude-fable-5'] + exa = logs['anthropic/claude-fable-5-exa'] + + assert native.evaluation_id.startswith('vals-ai/web_search_backends/') + details = native.source_metadata.additional_details + assert details['benchmark_slug'] == 'web_search_backends' + assert details['leaderboard_page_url'] == ( + 'https://www.vals.ai/benchmarks/web_search' + ) + assert native.evaluation_results[0].evaluation_name == ( + 'vals_ai.web_search_backends.overall' + ) + for log, tool in ((native, 'Native'), (exa, 'Exa')): + args = log.evaluation_results[0].generation_config.generation_args + assert args.agentic_eval_config.additional_details == { + 'vals_search_tool': tool + } + assert args.eval_limits is None + assert ( + log.evaluation_results[0].score_details.details['harness'] == tool + ) + + +def test_row_outcomes_are_json_encoded_in_score_details(): + payload = sample_payload() + row = payload['benchmarks'][0]['tasks']['overall']['openai/gpt-5.4'] + row['token_totals'] = {'input_tokens': 10, 'output_tokens': 2} + row['usage'] = {'trials': 3} + row['task_results'] = {'task-1': {'passed': True}} + row['tie_breaker_score'] = 53.5 + + bundles = adapter.make_logs(payload, retrieved_timestamp='1234567890.0') + finance = next( + bundle.log + for bundle in bundles + if bundle.log.model_info.id == 'openai/gpt-5.4' + ) + overall = next( + result + for result in finance.evaluation_results + if result.evaluation_name == 'vals_ai.finance_agent.overall' + ) + + details = overall.score_details.details + assert json.loads(details['token_totals']) == { + 'input_tokens': 10, + 'output_tokens': 2, + } + assert json.loads(details['usage']) == {'trials': 3} + assert json.loads(details['task_results']) == {'task-1': {'passed': True}} + assert details['tie_breaker_score'] == '53.5' + assert overall.generation_config.generation_args.eval_limits is None + + +def test_one_part_model_id_without_provider_fails_the_row(): + payload = sample_payload() + payload['benchmarks'][0]['tasks']['overall']['mystery-model'] = { + 'accuracy': 50.0, + } + + result = adapter.convert_logs(payload, retrieved_timestamp='1234567890.0') + + assert result.records + assert len(result.failures) == 1 + assert result.failures[0].source_ref == 'finance_agent/mystery-model' + assert 'must be known' in result.failures[0].reason + + +def _assert_one_route_kept(result, kept: str, skipped: str) -> None: + assert len(result.records) == 2 + for bundle in result.records: + details = bundle.log.source_metadata.additional_details + assert details['leaderboard_page_url'] == kept + assert len(bundle.log.evaluation_results) == 1 + failure = result.failures[0] + assert failure.source_ref == skipped + assert kept in failure.reason + assert "'srebench'" in failure.reason + assert failure.source_record == { + 'benchmark_slug': 'srebench', + 'source_url': skipped, + 'first_source_url': kept, + } + assert [row.source_ref for row in result.failures[1:]] == [ + f'{skipped}/overall/openai/gpt-6-astra', + f'{skipped}/overall/openai/gpt-5.6-sol', + ] + + +def test_routes_sharing_a_slug_keep_the_route_named_after_it(): + srebench = page_benchmark('srebench_page.html', 'srebench') + reverse_eng = page_benchmark('reverse_eng_page.html', 'reverse_eng') + assert srebench['metadata']['slug'] == 'srebench' + assert reverse_eng['metadata']['slug'] == 'srebench' + + for order in ((srebench, reverse_eng), (reverse_eng, srebench)): + _assert_one_route_kept( + convert_pages(*order), + kept='https://www.vals.ai/benchmarks/srebench', + skipped='https://www.vals.ai/benchmarks/reverse_eng', + ) + + +def test_routes_sharing_a_slug_fall_back_to_alphabetical_first(): + reverse_eng = page_benchmark('reverse_eng_page.html', 'reverse_eng') + sre_mirror = page_benchmark('srebench_page.html', 'sre_mirror') + + for order in ((sre_mirror, reverse_eng), (reverse_eng, sre_mirror)): + _assert_one_route_kept( + convert_pages(*order), + kept='https://www.vals.ai/benchmarks/reverse_eng', + skipped='https://www.vals.ai/benchmarks/sre_mirror', + ) + + +def test_row_failure_records_do_not_carry_methodology_text(): + benchmark = page_benchmark('sage_page.html', 'sage') + benchmark['tasks']['calculus']['mystery-model'] = {'accuracy': 50.0} + + result = convert_pages(benchmark) + + assert len(result.failures) == 1 + [record] = result.failures[0].source_record + assert record['benchmark_slug'] == 'sage' + assert record['page_details']['vals_methodology_url'] == ( + 'https://www.vals.ai/benchmarks/sage#methodology' + ) + assert benchmark['methodology_text'] not in json.dumps( + result.failures[0].model_dump() + ) + + +def test_present_but_unusable_row_values_are_not_page_filled(): + benchmark = page_benchmark('sage_page.html', 'sage') + row = benchmark['tasks']['calculus']['openai/gpt-5.5'] + row['max_output_tokens'] = 0 + row['temperature'] = 'abc' + + logs = logs_by_raw_id(convert_pages(benchmark)) + + result = logs['openai/gpt-5.5'].evaluation_results[0] + assert result.score_details.details['max_output_tokens'] == '0' + assert result.score_details.details['temperature'] == 'abc' + config = result.generation_config + assert config.generation_args is None + assert 'temperature_source' not in config.additional_details + assert 'max_tokens_source' not in config.additional_details + + +def test_skipped_benchmark_failures_add_up_to_source_rows(): + srebench = page_benchmark('srebench_page.html', 'srebench') + reverse_eng = page_benchmark('reverse_eng_page.html', 'reverse_eng') + finance = sample_payload()['benchmarks'][0] + benchmarks = [srebench, reverse_eng, finance] + source_rows = sum( + len(rows) + for benchmark in benchmarks + for rows in benchmark['tasks'].values() + ) + + result = adapter.iter_vals_metrics_result({'benchmarks': benchmarks}) + + assert [failure.source_ref for failure in result.failures] == [ + 'https://www.vals.ai/benchmarks/reverse_eng', + 'https://www.vals.ai/benchmarks/reverse_eng/overall/openai/gpt-6-astra', + 'https://www.vals.ai/benchmarks/reverse_eng/overall/openai/gpt-5.6-sol', + ] + assert result.total_records == source_rows + 1 + assert result.total_records == len(result.records) + len(result.failures)