From b3d668ca2752186c465c7c55c72d836ec0f5a648 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 01/16] hfopenllm_v2: read the leaderboard table and the harness dumps at pinned revisions The Space API returns HTTP 500. The leaderboard table now comes from the open-llm-leaderboard/contents parquet at a pinned revision (--contents-parquet for offline runs; --source-api keeps the old path). Run settings come from the open-llm-leaderboard/results dumps at a pinned revision (--results-dir for offline runs), matched per (model, precision). A field is written only when every subtask and every dump for that pair report the same value; otherwise it is left null and counted in the report. Fields written: output_type and num_fewshot on all six groups; temperature, max_tokens and do_sample on IFEval and MATH Lvl 5; samples_number, plus dataset_size when a run used a limit; hf_split; harness_dataset_path when the dump's dataset differs from hf_repo (IFEval, MATH); git_hash and transformers_version in eval_library.additional_details. evaluation_id now includes the precision. Before, 78 float16/bfloat16 pairs had the same id. Co-Authored-By: Claude Fable 5.1 --- .../adapters/hfopenllm_v2/adapter.py | 410 ++++++++++++- .../adapters/hfopenllm_v2/contents.py | 75 +++ .../adapters/hfopenllm_v2/dump_index.py | 345 +++++++++++ .../data/train-00000-of-00001.parquet | Bin 0 -> 17636 bytes .../results_2025-01-16T02-47-58.653318.json | 502 ++++++++++++++++ .../results_2025-02-13T18-27-04.338360.json | 502 ++++++++++++++++ .../results_2025-03-09T04-35-49.828129.json | 502 ++++++++++++++++ .../results_2025-02-13T18-27-04.338360.json | 502 ++++++++++++++++ tests/test_hfopenllm_v2_adapter.py | 539 +++++++++++++++++- 9 files changed, 3352 insertions(+), 25 deletions(-) create mode 100644 every_eval_ever/adapters/hfopenllm_v2/contents.py create mode 100644 every_eval_ever/adapters/hfopenllm_v2/dump_index.py create mode 100644 tests/data/hfopenllm_v2/contents/data/train-00000-of-00001.parquet create mode 100644 tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-01-16T02-47-58.653318.json create mode 100644 tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-02-13T18-27-04.338360.json create mode 100644 tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-03-09T04-35-49.828129.json create mode 100644 tests/data/hfopenllm_v2/results/databricks/dbrx-base/results_2025-02-13T18-27-04.338360.json diff --git a/every_eval_ever/adapters/hfopenllm_v2/adapter.py b/every_eval_ever/adapters/hfopenllm_v2/adapter.py index 223769451..a1422fdaa 100644 --- a/every_eval_ever/adapters/hfopenllm_v2/adapter.py +++ b/every_eval_ever/adapters/hfopenllm_v2/adapter.py @@ -1,25 +1,53 @@ """ Script to convert HuggingFace Open LLM Leaderboard v2 data to the EvalEval schema format. -Data source: -- HF Open LLM Leaderboard v2 API: https://open-llm-leaderboard-open-llm-leaderboard.hf.space/api/leaderboard/formatted +Data sources, both pinned Hugging Face datasets: +- ``open-llm-leaderboard/contents`` at ``CONTENTS_REVISION``: the leaderboard + table (scores, one row per model and precision). Reading it needs pyarrow. +- ``open-llm-leaderboard/results`` at ``RESULTS_REVISION``: the harness result + dumps (harness settings, sample counts, split) + +``--source-api`` reads the table from the leaderboard Space API instead, which +is unpinned and no longer served. Usage: uv run python -m every_eval_ever.adapters.hfopenllm_v2.adapter uv run python -m every_eval_ever.adapters.hfopenllm_v2.adapter \ - --output-dir /tmp/smoke/data/hfopenllm_v2 + --output-dir /tmp/smoke/data/hfopenllm_v2 \ + --contents-parquet /data/train-00000-of-00001.parquet \ + --results-dir """ import argparse +import json +import re import time +from collections import Counter from pathlib import Path from typing import Any, Dict, List +from every_eval_ever.adapters.hfopenllm_v2.contents import ( + CONTENTS_REPO, + CONTENTS_REVISION, + download_contents, + load_contents, +) +from every_eval_ever.adapters.hfopenllm_v2.dump_index import ( + RESULTS_PATTERN, + RESULTS_REPO, + RESULTS_REVISION, + DumpIndex, + HarnessGroup, + ModelEvidence, + build_dump_index, +) from every_eval_ever.eval_types import ( EvalLibrary, EvaluationLog, EvaluationResult, EvaluatorRelationship, + GenerationArgs, + GenerationConfig, MetricConfig, ScoreDetails, ScoreType, @@ -34,6 +62,7 @@ fetch_json, make_model_info, make_source_metadata, + raw_capture, save_evaluation_logs, save_failure_report, ) @@ -98,23 +127,102 @@ } +# eval_key -> (metric_id, metric_name, metric_kind, metric_unit) +METRIC_MAPPING = { + 'ifeval': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'bbh': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'math': ('exact_match', 'Exact Match', 'exact_match', 'proportion'), + 'gpqa': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'musr': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), + 'mmlu_pro': ('accuracy', 'Accuracy', 'accuracy', 'proportion'), +} + +HARNESS_FORK = 'https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess' + + +def _slug(name: str) -> str: + return '_'.join( + ''.join(c if c.isalnum() else ' ' for c in name.lower()).split() + ) + + +def _source_data( + base: SourceDataHf, group: HarnessGroup | None +) -> SourceDataHf: + """``base`` with the split and counts the dumps agree on. + + The dataset the harness loaded is recorded as ``harness_dataset_path`` + where it differs from ``base.hf_repo``. + """ + if group is None: + return base + update: Dict[str, Any] = { + 'hf_split': group.hf_split, + 'samples_number': group.samples_number, + } + details = {} + if group.dataset_path is not None and group.dataset_path != base.hf_repo: + details['harness_dataset_path'] = group.dataset_path + if group.dataset_size is not None: + details['dataset_size'] = str(group.dataset_size) + if details: + update['additional_details'] = details + return base.model_copy(update=update) + + +def _generation_config(group: HarnessGroup | None) -> GenerationConfig | None: + """The harness settings the dumps agree on, if any.""" + if group is None: + return None + details = { + name: value if isinstance(value, str) else json.dumps(value) + for name, value in ( + ('output_type', group.output_type), + ('num_fewshot', group.num_fewshot), + ('do_sample', group.do_sample), + ) + if value is not None + } + args = None + if group.temperature is not None or group.max_tokens is not None: + args = GenerationArgs( + temperature=group.temperature, max_tokens=group.max_tokens + ) + if args is None and not details: + return None + return GenerationConfig( + generation_args=args, additional_details=details or None + ) + + def convert_model( model_data: Dict[str, Any], retrieved_timestamp: str, *, source_ref: str | None = None, failures: list[SourceRecordFailure] | None = None, + evidence: ModelEvidence | None = None, ) -> EvaluationLog: """Convert one model, optionally retaining unusable metric provenance. The strict public behavior is unchanged when ``failures`` is omitted: any unusable metric rejects the model. Batch conversion supplies a failure list so valid metrics from the same model can still be published. + ``evidence`` is the dump evidence for this model and precision; without + it the harness fields are left out. """ model_id = model_data['model']['name'] if '/' not in model_id: raise ValueError(f"Expected 'org/model' format, got: {model_id}") developer, model_name = model_id.split('/', 1) + precision = model_data['model'].get('precision') + if not isinstance(precision, str) or not precision.strip(): + raise ValueError(f'precision is missing; got {precision!r}') + evaluation_id = ( + f'hfopenllm_v2/{developer}_{model_name}/{precision}/' + f'{retrieved_timestamp}' + ) + groups = evidence.groups if evidence is not None else {} # Build evaluation results eval_results: List[EvaluationResult] = [] @@ -129,26 +237,38 @@ def convert_model( display_name, f'Accuracy on {display_name}' ) source_data = SOURCE_DATA_MAPPING.get(eval_key) - if source_data is None: + if source_data is None or eval_key not in METRIC_MAPPING: raise ValueError( f"unknown evaluation key; add '{eval_key}' to " - 'SOURCE_DATA_MAPPING' + 'SOURCE_DATA_MAPPING and METRIC_MAPPING' ) + metric_id, metric_name, metric_kind, metric_unit = METRIC_MAPPING[ + eval_key + ] + group = groups.get(eval_key) eval_results.append( EvaluationResult( + evaluation_result_id=( + f'{evaluation_id}#{_slug(display_name)}#{metric_id}' + ), evaluation_name=display_name, - source_data=source_data, + source_data=_source_data(source_data, group), metric_config=MetricConfig( evaluation_description=description, lower_is_better=False, score_type=ScoreType.continuous, min_score=0.0, max_score=1.0, + metric_id=metric_id, + metric_name=metric_name, + metric_kind=metric_kind, + metric_unit=metric_unit, ), score_details=ScoreDetails( score=round(float(eval_data['value']), 4), ), + generation_config=_generation_config(group), ) ) except Exception as exc: @@ -193,10 +313,14 @@ def convert_model( additional_details=additional_details if additional_details else None, ) - # Build evaluation ID - evaluation_id = ( - f'hfopenllm_v2/{developer}_{model_name}/{retrieved_timestamp}' - ) + library_details = {'fork': HARNESS_FORK} + if evidence is not None: + if evidence.git_hash is not None: + library_details['git_hash'] = evidence.git_hash + if evidence.transformers_version is not None: + library_details['transformers_version'] = ( + evidence.transformers_version + ) return EvaluationLog( schema_version=SCHEMA_VERSION, @@ -210,20 +334,31 @@ def convert_model( eval_library=EvalLibrary( name='lm-evaluation-harness', version='0.4.0', - additional_details={ - 'fork': 'https://github.com/huggingface/lm-evaluation-harness/tree/adding_all_changess' - }, + additional_details=library_details, ), model_info=model_info, evaluation_results=eval_results, ) +def _lookup( + dump_index: DumpIndex | None, model_data: Dict[str, Any] +) -> ModelEvidence | None: + if dump_index is None: + return None + model = model_data.get('model') or {} + return dump_index.lookup(model.get('name'), model.get('precision')) + + def convert_models( models_data: List[Dict[str, Any]], retrieved_timestamp: str | None = None, + dump_index: DumpIndex | None = None, ) -> SourceConversionResult[EvaluationLogOutput]: - """Convert all usable models and preserve every rejected source row.""" + """Convert all usable models and preserve every rejected source row. + + Each row is joined to ``dump_index`` on its exact (model, precision). + """ timestamp = retrieved_timestamp or str(time.time()) outputs = [] failures: list[SourceRecordFailure] = [] @@ -242,6 +377,7 @@ def convert_models( timestamp, source_ref=source_ref, failures=failures, + evidence=_lookup(dump_index, model_data), ) outputs.append( EvaluationLogOutput( @@ -273,11 +409,61 @@ def convert_models( ) +def harness_gaps( + models_data: List[Dict[str, Any]], dump_index: DumpIndex | None +) -> Dict[str, Any]: + """Every harness field left null, with the reason, per model row.""" + unfilled = [] + for index, model_data in enumerate(models_data): + model = model_data.get('model') or {} + evidence = _lookup(dump_index, model_data) + for eval_key in model_data.get('evaluations') or {}: + if evidence is None: + gaps = {'all': 'no_dump'} + elif eval_key not in evidence.groups: + gaps = {'all': 'task_group_not_run'} + else: + gaps = evidence.groups[eval_key].gaps + for field_name, reason in sorted(gaps.items()): + unfilled.append( + { + 'source_ref': f'model row {index}', + 'model': model.get('name'), + 'precision': model.get('precision'), + 'evaluation_key': eval_key, + 'field': field_name, + 'reason': reason, + } + ) + counts = Counter( + f'{entry["evaluation_key"]}.{entry["field"]}: {entry["reason"]}' + for entry in unfilled + ) + return { + 'results_dataset': RESULTS_REPO, + 'results_revision': ( + dump_index.revision if dump_index else RESULTS_REVISION + ), + 'results_dir_unpinned': bool(dump_index and dump_index.unpinned), + 'dumps_read': dump_index.files_read if dump_index else 0, + 'dumps_rejected': dump_index.rejected if dump_index else [], + 'unfilled_counts': dict(sorted(counts.items())), + 'unfilled': unfilled, + } + + def process_models( - models_data: List[Dict[str, Any]], output_dir: str | Path = OUTPUT_DIR + models_data: List[Dict[str, Any]], + output_dir: str | Path = OUTPUT_DIR, + dump_index: DumpIndex | None = None, + leaderboard_input: Dict[str, Any] | None = None, ) -> int: - """Save valid models, report rejected rows, and signal incompleteness.""" - result = convert_models(models_data) + """Save valid models, report rejected rows and unfilled harness fields. + + The report is written whenever a row failed or a harness field stayed + null; only failed rows signal incompleteness. + """ + result = convert_models(models_data, dump_index=dump_index) outputs = [ EvaluationLogOutput( eval_log=record.eval_log, @@ -290,11 +476,21 @@ def process_models( paths = save_evaluation_logs(outputs) for path in paths: print(f'Saved: {path}') - if result.failures: + gaps = harness_gaps(models_data, dump_index) + if result.failures or gaps['unfilled'] or gaps['dumps_rejected']: report_path = save_failure_report( result, default_failure_report_path(output_dir), ) + report = json.loads(report_path.read_text(encoding='utf-8')) + report['harness_evidence'] = gaps + if leaderboard_input is not None: + report['leaderboard_input'] = leaderboard_input + report_path.write_text( + json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + + '\n', + encoding='utf-8', + ) print(f'Failure report: {report_path}') result.raise_if_incomplete() return len(paths) @@ -312,17 +508,189 @@ def parse_args(argv: List[str] | None = None) -> argparse.Namespace: default=Path(OUTPUT_DIR), help=f'Datastore collection directory (default: {OUTPUT_DIR}).', ) + parser.add_argument( + '--results-dir', + type=Path, + default=None, + help=( + f'Local Hub snapshot of {RESULTS_REPO} (a directory named after ' + f'its commit sha, which must be {RESULTS_REVISION}). Default: ' + 'download its results_*.json files once from the Hub.' + ), + ) + parser.add_argument( + '--allow-unpinned-results-dir', + action='store_true', + help=( + '(Debug) Accept a --results-dir at another commit, or with no ' + 'commit in its path; the revision is recorded and reported as is.' + ), + ) + table = parser.add_mutually_exclusive_group() + table.add_argument( + '--contents-parquet', + type=Path, + default=None, + help=( + f'Local copy of {CONTENTS_REPO} data/train-00000-of-00001.parquet ' + 'inside a Hub snapshot directory named after its commit sha, ' + f'which must be {CONTENTS_REVISION}. Default: download it once ' + 'from the Hub.' + ), + ) + table.add_argument( + '--source-api', + action='store_true', + help=( + '(Legacy) Read the table from the leaderboard Space API, which ' + 'is unpinned and no longer served, instead of the contents ' + 'dataset.' + ), + ) + parser.add_argument( + '--allow-unpinned-contents-parquet', + action='store_true', + help=( + '(Debug) Accept a --contents-parquet at another commit, or with ' + 'no commit in its path; the revision is recorded and reported ' + 'as is.' + ), + ) return parser.parse_args(argv) +_SHA = re.compile(r'[0-9a-f]{40}') + + +def _snapshot_revision( + snapshot_dir: Path, + *, + option: str, + repo: str, + pinned: str, + allow_unpinned: bool, +) -> tuple[str, bool]: + name = snapshot_dir.name + sha = name if _SHA.fullmatch(name) else None + if sha == pinned: + return sha, False + if not allow_unpinned: + found = sha or f'no commit sha (directory {name!r})' + raise ValueError( + f'{option} is at {found}, but this adapter is pinned to ' + f'{repo}@{pinned}. Pass a snapshot at that revision, or ' + f'--allow-unpinned-{option[2:]}.' + ) + return sha or 'local', True + + +def results_dir_revision( + results_dir: str | Path, *, allow_unpinned: bool = False +) -> tuple[str, bool]: + """The commit a local results snapshot is at, and whether it is unpinned. + + The commit is the directory's own name, as in the Hub cache layout + ``snapshots/``. A directory at another commit, or without one, is + an error unless ``allow_unpinned``; it is then reported as that sha, or + ``'local'``. + """ + return _snapshot_revision( + Path(results_dir).resolve(), + option='--results-dir', + repo=RESULTS_REPO, + pinned=RESULTS_REVISION, + allow_unpinned=allow_unpinned, + ) + + +def contents_parquet_revision( + parquet: str | Path, *, allow_unpinned: bool = False +) -> tuple[str, bool]: + """The commit a local contents parquet is at, and whether it is unpinned. + + The commit is the name of the snapshot directory holding ``data/``, as + in the Hub cache layout ``snapshots//data/``; the file itself + may be a symlink into the cache's blobs. Otherwise as + ``results_dir_revision``. + """ + return _snapshot_revision( + Path(parquet).absolute().parent.parent, + option='--contents-parquet', + repo=CONTENTS_REPO, + pinned=CONTENTS_REVISION, + allow_unpinned=allow_unpinned, + ) + + +def download_results() -> Path: + """Fetch the pinned results snapshot's dumps in one Hub download.""" + from huggingface_hub import snapshot_download + + return Path( + snapshot_download( + repo_id=RESULTS_REPO, + repo_type='dataset', + revision=RESULTS_REVISION, + allow_patterns=RESULTS_PATTERN, + ) + ) + + def main(argv: List[str] | None = None) -> int: args = parse_args(argv) + revision, unpinned = RESULTS_REVISION, False + contents_revision, contents_unpinned = CONTENTS_REVISION, False + try: + if args.results_dir is not None: + revision, unpinned = results_dir_revision( + args.results_dir, + allow_unpinned=args.allow_unpinned_results_dir, + ) + if args.contents_parquet is not None: + contents_revision, contents_unpinned = contents_parquet_revision( + args.contents_parquet, + allow_unpinned=args.allow_unpinned_contents_parquet, + ) + except ValueError as exc: + raise SystemExit(str(exc)) from exc - print(f'Fetching data from {SOURCE_URL}...') - all_models = fetch_json(SOURCE_URL) + if args.source_api: + print(f'Fetching data from {SOURCE_URL}...') + all_models = fetch_json(SOURCE_URL) + leaderboard_input = {'source_url': SOURCE_URL} + else: + raw_capture.record_hf_dataset( + CONTENTS_REPO, + revision=contents_revision, + label='Open LLM Leaderboard v2 table', + ) + parquet = args.contents_parquet or download_contents() + print(f'Reading {parquet} ({contents_revision})...') + all_models = load_contents(parquet) + leaderboard_input = { + 'dataset': CONTENTS_REPO, + 'revision': contents_revision, + 'unpinned': contents_unpinned, + } + + raw_capture.record_hf_dataset( + RESULTS_REPO, + revision=revision, + label='Open LLM Leaderboard v2 harness result dumps', + ) + results_dir = args.results_dir or download_results() + print(f'Indexing harness dumps in {results_dir} ({revision})...') + dump_index = build_dump_index(results_dir) + dump_index.revision, dump_index.unpinned = revision, unpinned + print( + f'Read {dump_index.files_read} dumps for {len(dump_index.entries)} ' + f'(model, precision) pairs; rejected {len(dump_index.rejected)}.' + ) print(f'Processing {len(all_models)} models...') - count = process_models(all_models, args.output_dir) + count = process_models( + all_models, args.output_dir, dump_index, leaderboard_input + ) print(f'Done! Processed {count} models.') return count diff --git a/every_eval_ever/adapters/hfopenllm_v2/contents.py b/every_eval_ever/adapters/hfopenllm_v2/contents.py new file mode 100644 index 000000000..36f2dba75 --- /dev/null +++ b/every_eval_ever/adapters/hfopenllm_v2/contents.py @@ -0,0 +1,75 @@ +"""The Open LLM Leaderboard v2 table, read from ``open-llm-leaderboard/contents``. + +The leaderboard publishes its final table as one parquet file. Each row is +one (model, precision) evaluation. ``load_contents`` turns the rows into the +shape the leaderboard's Space API served, which is what the adapter converts. +""" + +from __future__ import annotations + +import math +from pathlib import Path +from typing import Any + +CONTENTS_REPO = 'open-llm-leaderboard/contents' +CONTENTS_REVISION = '9c09a7cae43334062a82cb164f2ef255013dafa2' +CONTENTS_FILE = 'data/train-00000-of-00001.parquet' + +# eval_key -> (score column on the 0-1 scale, evaluation name) +SCORE_COLUMNS = { + 'ifeval': ('IFEval Raw', 'IFEval'), + 'bbh': ('BBH Raw', 'BBH'), + 'math': ('MATH Lvl 5 Raw', 'MATH Level 5'), + 'gpqa': ('GPQA Raw', 'GPQA'), + 'musr': ('MUSR Raw', 'MUSR'), + 'mmlu_pro': ('MMLU-PRO Raw', 'MMLU-PRO'), +} + + +def _stated(value: Any) -> Any: + if isinstance(value, float) and not math.isfinite(value): + return None + return value + + +def contents_row(row: dict[str, Any]) -> dict[str, Any]: + """One parquet row in the Space API's row shape.""" + model = { + 'name': row.get('fullname'), + 'precision': row.get('Precision'), + 'architecture': row.get('Architecture'), + } + metadata = {} + params = _stated(row.get('#Params (B)')) + if params is not None: + metadata['params_billions'] = params + return { + 'model': model, + 'metadata': metadata, + 'evaluations': { + key: {'name': name, 'value': _stated(row.get(column))} + for key, (column, name) in SCORE_COLUMNS.items() + }, + } + + +def load_contents(parquet_path: str | Path) -> list[dict[str, Any]]: + """Every row of the contents parquet, in the Space API's row shape.""" + import pyarrow.parquet as pq + + table = pq.read_table(parquet_path) + return [contents_row(row) for row in table.to_pylist()] + + +def download_contents() -> Path: + """Fetch the pinned contents parquet in one Hub download.""" + from huggingface_hub import hf_hub_download + + return Path( + hf_hub_download( + repo_id=CONTENTS_REPO, + filename=CONTENTS_FILE, + repo_type='dataset', + revision=CONTENTS_REVISION, + ) + ) diff --git a/every_eval_ever/adapters/hfopenllm_v2/dump_index.py b/every_eval_ever/adapters/hfopenllm_v2/dump_index.py new file mode 100644 index 000000000..8abc76692 --- /dev/null +++ b/every_eval_ever/adapters/hfopenllm_v2/dump_index.py @@ -0,0 +1,345 @@ +"""Index of the Open LLM Leaderboard v2 harness result dumps. + +The leaderboard publishes the lm-evaluation-harness ``results_*.json`` dump of +every run in ``open-llm-leaderboard/results``. The index states, per +``(model, precision)`` and per leaderboard benchmark, the harness settings and +sample counts every dump for that pair agrees on. A field the dumps disagree +on, or do not state, is ``None`` with a reason in ``gaps``. +""" + +from __future__ import annotations + +import json +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +RESULTS_REPO = 'open-llm-leaderboard/results' +RESULTS_REVISION = 'aa81ecc38fdc5708254b833923368970efdf5ef5' +RESULTS_PATTERN = '*/results_*.json' + +DTYPE_PRECISION = { + 'torch.bfloat16': 'bfloat16', + 'torch.float16': 'float16', +} + + +@dataclass(frozen=True) +class TaskGroup: + """The harness tasks that make up one leaderboard benchmark.""" + + prefix: str + subtasks: int + generative: bool = False + suffix: str = '' + + def tasks_of(self, configs: dict[str, Any]) -> dict[str, Any]: + """The group's tasks among a dump's ``configs``.""" + exact = not self.prefix.endswith('_') + return { + task: config + for task, config in configs.items() + if (task == self.prefix if exact else task.startswith(self.prefix)) + and task.endswith(self.suffix) + } + + +# Keyed on the leaderboard API's evaluation keys. +TASK_GROUPS: dict[str, TaskGroup] = { + 'ifeval': TaskGroup('leaderboard_ifeval', 1, generative=True), + 'bbh': TaskGroup('leaderboard_bbh_', 24), + 'math': TaskGroup('leaderboard_math_', 7, generative=True, suffix='_hard'), + 'gpqa': TaskGroup('leaderboard_gpqa_', 3), + 'musr': TaskGroup('leaderboard_musr_', 3), + 'mmlu_pro': TaskGroup('leaderboard_mmlu_pro', 1), +} + +CONFIG_FIELDS = ('output_type', 'num_fewshot') +DECODING_FIELDS = ('temperature', 'max_tokens', 'do_sample') + +_IDENTITY_ARGS = ('pretrained', 'peft', 'delta') + + +@dataclass +class HarnessGroup: + """What every dump of one (model, precision) states for one benchmark. + + The harness settings (output type, few-shot count, decoding) resolve as + one: a conflict on any of them leaves all of them null. + """ + + output_type: str | None = None + num_fewshot: int | None = None + temperature: float | None = None + max_tokens: int | None = None + do_sample: bool | None = None + samples_number: int | None = None + dataset_size: int | None = None + dataset_path: str | None = None + hf_split: str | None = None + gaps: dict[str, str] = field(default_factory=dict) + + +@dataclass +class ModelEvidence: + """Dump evidence for one (model, precision).""" + + dumps: list[str] + groups: dict[str, HarnessGroup] + git_hash: str | None = None + transformers_version: str | None = None + + +@dataclass +class DumpIndex: + """All dumps of a results snapshot, keyed on ``(model, precision)``.""" + + entries: dict[tuple[str, str], ModelEvidence] + files_read: int = 0 + rejected: list[dict[str, Any]] = field(default_factory=list) + revision: str = RESULTS_REVISION + unpinned: bool = False + + def lookup(self, model: str, precision: str | None) -> ModelEvidence | None: + """The evidence for an exact (model, precision), if any dump names it.""" + if precision is None: + return None + return self.entries.get((model, precision)) + + +def _launched_models(model_args: Any) -> dict[str, str]: + if not isinstance(model_args, str): + return {} + launched = {} + for part in model_args.split(','): + key, sep, value = part.partition('=') + if sep and key in _IDENTITY_ARGS: + launched[key] = value + return launched + + +def dump_identity(dump: dict[str, Any]) -> tuple[str | None, str | None]: + """The model a dump is about, or ``None`` and the reason it is rejected. + + Identity is the dump's own ``model_name``, which must equal the weights + the run loaded: ``peft=`` or ``delta=`` when given, else ``pretrained=``. + The directory a dump sits in is never used. + """ + name = dump.get('model_name') + if not isinstance(name, str) or not name: + return None, 'no_model_name' + launched = _launched_models((dump.get('config') or {}).get('model_args')) + if 'pretrained' not in launched: + return None, 'no_pretrained' + if 'peft' in launched and 'delta' in launched: + return None, 'ambiguous_adapter_args' + expected = launched.get( + 'peft', launched.get('delta', launched['pretrained']) + ) + if name != expected: + return None, 'identity_contradicts_model_args' + return name, None + + +def _unanimous(values: list[Any]) -> tuple[bool, Any]: + distinct = {json.dumps(v, sort_keys=True) for v in values} + if len(distinct) != 1: + return False, None + return True, values[0] + + +def _dump_statement( + group: TaskGroup, dump: dict[str, Any] +) -> dict[str, Any] | None: + """One dump's statement for one group, or ``None`` if it did not run it. + + Each field maps to ``(value, None)`` or ``(None, reason)``. + """ + tasks = group.tasks_of(dump.get('configs') or {}) + if not tasks: + return None + if len(tasks) != group.subtasks: + names = CONFIG_FIELDS + (DECODING_FIELDS if group.generative else ()) + return { + name: (None, 'incomplete_task_group') + for name in names + ('counts', 'dataset_path', 'hf_split') + } + + statement: dict[str, tuple[Any, str | None]] = {} + + def config_of(task_config: dict[str, Any]) -> dict[str, Any]: + kwargs = task_config.get('generation_kwargs') or {} + config = { + 'output_type': task_config.get('output_type'), + 'num_fewshot': task_config.get('num_fewshot'), + } + if group.generative: + config['temperature'] = kwargs.get('temperature') + config['max_tokens'] = kwargs.get('max_gen_toks') + config['do_sample'] = kwargs.get('do_sample') + return config + + agreed, config = _unanimous( + [config_of(c) for _, c in sorted(tasks.items())] + ) + for name in CONFIG_FIELDS + (DECODING_FIELDS if group.generative else ()): + if not agreed: + statement[name] = (None, 'subtask_conflict') + elif config[name] is None: + statement[name] = (None, 'not_stated') + else: + statement[name] = (config[name], None) + + n_samples = dump.get('n-samples') + counts: dict[str, tuple[int, int]] = {} + count_gap = None + if not isinstance(n_samples, dict): + count_gap = 'n_samples_absent' + else: + for task in sorted(tasks): + entry = n_samples.get(task) + entry = entry if isinstance(entry, dict) else {} + original, effective = entry.get('original'), entry.get('effective') + if not all( + isinstance(v, int) and not isinstance(v, bool) and v >= 1 + for v in (original, effective) + ): + count_gap = 'sample_count_absent' + break + counts[task] = (original, effective) + if count_gap: + statement['counts'] = (None, count_gap) + else: + statement['counts'] = (counts, None) + + paths = {c.get('dataset_path') for c in tasks.values()} + path = next(iter(paths)) + if len(paths) != 1: + statement['dataset_path'] = (None, 'subtask_conflict') + elif not isinstance(path, str) or not path: + statement['dataset_path'] = (None, 'not_stated') + else: + statement['dataset_path'] = (path, None) + + splits = {c.get('test_split') for c in tasks.values()} + split = next(iter(splits)) + if any(not isinstance(s, str) or not s for s in splits): + statement['hf_split'] = (None, 'not_stated') + elif len(splits) != 1: + statement['hf_split'] = (None, 'subtask_conflict') + else: + statement['hf_split'] = (split, None) + return statement + + +def _resolve_group(statements: list[dict[str, Any]]) -> HarnessGroup: + resolved = HarnessGroup() + names = ( + CONFIG_FIELDS + DECODING_FIELDS + ('counts', 'dataset_path', 'hf_split') + ) + for name in names: + stated = [s[name] for s in statements if name in s] + if not stated: + continue + reasons = {reason for _, reason in stated} + agreed, value = _unanimous([v for v, _ in stated]) + if reasons == {None} and agreed: + gap = None + elif len(reasons) == 1: + gap = reasons.pop() or 'cross_dump_conflict' + else: + gap = 'cross_dump_conflict' + if name == 'counts': + if gap: + resolved.gaps['samples_number'] = gap + continue + original = sum(o for o, _ in value.values()) + effective = sum(e for _, e in value.values()) + resolved.samples_number = effective + if effective != original: + resolved.dataset_size = original + continue + if gap: + resolved.gaps[name] = gap + else: + setattr(resolved, name, value) + config_names = CONFIG_FIELDS + DECODING_FIELDS + conflict = next( + ( + resolved.gaps[name] + for name in config_names + if resolved.gaps.get(name, '').endswith('conflict') + ), + None, + ) + if conflict: + for name in config_names: + if getattr(resolved, name) is not None or name in resolved.gaps: + setattr(resolved, name, None) + resolved.gaps[name] = conflict + return resolved + + +def build_dump_index(results_dir: str | Path) -> DumpIndex: + """Read every ``results_*.json`` under ``results_dir`` into a ``DumpIndex``.""" + root = Path(results_dir) + index = DumpIndex(entries={}) + by_key: dict[tuple[str, str], list[tuple[str, dict[str, Any]]]] = ( + defaultdict(list) + ) + for path in sorted(root.rglob('results_*.json')): + index.files_read += 1 + rel = path.relative_to(root).as_posix() + try: + dump = json.loads(path.read_text(encoding='utf-8')) + except ValueError as exc: + index.rejected.append( + {'path': rel, 'reason': 'unreadable', 'detail': str(exc)} + ) + continue + if not isinstance(dump, dict): + index.rejected.append({'path': rel, 'reason': 'not_an_object'}) + continue + model, reason = dump_identity(dump) + config = dump.get('config') or {} + if model is None: + index.rejected.append( + { + 'path': rel, + 'reason': reason, + 'model_name': dump.get('model_name'), + 'model_args': config.get('model_args'), + } + ) + continue + precision = DTYPE_PRECISION.get(config.get('model_dtype')) + if precision is None: + index.rejected.append( + { + 'path': rel, + 'reason': 'unmapped_model_dtype', + 'model_dtype': config.get('model_dtype'), + } + ) + continue + by_key[(model, precision)].append((rel, dump)) + + for key, dumps in sorted(by_key.items()): + groups = {} + for eval_key, group in TASK_GROUPS.items(): + statements = [ + s + for s in (_dump_statement(group, dump) for _, dump in dumps) + if s is not None + ] + if statements: + groups[eval_key] = _resolve_group(statements) + evidence = ModelEvidence(dumps=[rel for rel, _ in dumps], groups=groups) + for name in ('git_hash', 'transformers_version'): + values = {dump.get(name) for _, dump in dumps} + value = next(iter(values)) + if len(values) == 1 and isinstance(value, str) and value: + setattr(evidence, name, value) + index.entries[key] = evidence + return index diff --git a/tests/data/hfopenllm_v2/contents/data/train-00000-of-00001.parquet b/tests/data/hfopenllm_v2/contents/data/train-00000-of-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4622cc87b4d8547a63ef209260091eab4009ccd6 GIT binary patch literal 17636 zcmeHPdvIgrb-&m4dbMW5!VVI0vLV9k-WYbhdikZryVjLt$@1D-$=1VC$n0F{UOnt9 z$w=~R$xaEwP!ed^luk;3lt-Ahp){pQNomPs2xXF#CYhF35<)f`hA=c?5|d%lDZ^}g z&UbaCE8S~b+9hfK$Q>i!?|kQ+-}%n@9_KqU8)Rp#`Y!#uwEjtYTCdYndnxMlgV#_L z<>7>Uu~6YE#kG^3Vrn@r6e{jwCRZt*Os2&mSFsNFQKgSkA7F0S2kbmo;gY3XYO#Eh zPnOmOl3ZE1+%&cux-v@@+sUekp3+~THy#3{%kTP{dJ<0 z%j6222%4Imq?7xMwYNqu>CnuZY5grQha^Ltm8(GKj3{z>&RZPc$rVArPbdf_ zj^16tz(fj*h2m-fed-H_ZPPQ80>{r3c>&F6o+vx3+{QgkyjtUXeXst(9kMlF*|l-z zOEg7|ai$7a$_SOCeJ7J5S6J*bWlKW(XkWHcSt=hfo3qOqa6%@{rG&$&qFFH)&5diI z&s46gi^9>qR8cIJj+j=sQvbk!m@6y}RD`w403ly{y@@n;{+ybo;oHsh!buKRZ_rZ-#9v&cZQTP;LB$u{p;Fj(;qu#pXFM2j?Hw+9X z4#W7R&UVM~JDtVb*}XaV?;LthDx}PbfCYV;e_52s@v`=%!uf3h1pL zto7B4yFSi8jdj?CbmIp%*QsUHxeooC9rE%2IKA`O!f>X-zps zMv~ScYnrz>Y;3ozMJg2gdn z7m$}9;jN?msNHTI93HgTd46OtWgE6xlB0w6!C||D6RZF|On8SaBb?pBkBr*+G&ekK zPjP9xL$JV`eZ*!>jq-zFB0nN%IHJE*9Akg0BW-dW$vd2SkL)YL+gNSV`!nG8NR zBcz(DhNSeD>i1$wNRfQhNYn5G!sIt$-)WR`!9BpD=B28(|++!QJyz=(_za4-3uD^W!iEq6z{@>PrJaOR7{my58{>16GuY5$qRWgTB zC-b1I)YK+dsVDo--hQb6iNSN@zj^AfKfd;pH^*NbH@^G%ollHM$n%5-w9IML$-D@y zrfv(pvFr4nkInj>FFk*7S9gBSdG_t(@r}I6`6ziz8tY8vH0oqtgjQ3xh5ls6nfv&U zog4r46W8Vb>z;Gt553TRB>&*K@vlFH&kGu(mN|?%nFpcO)NP?JzwM=8UHflGo&RxL z-#KYU;Mo@jsp$d--S2j(_C#ji+|@eRurc z&*Sq?Pc&>Er`}U3Sv(>0&j>b`|soO$d{dg(++5C^jfA-C>(EoJ*XnZHk?-#mX z87C9`sUK^MTJACGWEmW_nz}9YPtj@CtDW<>g;=k0{ta2reNJghv1!b3xyPuJWe{3T z-4^#0gHXWHe99Kuw~IoUwi6OMtrYwHVuTNCO7D3<;?4F6JYP9VbJfut)hwu z_``tT9K~U(zlw@qdJ%-709&Yw<^M@*md7AEib-;euHej{72BNt#loJO0g>t2|oT^;FMWJ7* z8UZ%bZM;J33ZfB_-uDg+QO52v`0zWP!^I2dHT*6iU=!Nl1|) zD5+iH^`!6(rLwt-kg6<~8pL;yN|I8AF1;*@Dq{$nshs6x z)*dkYQg#LER2jka+GDr<`pLT_yDynsPMW53DWOoVA$0+B57J-z!Xtzcr7yWA$d$N! z+0^g477{$5T*0nDjfP8#f~i20T1V%d=zjTHT zPnxDzMAJ~cyQfZ+xOBc?h^_Z`VYAvZ6PcS;^k7r!**!Be9T^DDo~ZZgWRf(t?im3J zan*tDJ&{i5QaKnvuvA>hL5Zs@`Fn3<2};epDO@z6hdtm1IYv0S_RK9O^SLrEg@!&7 z5~JGtcf|p6x=dTUVmh>P1cUbeQOVm^M60GiP=(&yc!)L;YE;>MBiD$~8`u z-cK7Ip_xZ?%v(Fr;93E?p%ksw?$p%Yuu~BApX)H(-@$yPhdHy0czKHwvl`?P?LbP~ z4pG1H5yRtqnYS)u9=in9|Di~FaV7KZij99Ll1krOY+bol)CT{56iL-9>o0mW*Kh6k zCX4IA4#T4z%zIZcuYMR8S1XlIHPSuWYj{^JCm>M2=ra7ai}}no%xxxw`rXtP)y6&D z(cp%j>LJyXnh*qo`~+Vv;mIXTd!sQxgW-&E*u)*98gmsBUJ%5j74^zGB=a)-k_OZ zT*usd9pI^}hKQ%8Zi`3t9_XfewG%&r8#c@{Hy;{P)f{{GL9T!)!&rm;RehK(*2I5U z6<3xaw-%RE%$kbF2f%`+Yv9%@8uQYcqT3HkLutVlITX#=EYT%R<)_Wexx)|+b!8I6 zp{8yd4jEICD=Xl>W@TQs0GzrSiEwIa%W&09Tmki%o%yZ}pw!hzgi=#mhLZDq1=hnu z%rk?4r7kxjmYUi!7TzKhIQNV&PYwf)y0(ZoYHG_kO?MOppjbqH2V*sSC z6C#kB+A@$%z8fmQ-f%LvIRQ*v8ALEOwPmm_1XBRL>SEqw0Z3g3L?AV_WuS(8tpf0Q z5Az#00II8h2&ksE3|PB0E5N?vW!{_sFm=5T!PL}k!Kw#K^-4~V@Mt>gYxiK#xX0JD zeQmPk(?PzBYux^cy4PsKYc%t&pZSa*bC1nyhN^tP5Nt?5WT56c0zrA1HvBuyoITEL zOes)mbwai7u?F3>hrz7GkbY*%VJvp>4$5+Dxr7Zlip%SQIxIBc$#v^Hxj2E%P@Ni(8jE0p7EW+UgbM zM$xUEQ7=2UFV&7jwSe8GNIUHHsyCLbFHEu>DY>ZLu1q`pathY2Ok4blLjQ%xwIxHo zGH8_8VzH^eRA^aL{k21*=$2bj)0Yx$NY+z@Mxm{?vHG_fjWU~?c>45d%D9`Iojnmd zB7IqBe7fI^&w3`vvTO+c!{<254zjKc>t)#t>&_6G%hk}oio}Up|4e97YQrbdLi+tI zyUKzb{6;12xC_Y821*(DT_gUyxSYt3=1%1BJt4;DXRWDx^pt<0m=V^eL_Y5=&)bHw z$ylVyGiwp%T_X1T3js?C<=6dXzk4dpPl~IFIsY)c$KtUyYhrH7G3{n^k;y4BZWB*& zF~@RM@1lbD6hAq&l+34c$UASJ9ZLBSi_Jv=n+M$V+-k-z%5=1|eH)@~uxqlMOH9go z*j&m!yUxXi3jQ4GchtuvCbG%NsOT?Pa9s0=7<_Lz8OqdbzA!zmR6bDT=4ZvY9esS^ z{tU+)i;39k1;-;%h?bMSi;HR1hD^$q&0cU!7rnN`{8Tn!i)7{}BkS=6);2#G%_qXH zyoSzv(&xzXp15V3c=`F5KNI$e3yIj!DKZ}~EM7zp+7&nzw`^h8QnC=T1>9S&0l(Fi zP366n)H+)Xh7cr*Eh}Y}7 z*!av^U9+K!m?zb z&f1puD65#t^I}5d%#G(AE6Hv4%CIew=VF1Scx*KTxx)&d7Z>52kaH5{8fs;Ki^ATr z5|80NGn$#heR5(}d#`CVpRq@C$(VR5WfO7#oq(K)eF1K*zx#RNz(LZ5bN%*2=jWxBsE%R1KI>b(6Y6xdZAz_2O&X&l=Z4q_~i7X5G zq6c$QR%GKW>s$2!Z4}#XYzI~$S9Od3#xMQuhN53t_D8>e&}G@Y(5n%n9Zmz!K(j453U$w){nPL3^vzwjpj8^A=)qhR%7uP(jKgf01)fIa+Uan%* zf1AaNxFWPT<(GW!S+3ztg`4o2{f(h_P0cPwA)f~4x=JuLa1Hv|)u!>P{7uFy`ex_h zp6IU5vx&O=Xp_9kUl_}YKya{l+})6`hFk~D()x6>?r=5!6FxFYRP!CTHkXI_h=UG zeqPpx`VVHB_;C|e@)P+G_=$_HW#joZ(eHyAz~pQZ?k+{Rry)CAKzkhv@_08{Uu9%n zi1@tg@tF0L-<@?I54mz!IsrO(XQ3Lu3Ag00=J|!DxyYSvaLKkfGLpl33mCA9!VqFI zKY&!-uP4H?zD>>xv`5lcyK9D63&t-MgsnocUZI!+53z^5N$aJ4_Z_@(-B+~-`r~Mi z4}VJ!u76oyM0@=sRX+PC;ZB9^sZ+Ss#jEql<(ehhFzCwokN_y^0fcFt;Dte4d0sZJ z9}KN4khHNiL513NbHCrOo*^)3`$Lp(TdARz*O$1!UmegKPzkqUjy{v!GkxX*YU%P{YWE#Oau zD9cht(-K#@aan+W`(SBlfGd@Xt0wphdh)l~Hv3`AVGI1JHT(|%zqcRiPc0M`gn!@U Fe*prxCvE@$ literal 0 HcmV?d00001 diff --git a/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-01-16T02-47-58.653318.json b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-01-16T02-47-58.653318.json new file mode 100644 index 000000000..1995549ff --- /dev/null +++ b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-01-16T02-47-58.653318.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": null, + "model_args": "pretrained=Daemontatox/DocumentCogito,revision=23dcfc6bf91d84db1c977b151fd0923270d3e3ef,trust_remote_code=False,dtype=bfloat16,parallelize=False", + "model_dtype": "torch.bfloat16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a781a6b", + "model_name": "Daemontatox/DocumentCogito", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 187, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 146, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 178, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 250, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 198, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 546, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 448, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 541, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 307, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 123, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 132, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 280, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 154, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 193, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 135, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 12032, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 250, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 256, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 250, + "original": 250 + } + }, + "transformers_version": "4.48.0" +} diff --git a/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-02-13T18-27-04.338360.json b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-02-13T18-27-04.338360.json new file mode 100644 index 000000000..1995549ff --- /dev/null +++ b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-02-13T18-27-04.338360.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": null, + "model_args": "pretrained=Daemontatox/DocumentCogito,revision=23dcfc6bf91d84db1c977b151fd0923270d3e3ef,trust_remote_code=False,dtype=bfloat16,parallelize=False", + "model_dtype": "torch.bfloat16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a781a6b", + "model_name": "Daemontatox/DocumentCogito", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 187, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 146, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 178, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 250, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 198, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 546, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 448, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 541, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 307, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 123, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 132, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 280, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 154, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 193, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 135, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 12032, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 250, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 256, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 250, + "original": 250 + } + }, + "transformers_version": "4.48.0" +} diff --git a/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-03-09T04-35-49.828129.json b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-03-09T04-35-49.828129.json new file mode 100644 index 000000000..c5be6dfb3 --- /dev/null +++ b/tests/data/hfopenllm_v2/results/Daemontatox/DocumentCogito/results_2025-03-09T04-35-49.828129.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": null, + "model_args": "pretrained=Daemontatox/DocumentCogito,revision=9bdbfd8f330754c4103822ce180e0e3e3ce0973e,trust_remote_code=False,dtype=float16,parallelize=False", + "model_dtype": "torch.float16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1024, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a781a6b", + "model_name": "Daemontatox/DocumentCogito", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 187, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 146, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 178, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 250, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 250, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 198, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 546, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 448, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 541, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 307, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 123, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 132, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 280, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 154, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 193, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 135, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 12032, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 250, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 256, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 250, + "original": 250 + } + }, + "transformers_version": "4.49.0" +} diff --git a/tests/data/hfopenllm_v2/results/databricks/dbrx-base/results_2025-02-13T18-27-04.338360.json b/tests/data/hfopenllm_v2/results/databricks/dbrx-base/results_2025-02-13T18-27-04.338360.json new file mode 100644 index 000000000..74fd8b78e --- /dev/null +++ b/tests/data/hfopenllm_v2/results/databricks/dbrx-base/results_2025-02-13T18-27-04.338360.json @@ -0,0 +1,502 @@ +{ + "config": { + "limit": 100.0, + "model_args": "pretrained=databricks/dbrx-base,revision=main,dtype=float16,trust_remote_code=False,parallelize=True", + "model_dtype": "torch.float16" + }, + "configs": { + "leaderboard_bbh_boolean_expressions": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_boolean_expressions", + "test_split": "test" + }, + "leaderboard_bbh_causal_judgement": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_causal_judgement", + "test_split": "test" + }, + "leaderboard_bbh_date_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_date_understanding", + "test_split": "test" + }, + "leaderboard_bbh_disambiguation_qa": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_disambiguation_qa", + "test_split": "test" + }, + "leaderboard_bbh_formal_fallacies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_formal_fallacies", + "test_split": "test" + }, + "leaderboard_bbh_geometric_shapes": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_geometric_shapes", + "test_split": "test" + }, + "leaderboard_bbh_hyperbaton": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_hyperbaton", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_logical_deduction_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_movie_recommendation": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_movie_recommendation", + "test_split": "test" + }, + "leaderboard_bbh_navigate": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_navigate", + "test_split": "test" + }, + "leaderboard_bbh_object_counting": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_object_counting", + "test_split": "test" + }, + "leaderboard_bbh_penguins_in_a_table": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_penguins_in_a_table", + "test_split": "test" + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_reasoning_about_colored_objects", + "test_split": "test" + }, + "leaderboard_bbh_ruin_names": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_ruin_names", + "test_split": "test" + }, + "leaderboard_bbh_salient_translation_error_detection": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_salient_translation_error_detection", + "test_split": "test" + }, + "leaderboard_bbh_snarks": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_snarks", + "test_split": "test" + }, + "leaderboard_bbh_sports_understanding": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_sports_understanding", + "test_split": "test" + }, + "leaderboard_bbh_temporal_sequences": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_temporal_sequences", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_five_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_seven_objects", + "test_split": "test" + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_tracking_shuffled_objects_three_objects", + "test_split": "test" + }, + "leaderboard_bbh_web_of_lies": { + "dataset_path": "SaylorTwift/bbh", + "num_fewshot": 3, + "output_type": "multiple_choice", + "task": "leaderboard_bbh_web_of_lies", + "test_split": "test" + }, + "leaderboard_gpqa_diamond": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_diamond" + }, + "leaderboard_gpqa_extended": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_extended" + }, + "leaderboard_gpqa_main": { + "dataset_path": "Idavidrein/gpqa", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_gpqa_main" + }, + "leaderboard_ifeval": { + "dataset_path": "wis-k/instruction-following-eval", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 1280, + "temperature": 0.0, + "until": [] + }, + "num_fewshot": 0, + "output_type": "generate_until", + "task": "leaderboard_ifeval", + "test_split": "train" + }, + "leaderboard_math_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_counting_and_prob_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_counting_and_prob_hard", + "test_split": "test" + }, + "leaderboard_math_geometry_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_geometry_hard", + "test_split": "test" + }, + "leaderboard_math_intermediate_algebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_intermediate_algebra_hard", + "test_split": "test" + }, + "leaderboard_math_num_theory_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_num_theory_hard", + "test_split": "test" + }, + "leaderboard_math_prealgebra_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_prealgebra_hard", + "test_split": "test" + }, + "leaderboard_math_precalculus_hard": { + "dataset_path": "lighteval/MATH-Hard", + "generation_kwargs": { + "do_sample": false, + "max_gen_toks": 512, + "temperature": 0.0, + "until": [ + "Problem:" + ] + }, + "num_fewshot": 4, + "output_type": "generate_until", + "task": "leaderboard_math_precalculus_hard", + "test_split": "test" + }, + "leaderboard_mmlu_pro": { + "dataset_path": "TIGER-Lab/MMLU-Pro", + "num_fewshot": 5, + "output_type": "multiple_choice", + "task": "leaderboard_mmlu_pro", + "test_split": "test" + }, + "leaderboard_musr_murder_mysteries": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_murder_mysteries", + "test_split": "murder_mysteries" + }, + "leaderboard_musr_object_placements": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_object_placements", + "test_split": "object_placements" + }, + "leaderboard_musr_team_allocation": { + "dataset_path": "TAUR-Lab/MuSR", + "num_fewshot": 0, + "output_type": "multiple_choice", + "task": "leaderboard_musr_team_allocation", + "test_split": "team_allocation" + } + }, + "git_hash": "a4d01d5f", + "model_name": "databricks/dbrx-base", + "n-samples": { + "leaderboard_bbh_boolean_expressions": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_causal_judgement": { + "effective": 100, + "original": 187 + }, + "leaderboard_bbh_date_understanding": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_disambiguation_qa": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_formal_fallacies": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_geometric_shapes": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_hyperbaton": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_five_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_seven_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_logical_deduction_three_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_movie_recommendation": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_navigate": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_object_counting": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_penguins_in_a_table": { + "effective": 100, + "original": 146 + }, + "leaderboard_bbh_reasoning_about_colored_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_ruin_names": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_salient_translation_error_detection": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_snarks": { + "effective": 100, + "original": 178 + }, + "leaderboard_bbh_sports_understanding": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_temporal_sequences": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_five_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_seven_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_tracking_shuffled_objects_three_objects": { + "effective": 100, + "original": 250 + }, + "leaderboard_bbh_web_of_lies": { + "effective": 100, + "original": 250 + }, + "leaderboard_gpqa_diamond": { + "effective": 100, + "original": 198 + }, + "leaderboard_gpqa_extended": { + "effective": 100, + "original": 546 + }, + "leaderboard_gpqa_main": { + "effective": 100, + "original": 448 + }, + "leaderboard_ifeval": { + "effective": 100, + "original": 541 + }, + "leaderboard_math_algebra_hard": { + "effective": 100, + "original": 307 + }, + "leaderboard_math_counting_and_prob_hard": { + "effective": 100, + "original": 123 + }, + "leaderboard_math_geometry_hard": { + "effective": 100, + "original": 132 + }, + "leaderboard_math_intermediate_algebra_hard": { + "effective": 100, + "original": 280 + }, + "leaderboard_math_num_theory_hard": { + "effective": 100, + "original": 154 + }, + "leaderboard_math_prealgebra_hard": { + "effective": 100, + "original": 193 + }, + "leaderboard_math_precalculus_hard": { + "effective": 100, + "original": 135 + }, + "leaderboard_mmlu_pro": { + "effective": 100, + "original": 12032 + }, + "leaderboard_musr_murder_mysteries": { + "effective": 100, + "original": 250 + }, + "leaderboard_musr_object_placements": { + "effective": 100, + "original": 256 + }, + "leaderboard_musr_team_allocation": { + "effective": 100, + "original": 250 + } + }, + "transformers_version": "4.40.2" +} diff --git a/tests/test_hfopenllm_v2_adapter.py b/tests/test_hfopenllm_v2_adapter.py index 73e7e2fe5..6c999bdfd 100644 --- a/tests/test_hfopenllm_v2_adapter.py +++ b/tests/test_hfopenllm_v2_adapter.py @@ -1,9 +1,29 @@ import json +import shutil +import subprocess +import sys +from pathlib import Path +import pytest + +from every_eval_ever.adapters.hfopenllm_v2 import adapter from every_eval_ever.adapters.hfopenllm_v2.adapter import ( + HARNESS_FORK, + contents_parquet_revision, convert_model, convert_models, process_models, + results_dir_revision, +) +from every_eval_ever.adapters.hfopenllm_v2.contents import ( + CONTENTS_FILE, + CONTENTS_REVISION, + contents_row, +) +from every_eval_ever.adapters.hfopenllm_v2.dump_index import ( + RESULTS_REVISION, + build_dump_index, + dump_identity, ) from every_eval_ever.helpers.io import SourceRecordsError from every_eval_ever.validate import validate_file @@ -33,7 +53,9 @@ def test_single_model_conversion_remains_strict(): except ValueError as exc: assert "Evaluation 'bbh' could not be converted" in str(exc) else: - raise AssertionError('expected strict conversion to reject missing score') + raise AssertionError( + 'expected strict conversion to reject missing score' + ) def test_batch_keeps_valid_metrics_and_records_missing_metric(): @@ -82,10 +104,519 @@ def test_process_models_writes_valid_output_and_external_failure_report( assert len(outputs) == 1 assert validate_file(outputs[0]).valid - report_path = ( - tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' - ) + report_path = tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' report = json.loads(report_path.read_text()) assert report['converted_records'] == 1 assert len(report['failed_records']) == 1 assert not report_path.is_relative_to(output_dir) + + +RESULTS = Path(__file__).parent / 'data' / 'hfopenllm_v2' / 'results' +COGITO = 'Daemontatox/DocumentCogito' +DBRX = 'databricks/dbrx-base' + + +def _row(name, precision, keys=('ifeval', 'bbh', 'math', 'gpqa', 'musr')): + names = { + 'ifeval': 'IFEval', + 'bbh': 'BBH', + 'math': 'MATH Level 5', + 'gpqa': 'GPQA', + 'musr': 'MUSR', + 'mmlu_pro': 'MMLU-PRO', + } + return { + 'model': {'name': name, 'precision': precision}, + 'metadata': {}, + 'evaluations': { + key: {'name': names[key], 'value': 0.5} + for key in keys + ('mmlu_pro',) + }, + } + + +def _results(log): + return {r.evaluation_name: r for r in log.evaluation_results} + + +def _copy_results(tmp_path, *paths): + root = tmp_path / 'results' + for rel in paths: + target = root / rel + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(RESULTS / rel, target) + return root + + +def _convert(rows, results_dir=RESULTS): + index = build_dump_index(results_dir) + result = convert_models(rows, retrieved_timestamp='1234', dump_index=index) + assert not result.failures + return [record.eval_log for record in result.records] + + +def test_record_matches_datastore_shape_without_dumps(): + (log,) = _convert([_row('example/model', 'bfloat16')], RESULTS / 'none') + + assert log.evaluation_id == 'hfopenllm_v2/example_model/bfloat16/1234' + ifeval = _results(log)['IFEval'] + assert ifeval.evaluation_result_id == ( + 'hfopenllm_v2/example_model/bfloat16/1234#ifeval#accuracy' + ) + assert _results(log)['MATH Level 5'].evaluation_result_id.endswith( + '#math_level_5#exact_match' + ) + metric = _results(log)['MMLU-PRO'].metric_config + assert ( + metric.metric_id, + metric.metric_name, + metric.metric_kind, + metric.metric_unit, + ) == ('accuracy', 'Accuracy', 'accuracy', 'proportion') + assert ifeval.generation_config is None + assert ifeval.source_data.hf_repo == 'google/IFEval' + assert ifeval.source_data.hf_split is None + assert ifeval.source_data.samples_number is None + assert log.eval_library.additional_details == { + 'fork': HARNESS_FORK, + } + + +def test_two_precisions_get_distinct_ids_and_their_own_dumps(): + bf16, fp16 = _convert([_row(COGITO, 'bfloat16'), _row(COGITO, 'float16')]) + + assert bf16.evaluation_id != fp16.evaluation_id + assert bf16.evaluation_id.endswith('/bfloat16/1234') + assert fp16.evaluation_id.endswith('/float16/1234') + assert bf16.eval_library.additional_details['transformers_version'] == ( + '4.48.0' + ) + assert fp16.eval_library.additional_details['transformers_version'] == ( + '4.49.0' + ) + assert bf16.eval_library.additional_details['git_hash'] == 'a781a6b' + + +def test_generative_groups_get_decoding_and_multiple_choice_do_not(): + (log,) = _convert([_row(COGITO, 'bfloat16')]) + results = _results(log) + + ifeval = results['IFEval'].generation_config + assert ifeval.generation_args.temperature == 0.0 + assert ifeval.generation_args.max_tokens == 1280 + assert ifeval.additional_details == { + 'output_type': 'generate_until', + 'num_fewshot': '0', + 'do_sample': 'false', + } + math = results['MATH Level 5'].generation_config + assert math.generation_args.max_tokens == 1024 + assert math.additional_details['num_fewshot'] == '4' + + bbh = results['BBH'].generation_config + assert bbh.generation_args is None + assert bbh.additional_details == { + 'output_type': 'multiple_choice', + 'num_fewshot': '3', + } + for name in ('GPQA', 'MUSR', 'MMLU-PRO'): + config = results[name].generation_config + assert config.generation_args is None + assert config.additional_details['output_type'] == 'multiple_choice' + + +def test_dataset_split_and_counts_come_from_the_dumps(): + (log,) = _convert([_row(COGITO, 'bfloat16')]) + results = _results(log) + + ifeval = results['IFEval'].source_data + assert ifeval.hf_repo == 'google/IFEval' + assert ifeval.additional_details == { + 'harness_dataset_path': 'wis-k/instruction-following-eval' + } + assert ifeval.hf_split == 'train' + assert ifeval.samples_number == 541 + math = results['MATH Level 5'].source_data + assert math.hf_repo == 'DigitalLearningGmbH/MATH-lighteval' + assert math.additional_details == { + 'harness_dataset_path': 'lighteval/MATH-Hard' + } + assert math.hf_split == 'test' + assert math.samples_number == 1324 + assert results['BBH'].source_data.samples_number == 5761 + assert results['MMLU-PRO'].source_data.samples_number == 12032 + + gpqa = results['GPQA'].source_data + assert gpqa.samples_number == 1192 + assert gpqa.hf_split is None + musr = results['MUSR'].source_data + assert musr.hf_repo == 'TAUR-Lab/MuSR' + assert musr.additional_details is None + assert results['BBH'].source_data.additional_details is None + assert musr.samples_number == 756 + assert musr.hf_split is None + + +def test_limited_run_counts_scored_items_and_reports_dataset_size(): + (log,) = _convert([_row(DBRX, 'float16')]) + results = _results(log) + + math = results['MATH Level 5'] + assert math.generation_config.generation_args.max_tokens == 512 + assert math.source_data.samples_number == 700 + assert math.source_data.additional_details == { + 'harness_dataset_path': 'lighteval/MATH-Hard', + 'dataset_size': '1324', + } + bbh = results['BBH'].source_data + assert bbh.samples_number == 2400 + assert bbh.additional_details == {'dataset_size': '5761'} + + +def test_cross_dump_disagreement_leaves_field_null(tmp_path): + first, second = ( + f'{COGITO}/results_2025-01-16T02-47-58.653318.json', + f'{COGITO}/results_2025-02-13T18-27-04.338360.json', + ) + root = _copy_results(tmp_path, first, second) + path = root / second + dump = json.loads(path.read_text()) + for task, config in dump['configs'].items(): + if task.startswith('leaderboard_math_'): + config['generation_kwargs']['max_gen_toks'] = 512 + dump['n-samples']['leaderboard_ifeval']['effective'] = 100 + path.write_text(json.dumps(dump)) + + (log,) = _convert([_row(COGITO, 'bfloat16')], root) + results = _results(log) + + assert results['MATH Level 5'].generation_config is None + assert results['MATH Level 5'].source_data.samples_number == 1324 + assert results['IFEval'].source_data.samples_number is None + assert results['IFEval'].generation_config.generation_args.max_tokens == ( + 1280 + ) + index = build_dump_index(root) + gaps = index.lookup(COGITO, 'bfloat16').groups + assert gaps['math'].gaps['max_tokens'] == 'cross_dump_conflict' + assert gaps['ifeval'].gaps['samples_number'] == 'cross_dump_conflict' + + +def test_subtask_disagreement_within_a_dump_leaves_group_null(tmp_path): + rel = f'{DBRX}/results_2025-02-13T18-27-04.338360.json' + root = _copy_results(tmp_path, rel) + dump = json.loads((root / rel).read_text()) + dump['configs']['leaderboard_math_algebra_hard']['generation_kwargs'][ + 'max_gen_toks' + ] = 1024 + (root / rel).write_text(json.dumps(dump)) + + (log,) = _convert([_row(DBRX, 'float16')], root) + + math = _results(log)['MATH Level 5'] + assert math.generation_config is None + assert math.source_data.samples_number == 700 + + +def test_dump_contradicting_its_model_args_is_rejected(tmp_path): + rel = f'{DBRX}/results_2025-02-13T18-27-04.338360.json' + root = _copy_results(tmp_path, rel) + dump = json.loads((root / rel).read_text()) + dump['config']['model_args'] = dump['config']['model_args'].replace( + 'pretrained=databricks/dbrx-base', 'pretrained=databricks/dbrx-instruct' + ) + (root / rel).write_text(json.dumps(dump)) + + index = build_dump_index(root) + + assert index.entries == {} + assert index.rejected[0]['reason'] == 'identity_contradicts_model_args' + (log,) = _convert([_row(DBRX, 'float16')], root) + assert _results(log)['BBH'].generation_config is None + + +def test_adapter_dump_is_named_after_the_adapter(): + assert dump_identity( + { + 'model_name': 'org/adapter', + 'config': {'model_args': 'pretrained=org/base,peft=org/adapter'}, + } + ) == ('org/adapter', None) + assert dump_identity( + { + 'model_name': 'org/base', + 'config': {'model_args': 'pretrained=org/base,peft=org/adapter'}, + } + ) == (None, 'identity_contradicts_model_args') + + +def test_precision_without_a_dump_converts_with_nulls(): + (log,) = _convert([_row(DBRX, 'bfloat16')]) + + for result in log.evaluation_results: + assert result.generation_config is None + assert result.source_data.samples_number is None + assert result.source_data.hf_split is None + assert set(log.eval_library.additional_details) == {'fork'} + + +def test_process_models_output_validates_and_reports_gaps(tmp_path): + output_dir = tmp_path / 'data' / 'hfopenllm_v2' + rows = [ + _row(COGITO, 'bfloat16'), + _row(COGITO, 'float16'), + _row(DBRX, 'float16'), + _row(DBRX, 'bfloat16'), + ] + + count = process_models(rows, output_dir, build_dump_index(RESULTS)) + + assert count == 4 + outputs = sorted(output_dir.glob('*/*/*.json')) + assert len(outputs) == 4 + for path in outputs: + report = validate_file(path) + assert report.valid, report + cli = subprocess.run( + [sys.executable, '-m', 'every_eval_ever', 'validate', str(path)], + capture_output=True, + text=True, + ) + assert cli.returncode == 0, cli.stdout + cli.stderr + + report = json.loads( + ( + tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' + ).read_text() + ) + assert report['failed_record_count'] == 0 + counts = report['harness_evidence']['unfilled_counts'] + assert counts['gpqa.hf_split: not_stated'] == 3 + assert counts['musr.hf_split: subtask_conflict'] == 3 + assert counts['ifeval.all: no_dump'] == 1 + assert report['harness_evidence']['dumps_read'] == 4 + + +OTHER_SHA = 'b' * 40 + + +def _snapshot(tmp_path, name): + root = tmp_path / 'snapshots' / name + shutil.copytree(RESULTS, root) + return root + + +def _run_main(monkeypatch, tmp_path, results_dir, *extra): + monkeypatch.setattr( + adapter, 'fetch_json', lambda url: [_row(DBRX, 'float16')] + ) + output_dir = tmp_path / 'data' / 'hfopenllm_v2' + adapter.main( + [ + '--output-dir', + str(output_dir), + '--source-api', + '--results-dir', + str(results_dir), + *extra, + ] + ) + report = tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' + return json.loads(report.read_text())['harness_evidence'] + + +def test_results_dir_at_the_pinned_revision_is_accepted(monkeypatch, tmp_path): + root = _snapshot(tmp_path, RESULTS_REVISION) + + assert results_dir_revision(root) == (RESULTS_REVISION, False) + evidence = _run_main(monkeypatch, tmp_path, root) + assert evidence['results_revision'] == RESULTS_REVISION + assert evidence['results_dir_unpinned'] is False + + +def test_results_dir_at_another_revision_fails_naming_both( + monkeypatch, tmp_path +): + root = _snapshot(tmp_path, OTHER_SHA) + monkeypatch.setattr( + adapter, + 'fetch_json', + lambda url: pytest.fail('fetched before checking the results dir'), + ) + + with pytest.raises(SystemExit) as exc: + adapter.main(['--results-dir', str(root)]) + assert OTHER_SHA in str(exc.value) + assert RESULTS_REVISION in str(exc.value) + with pytest.raises(ValueError, match='no commit sha'): + results_dir_revision(RESULTS) + + +def test_unpinned_results_dir_is_recorded_and_reported(monkeypatch, tmp_path): + recorded = [] + monkeypatch.setattr( + adapter.raw_capture, + 'record_hf_dataset', + lambda repo, **kwargs: recorded.append(kwargs['revision']), + ) + root = _snapshot(tmp_path, OTHER_SHA) + + evidence = _run_main( + monkeypatch, tmp_path, root, '--allow-unpinned-results-dir' + ) + + assert recorded == [OTHER_SHA] + assert evidence['results_revision'] == OTHER_SHA + assert evidence['results_dir_unpinned'] is True + assert results_dir_revision(RESULTS, allow_unpinned=True) == ( + 'local', + True, + ) + + +def test_stated_model_details_are_kept_with_the_library_version(): + source = _row('example/model', 'float16') + source['model']['architecture'] = 'LlamaForCausalLM' + source['metadata']['params_billions'] = 4.65 + + log = convert_model(source, '1234') + + details = log.model_info.additional_details + assert details['precision'] == 'float16' + assert details['architecture'] == 'LlamaForCausalLM' + assert details['params_billions'] == '4.65' + assert log.eval_library.name == 'lm-evaluation-harness' + assert log.eval_library.version == '0.4.0' + + +def test_split_does_not_depend_on_the_harness_dataset_path(tmp_path): + rel = f'{DBRX}/results_2025-02-13T18-27-04.338360.json' + root = _copy_results(tmp_path, rel) + dump = json.loads((root / rel).read_text()) + dump['configs']['leaderboard_math_algebra_hard']['dataset_path'] = 'x/y' + (root / rel).write_text(json.dumps(dump)) + + (log,) = _convert([_row(DBRX, 'float16')], root) + + math = _results(log)['MATH Level 5'].source_data + assert math.hf_repo == 'DigitalLearningGmbH/MATH-lighteval' + assert math.hf_split == 'test' + assert math.additional_details == {'dataset_size': '1324'} + + +CONTENTS = ( + Path(__file__).parent / 'data' / 'hfopenllm_v2' / 'contents' / CONTENTS_FILE +) + + +def test_contents_row_has_the_api_row_shape(): + row = contents_row( + { + 'fullname': 'org/model', + 'Precision': 'float16', + 'Architecture': 'LlamaForCausalLM', + '#Params (B)': float('nan'), + 'IFEval Raw': 0.5, + 'IFEval': 50.0, + 'MATH Lvl 5 Raw': None, + } + ) + + assert row['model'] == { + 'name': 'org/model', + 'precision': 'float16', + 'architecture': 'LlamaForCausalLM', + } + assert row['metadata'] == {} + assert row['evaluations']['ifeval'] == {'name': 'IFEval', 'value': 0.5} + assert row['evaluations']['math'] == { + 'name': 'MATH Level 5', + 'value': None, + } + assert set(row['evaluations']) == { + 'ifeval', + 'bbh', + 'math', + 'gpqa', + 'musr', + 'mmlu_pro', + } + + +def test_contents_parquet_rows_convert(): + pytest.importorskip('pyarrow') + from every_eval_ever.adapters.hfopenllm_v2.contents import load_contents + + rows = load_contents(CONTENTS) + + by_key = {(r['model']['name'], r['model']['precision']): r for r in rows} + assert set(by_key) == { + (COGITO, 'bfloat16'), + (COGITO, 'float16'), + (DBRX, 'float16'), + ('gpt2', 'float16'), + } + dbrx = by_key[(DBRX, 'float16')] + assert dbrx['model']['architecture'] == 'Unknown' + assert dbrx['metadata']['params_billions'] == 0.0 + assert 0 <= dbrx['evaluations']['math']['value'] <= 1 + + +def test_main_reads_pinned_contents_and_results(tmp_path): + pytest.importorskip('pyarrow') + contents = tmp_path / 'contents' / CONTENTS_REVISION / CONTENTS_FILE + contents.parent.mkdir(parents=True) + shutil.copy(CONTENTS, contents) + results = _snapshot(tmp_path, RESULTS_REVISION) + output_dir = tmp_path / 'data' / 'hfopenllm_v2' + + with pytest.raises(SourceRecordsError, match='gpt2'): + adapter.main( + [ + '--output-dir', + str(output_dir), + '--contents-parquet', + str(contents), + '--results-dir', + str(results), + ] + ) + + outputs = sorted(output_dir.glob('*/*/*.json')) + assert len(outputs) == 3 + for path in outputs: + assert validate_file(path).valid + report = json.loads( + ( + tmp_path / 'adapter_reports' / 'hfopenllm_v2_failures.json' + ).read_text() + ) + assert report['leaderboard_input'] == { + 'dataset': 'open-llm-leaderboard/contents', + 'revision': CONTENTS_REVISION, + 'unpinned': False, + } + assert report['failed_record_count'] == 1 + + +def test_contents_parquet_pinning(tmp_path): + other = tmp_path / OTHER_SHA / CONTENTS_FILE + + assert contents_parquet_revision( + tmp_path / CONTENTS_REVISION / CONTENTS_FILE + ) == (CONTENTS_REVISION, False) + with pytest.raises(ValueError) as exc: + contents_parquet_revision(other) + assert OTHER_SHA in str(exc.value) + assert CONTENTS_REVISION in str(exc.value) + assert '--allow-unpinned-contents-parquet' in str(exc.value) + assert contents_parquet_revision(other, allow_unpinned=True) == ( + OTHER_SHA, + True, + ) + assert contents_parquet_revision(CONTENTS, allow_unpinned=True) == ( + 'local', + True, + ) + with pytest.raises(SystemExit): + adapter.main(['--contents-parquet', str(CONTENTS)]) From da950d8ae9fcfd97368d6098a28a654383569e40 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 02/16] converters: read generation args from the merged config; record items scored Inspect: generation args are read from model_generate_config.merge(plan.config), which is the config inspect_ai applies to each call. The converter read only model_generate_config, so settings defined on the task were null. If a plan step's params name temperature, max_tokens, top_p or top_k, or pass a model, that field is left null and listed in overridden_by_step. terminal_solver and generation_args_source are recorded. reasoning is set only when reasoning_effort is present (it was False when absent). samples_number is the number of sample ids, with dataset_size recorded when it differs. hf_split is read from task_args.split when it is train, val or test. eval_library.version is the inspect_ai version; other packages go to additional_details. lm_eval: output_type is always written. generation_args is None for loglikelihood and multiple_choice tasks. samples_number is the effective n-samples count, with dataset_size when the original count differs. HELM: adapter_method is written as given. hf_split is set from the split already in details. Duplicate sample_ids are removed. Schema: the samples_number description is now "Number of items the reported score covers". eval_types.py is regenerated; only that description changes. Co-Authored-By: Claude Fable 5.1 --- every_eval_ever/converters/helm/adapter.py | 27 +- every_eval_ever/converters/inspect/adapter.py | 161 ++++++++-- every_eval_ever/converters/lm_eval/adapter.py | 53 +++- every_eval_ever/eval_types.py | 3 +- every_eval_ever/schemas/eval.schema.json | 2 +- tests/test_helm_adapter.py | 62 ++++ tests/test_inspect_adapter.py | 281 +++++++++++++++++- tests/test_lm_eval_adapter.py | 56 +++- 8 files changed, 605 insertions(+), 40 deletions(-) diff --git a/every_eval_ever/converters/helm/adapter.py b/every_eval_ever/converters/helm/adapter.py index 9c400aaea..d35f0fadf 100644 --- a/every_eval_ever/converters/helm/adapter.py +++ b/every_eval_ever/converters/helm/adapter.py @@ -90,6 +90,14 @@ require_uuid4, ) +_SCHEMA_SPLITS = { + 'train': 'train', + 'val': 'val', + 'valid': 'val', + 'validation': 'val', + 'test': 'test', +} + def _instance_counts_by_result_id(per_instance_stats: List) -> Dict[str, int]: """How many distinct instances each aggregate result was computed over. @@ -533,7 +541,11 @@ def _transform_single( samples_number=len( set(state.instance.id for state in request_states) ), - sample_ids=[str(state.instance.id) for state in request_states], + sample_ids=list( + dict.fromkeys( + str(state.instance.id) for state in request_states + ) + ), additional_details={ 'scenario_name': str(run_spec.scenario_spec.class_name), 'scenario_args': json.dumps(run_spec.scenario_spec.args) @@ -617,11 +629,17 @@ def _transform_single( or None ) + hf_split = _SCHEMA_SPLITS.get(_stat_name_part(split) or '') + evaluation_results.append( EvaluationResult( evaluation_result_id=evaluation_result_id, evaluation_name=source_data.dataset_name, - source_data=source_data, + source_data=( + source_data.model_copy(update={'hf_split': hf_split}) + if hf_split + else source_data + ), evaluation_timestamp=evaluation_timestamp, metric_config=metric_config, score_details=ScoreDetails( @@ -664,6 +682,11 @@ def _transform_single( 'num_completions': str( request_states[0].request.num_completions ), + **( + {'adapter_method': str(adapter_spec.method)} + if adapter_spec.method + else {} + ), }, ), ) diff --git a/every_eval_ever/converters/inspect/adapter.py b/every_eval_ever/converters/inspect/adapter.py index ac4b6f17f..802b41257 100644 --- a/every_eval_ever/converters/inspect/adapter.py +++ b/every_eval_ever/converters/inspect/adapter.py @@ -117,6 +117,23 @@ def _require_inspect_dependencies() -> None: _STDERR_PREFERENCE = ('analytic', 'bootstrap') _UNCERTAINTY_METRICS = _STDDEV_METRICS | frozenset(_STDERR_METHODS) +# Plan-step parameter names that override a generation field for the calls +# that step makes, keyed by the `GenerationArgs` field they override. +_STEP_OVERRIDE_PARAMS = { + 'temperature': frozenset({'temperature'}), + 'max_tokens': frozenset( + {'max_tokens', 'max_completion_tokens', 'max_output_tokens'} + ), + 'top_p': frozenset({'top_p'}), + 'top_k': frozenset({'top_k'}), +} +_SCHEMA_SPLITS = { + 'train': 'train', + 'val': 'val', + 'validation': 'val', + 'test': 'test', +} + class InspectAIAdapter(BaseEvaluationAdapter): """ @@ -369,13 +386,26 @@ def _looks_like_local_path(cls, location: str | None) -> bool: return not cls._HF_REPO_RE.match(location) def _extract_source_data( - self, dataset: EvalDataset, task_name: str + self, + dataset: EvalDataset, + task_name: str, + task_args: Dict[str, Any] | None = None, ) -> SourceDataHf | SourceDataPrivate: sample_ids = ( [str(sid) for sid in dataset.sample_ids] if dataset.sample_ids is not None else None ) + samples_number = ( + len(sample_ids) if sample_ids is not None else dataset.samples + ) + dataset_size = ( + str(dataset.samples) + if dataset.samples is not None and dataset.samples != samples_number + else None + ) + split = (task_args or {}).get('split') + split = _SCHEMA_SPLITS.get(split) if isinstance(split, str) else None if self._looks_like_local_path(dataset.location): # dataset.location is not a valid HF repo identifier: it may @@ -403,10 +433,14 @@ def _extract_source_data( ) if dataset.name: additional_details['inspect_dataset_name'] = dataset.name - if dataset.samples is not None: - additional_details['samples_number'] = str(dataset.samples) + if samples_number is not None: + additional_details['samples_number'] = str(samples_number) + if dataset_size is not None: + additional_details['dataset_size'] = dataset_size if sample_ids is not None: additional_details['sample_ids'] = ','.join(sample_ids) + if split is not None: + additional_details['split'] = split return SourceDataPrivate( source_type='other', dataset_name=dataset_name, @@ -421,13 +455,17 @@ def _extract_source_data( if dataset.name else task_name.split('/')[-1] ) - return SourceDataHf( # TODO add hf_split + additional_details = {'shuffled': str(dataset.shuffled)} + if dataset_size is not None: + additional_details['dataset_size'] = dataset_size + return SourceDataHf( source_type='hf_dataset', dataset_name=dataset_name, hf_repo=dataset.location, - samples_number=dataset.samples, + hf_split=split, + samples_number=samples_number, sample_ids=sample_ids, - additional_details={'shuffled': str(dataset.shuffled)}, + additional_details=additional_details, ) def _safe_get(self, obj: Any, field: str): @@ -477,10 +515,49 @@ def _extract_prompt_template(self, plan: InspectEvalPlan) -> str | None: return None + @staticmethod + def _param_names(params: Any) -> set[str]: + """Every dict key in a step's parameters, at any depth, outside tool definitions.""" + if isinstance(params, dict): + if params.get('type') == 'tool': + return set() + names = {str(key) for key in params} + for key, value in params.items(): + if key != 'tools': + names |= InspectAIAdapter._param_names(value) + return names + if isinstance(params, (list, tuple)): + names = set() + for item in params: + names |= InspectAIAdapter._param_names(item) + return names + return set() + + def _overridden_generation_fields( + self, inspect_plan: InspectEvalPlan + ) -> set[str]: + """Generation fields a plan step may set per call, out of the plan config's reach.""" + steps = list(inspect_plan.steps) + named: set[str] = set() + for step in steps: + named |= self._param_names(step.params) + named |= self._param_names(step.params_passed) + overridden = { + field + for field, params in _STEP_OVERRIDE_PARAMS.items() + if named & params + } + if steps and ( + (steps[-1].params or {}).get('model') is not None + or (steps[-1].params_passed or {}).get('model') is not None + ): + overridden |= set(_STEP_OVERRIDE_PARAMS) + return overridden + def _extract_generation_config( self, spec: EvalSpec, inspect_plan: InspectEvalPlan ) -> GenerationConfig: - eval_config = spec.model_generate_config + eval_config = spec.model_generate_config.merge(inspect_plan.config) eval_generation_config = { gen_config: json.dumps(value) for gen_config, value in vars(eval_config).items() @@ -515,25 +592,55 @@ def _extract_generation_config( ) max_attempts = ( - spec.task_args.get('max_attempts') or eval_config.max_retries + spec.task_args.get('max_attempts') + or spec.model_generate_config.max_retries ) # TODO not sure if max_attempts == max_retries in this case + reasoning_effort = eval_config.reasoning_effort reasoning = ( - True - if eval_config.reasoning_effort - and eval_config.reasoning_effort.lower() != 'none' - else False + reasoning_effort.lower() != 'none' + if isinstance(reasoning_effort, str) + else None ) + overridden = self._overridden_generation_fields(inspect_plan) + filled = { + field: getattr(eval_config, field) + for field in _STEP_OVERRIDE_PARAMS + if field not in overridden + and getattr(eval_config, field) is not None + } + for field in overridden: + eval_generation_config.pop(field, None) + if overridden: + eval_generation_config['overridden_by_step'] = ','.join( + sorted(overridden) + ) + if filled: + from_plan = { + getattr(inspect_plan.config, field) is not None + for field in filled + } + eval_generation_config['generation_args_source'] = ( + 'mixed' + if len(from_plan) > 1 + else 'plan_config' + if True in from_plan + else 'model_config' + ) + + if steps := inspect_plan.steps: + eval_generation_config['terminal_solver'] = steps[-1].solver + available_tools: List[AvailableTool] = self._extract_available_tools( inspect_plan ) generation_args = GenerationArgs( - temperature=eval_config.temperature, - top_p=eval_config.top_p, - top_k=eval_config.top_k, - max_tokens=eval_config.max_tokens, + temperature=filled.get('temperature'), + top_p=filled.get('top_p'), + top_k=filled.get('top_k'), + max_tokens=filled.get('max_tokens'), reasoning=reasoning, prompt_template=self._extract_prompt_template(inspect_plan), agentic_eval_config=AgenticEvalConfig( @@ -552,11 +659,16 @@ def _extract_generation_config( additional_details=additional_details or None, ) - def _extract_library_version(self, packages: Dict[str, str]) -> str: - parts = [ - f'{name}:{version}' for name, version in packages.items() if version - ] - return ','.join(parts) + def _extract_library_version( + self, packages: Dict[str, str] + ) -> tuple[str | None, dict[str, str] | None]: + """The inspect_ai version, and the other packages the log records.""" + others = { + str(name): str(version) + for name, version in packages.items() + if version and name != 'inspect_ai' + } + return packages.get('inspect_ai') or None, others or None def transform_from_directory( self, dir_path: Union[str, Path], metadata_args: Dict[str, Any] = None @@ -680,11 +792,14 @@ def _transform_single( if not evaluation_unix_timestamp: evaluation_unix_timestamp = retrieved_unix_timestamp - library_version = self._extract_library_version(eval_spec.packages) + library_version, library_packages = self._extract_library_version( + eval_spec.packages + ) eval_library = EvalLibrary( name=metadata_args.get('eval_library_name', 'inspect_ai'), version=library_version or metadata_args.get('eval_library_version', 'unknown'), + additional_details=library_packages, ) evaluator_relationship = metadata_args.get( @@ -696,7 +811,7 @@ def _transform_single( ) source_data = self._extract_source_data( - eval_spec.dataset, eval_spec.task + eval_spec.dataset, eval_spec.task, eval_spec.task_args ) model_path = eval_spec.model diff --git a/every_eval_ever/converters/lm_eval/adapter.py b/every_eval_ever/converters/lm_eval/adapter.py index 406fea52f..51c5ca936 100644 --- a/every_eval_ever/converters/lm_eval/adapter.py +++ b/every_eval_ever/converters/lm_eval/adapter.py @@ -163,10 +163,23 @@ def _get_tasks(self, raw_data: Dict[str, Any]) -> List[str]: tasks.append(task_name) return tasks - def _build_source_data(self, task_config: Dict[str, Any], task_name: str): - """Build source_data from task config.""" + def _build_source_data( + self, + task_config: Dict[str, Any], + task_name: str, + n_samples: Dict[str, Any] | None = None, + ): + """Build source_data from task config and the task's `n-samples` entry.""" dataset_path = task_config.get('dataset_path', '') dataset_name = task_config.get('task', task_name) + n_samples = n_samples or {} + original = n_samples.get('original') + original = original if isinstance(original, int) else None + effective = n_samples.get('effective') + samples_number = effective if isinstance(effective, int) else original + additional: Dict[str, str] = {} + if original is not None and original != samples_number: + additional['dataset_size'] = str(original) if ( dataset_path @@ -181,33 +194,51 @@ def _build_source_data(self, task_config: Dict[str, Any], task_name: str): task_config.get('test_split') or task_config.get('validation_split') ), + samples_number=samples_number, + additional_details=additional or None, ) + if samples_number is not None: + additional['samples_number'] = str(samples_number) return SourceDataPrivate( dataset_name=dataset_name, source_type='other', + additional_details=additional or None, ) def _build_generation_config( self, task_config: Dict[str, Any] ) -> Optional[GenerationConfig]: """Build generation config from task config.""" - gen_kwargs = task_config.get('generation_kwargs', {}) - if not gen_kwargs: - return None + gen_kwargs = task_config.get('generation_kwargs') or {} + output_type = task_config.get('output_type') + output_type = output_type if isinstance(output_type, str) else None + scored_by_likelihood = output_type in ( + 'loglikelihood', + 'loglikelihood_rolling', + 'multiple_choice', + ) - args = GenerationArgs( - temperature=gen_kwargs.get('temperature'), - top_p=gen_kwargs.get('top_p'), - top_k=gen_kwargs.get('top_k'), - max_tokens=gen_kwargs.get('max_gen_toks'), + args = ( + GenerationArgs( + temperature=gen_kwargs.get('temperature'), + top_p=gen_kwargs.get('top_p'), + top_k=gen_kwargs.get('top_k'), + max_tokens=gen_kwargs.get('max_gen_toks'), + ) + if gen_kwargs and not scored_by_likelihood + else None ) additional = {} + if output_type is not None: + additional['output_type'] = output_type for k, v in gen_kwargs.items(): if k not in ('temperature', 'top_p', 'top_k', 'max_gen_toks'): additional[k] = json.dumps(v) if not isinstance(v, str) else v if task_config.get('num_fewshot') is not None: additional['num_fewshot'] = str(task_config['num_fewshot']) + if args is None and not additional: + return None return GenerationConfig( generation_args=args, @@ -229,7 +260,7 @@ def _build_evaluation_results( if not isinstance(bootstrap_iters, int): bootstrap_iters = None - source_data = self._build_source_data(task_config, task_name) + source_data = self._build_source_data(task_config, task_name, n_samples) gen_config = self._build_generation_config(task_config) eval_timestamp = raw_data.get('date') if eval_timestamp is not None: diff --git a/every_eval_ever/eval_types.py b/every_eval_ever/eval_types.py index c9e0946d6..f44bcf3eb 100644 --- a/every_eval_ever/eval_types.py +++ b/every_eval_ever/eval_types.py @@ -364,7 +364,8 @@ class SourceDataHf(BaseModel): ) hf_split: str | None = Field(None, description='One of train, val or test.') samples_number: int | None = Field( - None, description='Number of samples in the dataset' + None, + description='Number of items the reported score covers (the evaluated subset when it differs from the dataset size)', ) sample_ids: list[str] | None = Field( None, description='Array of sample ids used for evaluation' diff --git a/every_eval_ever/schemas/eval.schema.json b/every_eval_ever/schemas/eval.schema.json index ed4276611..f4cdf57cb 100644 --- a/every_eval_ever/schemas/eval.schema.json +++ b/every_eval_ever/schemas/eval.schema.json @@ -752,7 +752,7 @@ }, "samples_number": { "type": "integer", - "description": "Number of samples in the dataset" + "description": "Number of items the reported score covers (the evaluated subset when it differs from the dataset size)" }, "sample_ids": { "type": "array", diff --git a/tests/test_helm_adapter.py b/tests/test_helm_adapter.py index 802901fcf..68a5b138d 100644 --- a/tests/test_helm_adapter.py +++ b/tests/test_helm_adapter.py @@ -551,3 +551,65 @@ def test_missing_model_deployment_falls_back_to_model(): assert converted_eval.model_info.id == 'openai/gpt2' assert converted_eval.model_info.developer == 'openai' assert converted_eval.model_info.inference_platform == 'unknown' + + +MMLU_RUN = ( + 'tests/data/helm/' + 'mmlu-subject=philosophy,method=multiple_choice_joint,model=openai_gpt2' +) +NARRATIVE_QA_RUN = 'tests/data/helm/narrative_qa-model=openai_gpt2' +_METADATA_ARGS = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, +} + + +@pytest.mark.parametrize( + ('run', 'method'), + [ + (HELLASWAG_RUN, 'multiple_choice_joint'), + (NARRATIVE_QA_RUN, 'generation'), + ], +) +def test_adapter_method_is_recorded_verbatim(run, method): + converted_eval = _load_eval(HELMAdapter(), run, _METADATA_ARGS) + + assert { + result.generation_config.additional_details['adapter_method'] + for result in converted_eval.evaluation_results + } == {method} + + +def test_hf_split_is_the_split_each_score_was_computed_on(): + """HELM's `valid` split is the schema's `val`.""" + converted_eval = _load_eval(HELMAdapter(), MMLU_RUN, _METADATA_ARGS) + + splits = { + (result.score_details.details['split'], result.source_data.hf_split) + for result in converted_eval.evaluation_results + } + assert splits == {('test', 'test'), ('valid', 'val')} + + +def test_sample_ids_count_each_instance_once(tmp_path): + """A run with two train trials requests every instance twice.""" + import copy + import json + import shutil + + destination = tmp_path / 'run' + shutil.copytree(Path(HELLASWAG_RUN), destination) + state_path = destination / 'scenario_state.json' + scenario_state = json.loads(state_path.read_text()) + second_trial = copy.deepcopy(scenario_state['request_states']) + for state in second_trial: + state['train_trial_index'] = 1 + scenario_state['request_states'] += second_trial + state_path.write_text(json.dumps(scenario_state), encoding='utf-8') + + converted_eval = _load_eval(HELMAdapter(), destination, _METADATA_ARGS) + + source_data = converted_eval.evaluation_results[0].source_data + assert source_data.samples_number == 10 + assert len(source_data.sample_ids) == 10 + assert len(set(source_data.sample_ids)) == 10 diff --git a/tests/test_inspect_adapter.py b/tests/test_inspect_adapter.py index 27cbd418f..ddaa74120 100644 --- a/tests/test_inspect_adapter.py +++ b/tests/test_inspect_adapter.py @@ -6,6 +6,7 @@ ) import contextlib +import json import logging import tempfile from pathlib import Path @@ -29,6 +30,7 @@ SourceDataUrl, SourceMetadata, ) +from tests.converter_cases import CASES TEST_UUID = '123e4567-e89b-42d3-a456-426614174000' OTHER_TEST_UUID = '123e4567-e89b-42d3-a456-426614174001' @@ -472,6 +474,280 @@ def test_humaneval_eval(): assert converted_eval.detailed_evaluation_results is not None +CYSE2_LOG = 'tests/data/inspect/data_cyse2_vuln_exploit_challenges.json' +GAIA_LOG = 'tests/data/inspect/2026-02-07T11-26-57+00-00_gaia_4V8zHbbRKpU5Yv2BMoBcjE.json' +_METADATA_ARGS = { + 'source_organization_name': 'TestOrg', + 'evaluator_relationship': EvaluatorRelationship.first_party, +} + + +def _load_rewritten_eval(tmp_path, fixture, rewrite): + """Convert a copy of an Inspect fixture with `rewrite` applied to its JSON.""" + log = json.loads(Path(fixture).read_text(encoding='utf-8')) + rewrite(log) + path = tmp_path / Path(fixture).name + path.write_text(json.dumps(log), encoding='utf-8') + return _load_eval(InspectAIAdapter(), path, _METADATA_ARGS) + + +def test_generation_args_come_from_the_plan_config(): + """Inspect runs every call under the model config merged with the plan config. + + This log's model config is empty and its task sets temperature and max_tokens, + which only the plan config carries. + """ + case = next(case for case in CASES if case.source == 'inspect') + converted_eval = _load_eval( + InspectAIAdapter(), case.log_path, _METADATA_ARGS + ) + + for result in converted_eval.evaluation_results: + generation_config = result.generation_config + args = generation_config.generation_args + assert args.temperature == 0.75 + assert args.max_tokens == 2048 + assert args.top_p is None + assert args.reasoning is None + details = generation_config.additional_details + assert details['generation_args_source'] == 'plan_config' + assert details['terminal_solver'] == 'generate' + assert details['temperature'] == '0.75' + assert details['max_tokens'] == '2048' + + +def test_max_tokens_is_the_plan_configs_per_call_cap(): + converted_eval = _load_eval( + InspectAIAdapter(), + 'tests/data/inspect/data_with_llm_scoring.json', + _METADATA_ARGS, + ) + + result = converted_eval.evaluation_results[0] + args = result.generation_config.generation_args + assert args.max_tokens == 128000 + assert args.temperature is None + + +def test_generation_args_from_the_model_config_alone_say_so(tmp_path): + def rewrite(log): + log['eval']['model_generate_config'] = {'temperature': 0.3} + log['plan']['config'] = {} + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + assert generation_config.generation_args.temperature == 0.3 + assert generation_config.generation_args.max_tokens is None + assert ( + generation_config.additional_details['generation_args_source'] + == 'model_config' + ) + + +@pytest.mark.parametrize( + ('step', 'temperature', 'max_tokens', 'overridden'), + [ + ( + {'solver': 'generate', 'params': {'temperature': 0.1}}, + None, + 2048, + 'temperature', + ), + ( + { + 'solver': 'generate', + 'params': {'kwargs': {'max_completion_tokens': 16}}, + }, + 0.75, + None, + 'max_tokens', + ), + ( + { + 'solver': 'my_agent', + 'params': {}, + 'params_passed': {'config': {'max_output_tokens': 16}}, + }, + 0.75, + None, + 'max_tokens', + ), + ( + {'solver': 'my_agent', 'params': {'model': 'openai/gpt-4o'}}, + None, + None, + 'max_tokens,temperature,top_k,top_p', + ), + ( + {'solver': 'my_agent', 'params': {'model': None}}, + 0.75, + 2048, + None, + ), + ], + ids=[ + 'step-temperature', + 'nested-max-tokens', + 'passed-only', + 'model', + 'model-default-none', + ], +) +def test_a_step_that_sets_its_own_generation_args_leaves_them_unstated( + tmp_path, step, temperature, max_tokens, overridden +): + """A solver passing its own value overrides the plan config for its calls.""" + + def rewrite(log): + log['plan']['steps'] = [step] + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + details = generation_config.additional_details + assert generation_config.generation_args.temperature == temperature + assert generation_config.generation_args.max_tokens == max_tokens + assert details.get('temperature') == ( + None if temperature is None else '0.75' + ) + assert details.get('max_tokens') == (None if max_tokens is None else '2048') + assert details.get('overridden_by_step') == overridden + assert details['terminal_solver'] == step['solver'] + if temperature is None and max_tokens is None: + assert 'generation_args_source' not in details + + +def test_generation_args_from_both_configs_are_mixed(tmp_path): + def rewrite(log): + log['eval']['model_generate_config'] = {'top_p': 0.9} + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + assert generation_config.generation_args.top_p == 0.9 + assert generation_config.generation_args.temperature == 0.75 + assert ( + generation_config.additional_details['generation_args_source'] + == 'mixed' + ) + + +def test_a_tool_argument_named_like_a_generation_arg_does_not_override( + tmp_path, +): + tool = { + 'type': 'tool', + 'name': 'sampler', + 'params': {'temperature': 0.1, 'max_tokens': 16}, + } + + def rewrite(log): + log['plan']['steps'] = [ + { + 'solver': 'use_tools', + 'params': {'tools': [[tool]], 'append': True}, + 'params_passed': {'tools': [[tool]]}, + }, + {'solver': 'generate', 'params': {}}, + ] + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + generation_config = converted_eval.evaluation_results[0].generation_config + assert generation_config.generation_args.temperature == 0.75 + assert generation_config.generation_args.max_tokens == 2048 + + +@pytest.mark.parametrize( + ('reasoning_effort', 'reasoning'), + [('high', True), ('none', False)], +) +def test_reasoning_is_what_the_log_states( + tmp_path, reasoning_effort, reasoning +): + def rewrite(log): + log['plan']['config']['reasoning_effort'] = reasoning_effort + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + result = converted_eval.evaluation_results[0] + args = result.generation_config.generation_args + assert args.reasoning is reasoning + + +def test_gaia_states_its_split_and_how_many_samples_it_scored(): + converted_eval = _load_eval(InspectAIAdapter(), GAIA_LOG, _METADATA_ARGS) + + result = converted_eval.evaluation_results[0] + args = result.generation_config.generation_args + assert args.temperature == 0.5 + assert args.reasoning is None + assert result.generation_config.additional_details['temperature'] == '0.5' + assert ( + result.generation_config.additional_details['terminal_solver'] + == 'basic_agent_loop' + ) + + details = result.source_data.additional_details + assert details['samples_number'] == '2' + assert details['dataset_size'] == '165' + assert details['split'] == 'val' + + +@pytest.mark.parametrize( + ('split', 'hf_split'), + [('test', 'test'), ('validation', 'val'), ('dev', None)], +) +def test_hf_split_is_taken_only_from_a_schema_split(tmp_path, split, hf_split): + def rewrite(log): + log['eval']['task_args'] = {'split': split} + + converted_eval = _load_rewritten_eval( + tmp_path, 'tests/data/inspect/data_arc_qwen.json', rewrite + ) + + source_data = converted_eval.evaluation_results[0].source_data + assert source_data.hf_split == hf_split + assert source_data.samples_number == 3 + assert source_data.additional_details['dataset_size'] == '2376' + + +def test_a_complete_run_states_no_separate_dataset_size(): + converted_eval = _load_eval( + InspectAIAdapter(), + 'tests/data/inspect/data_with_llm_scoring.json', + _METADATA_ARGS, + ) + + source_data = converted_eval.evaluation_results[0].source_data + assert source_data.samples_number == 1000 + assert source_data.hf_split is None + assert 'dataset_size' not in source_data.additional_details + + +def test_library_version_is_inspect_ai_and_other_packages_are_details(): + converted_eval = _load_eval(InspectAIAdapter(), GAIA_LOG, _METADATA_ARGS) + + assert converted_eval.eval_library.name == 'inspect_ai' + assert converted_eval.eval_library.version == '0.3.171' + assert converted_eval.eval_library.additional_details == { + 'inspect_evals': '0.3.106' + } + + +def test_library_version_without_inspect_ai_package_is_unknown(tmp_path): + def rewrite(log): + log['eval']['packages'] = {'inspect_evals': '0.3.106'} + + converted_eval = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + + assert converted_eval.eval_library.version == 'unknown' + assert converted_eval.eval_library.additional_details == { + 'inspect_evals': '0.3.106' + } + + def test_extract_evaluation_results_one_scorer_with_two_metrics(): adapter = InspectAIAdapter() source_data = SourceDataHf( @@ -802,7 +1078,10 @@ def test_supplemental_eval_details_fill_only_top_level_fields(): assert result.source_data.additional_details['subset'] == '{"name": "full"}' assert result.generation_config is not None - assert result.generation_config.additional_details == {'runner': 'inspect'} + assert result.generation_config.additional_details == { + 'runner': 'inspect', + 'terminal_solver': 'multiple_choice', + } assert result.generation_config.generation_args is not None assert ( result.generation_config.generation_args.agentic_eval_config is not None diff --git a/tests/test_lm_eval_adapter.py b/tests/test_lm_eval_adapter.py index cb94833f1..0d688c6e4 100644 --- a/tests/test_lm_eval_adapter.py +++ b/tests/test_lm_eval_adapter.py @@ -242,6 +242,59 @@ def test_transform_from_file_generation_config(): mode='json', exclude_none=True ) assert gen.additional_details['num_fewshot'] == '0' + assert gen.additional_details['output_type'] == 'generate_until' + + +def test_source_data_states_the_samples_scored(): + adapter = LMEvalAdapter() + logs = adapter.transform_from_file(RESULTS_FILE, _make_metadata_args()) + + for log in logs: + source_data = log.evaluation_results[0].source_data + assert source_data.samples_number == 5000 + assert source_data.additional_details is None + + +def _multiple_choice_task(n_samples): + return { + 'results': {'mc_task': {'acc,none': 0.25, 'acc_stderr,none': 0.01}}, + 'configs': { + 'mc_task': { + 'task': 'mc_task', + 'dataset_path': 'org/mc_dataset', + 'test_split': 'test', + 'output_type': 'multiple_choice', + 'num_fewshot': 5, + } + }, + 'n-samples': {'mc_task': n_samples}, + } + + +def test_multiple_choice_task_states_its_scoring_mode_and_no_generation_args(): + adapter = LMEvalAdapter() + results = adapter._build_evaluation_results( + _multiple_choice_task({'original': 1000, 'effective': 1000}), 'mc_task' + ) + + gen = results[0].generation_config + assert gen.generation_args is None + assert gen.additional_details == { + 'output_type': 'multiple_choice', + 'num_fewshot': '5', + } + + +def test_a_limited_run_states_how_many_samples_it_scored(): + adapter = LMEvalAdapter() + results = adapter._build_evaluation_results( + _multiple_choice_task({'original': 1000, 'effective': 100}), 'mc_task' + ) + + source_data = results[0].source_data + assert source_data.samples_number == 100 + assert source_data.additional_details == {'dataset_size': '1000'} + assert results[0].score_details.uncertainty.num_samples == 100 def test_transform_from_file_eval_timestamp(): @@ -618,7 +671,8 @@ def test_directory_conversion_tracks_each_results_file_parent(tmp_path): second.write_text(json.dumps(second_source), encoding='utf-8') result = adapter.transform_from_directory_result( - tmp_path, {'parent_eval_output_dir': str(tmp_path)} + tmp_path, + _make_metadata_args(parent_eval_output_dir=str(tmp_path)), ) parents = { From a773b65c777c2ef3feaa10141ea5207aa4c2402a Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 03/16] vals_ai: write null generation settings from the page's methodology statement If a model entry has a temperature or max_output_tokens value, it is used. If the entry value is null and the page's methodology section states one value for all models, that value is used: SAGE, MedCode and MedScribe temperature 1 and max_tokens 30000; MMMU temperature 0. temperature_source and max_tokens_source record whether the value came from the row or from the page. vals_methodology_url points at the section. The run fails if the quoted sentence is not found in the page's methodology section. Also written from the entries: reasoning (only when it is a boolean), the harness name in agentic_eval_config.additional_details, and token totals and task results in score_details.details. Two routes that embed the same slug are converted once (reverse_eng now serves the srebench page). Co-Authored-By: Claude Fable 5.1 --- every_eval_ever/adapters/vals_ai/adapter.py | 321 +++++++++++++++- tests/data/vals_ai/mmmu_page.html | 38 ++ tests/data/vals_ai/reverse_eng_page.html | 15 + tests/data/vals_ai/sage_page.html | 23 ++ .../vals_ai/sage_page_statement_removed.html | 23 ++ tests/data/vals_ai/srebench_page.html | 15 + tests/data/vals_ai/web_search_page.html | 55 +++ tests/test_vals_ai_adapter.py | 354 +++++++++++++++++- 8 files changed, 830 insertions(+), 14 deletions(-) create mode 100644 tests/data/vals_ai/mmmu_page.html create mode 100644 tests/data/vals_ai/reverse_eng_page.html create mode 100644 tests/data/vals_ai/sage_page.html create mode 100644 tests/data/vals_ai/sage_page_statement_removed.html create mode 100644 tests/data/vals_ai/srebench_page.html create mode 100644 tests/data/vals_ai/web_search_page.html diff --git a/every_eval_ever/adapters/vals_ai/adapter.py b/every_eval_ever/adapters/vals_ai/adapter.py index c627c381b..c6f76f8cb 100644 --- a/every_eval_ever/adapters/vals_ai/adapter.py +++ b/every_eval_ever/adapters/vals_ai/adapter.py @@ -15,6 +15,7 @@ from urllib.request import Request, urlopen from every_eval_ever.eval_types import ( + AgenticEvalConfig, EvalLibrary, EvaluationLog, EvaluationResult, @@ -61,6 +62,87 @@ 'qwen': 'alibaba', 'togethercomputer': 'together', } +METHODOLOGY_BLOCK_TAGS = frozenset( + { + 'blockquote', + 'br', + 'dd', + 'div', + 'dl', + 'dt', + 'h3', + 'h4', + 'h5', + 'h6', + 'hr', + 'li', + 'ol', + 'p', + 'pre', + 'table', + 'td', + 'th', + 'tr', + 'ul', + } +) +METHODOLOGY_SKIPPED_TAGS = frozenset( + {'astro-island', 'script', 'style', 'template'} +) + + +class ValsMethodologyStatementMissing(RuntimeError): + """A benchmark page no longer states a value the adapter fills from it.""" + + +@dataclass(frozen=True) +class PageStatement: + """A generation setting stated as a number in a benchmark's methodology.""" + + field: str + value: float | int + sentence: str + + +_SAGE_SENTENCE = ( + 'All models are evaluated with temperature 1, and produce at most 30K ' + 'tokens (the length of a short book - more than enough to adequately ' + 'grade student work).' +) +_MED_SENTENCE = ( + 'All models are evaluated with temperature 1, and produce at most 30k ' + 'tokens.' +) +_MMMU_SENTENCE = 'All models were ran with a temperature of 0.' + +PAGE_STATEMENTS: dict[str, tuple[PageStatement, ...]] = { + 'sage': ( + PageStatement('temperature', 1.0, _SAGE_SENTENCE), + PageStatement('max_tokens', 30000, _SAGE_SENTENCE), + ), + 'medcode': ( + PageStatement('temperature', 1.0, _MED_SENTENCE), + PageStatement('max_tokens', 30000, _MED_SENTENCE), + ), + 'medscribe': ( + PageStatement('temperature', 1.0, _MED_SENTENCE), + PageStatement('max_tokens', 30000, _MED_SENTENCE), + ), + 'mmmu': (PageStatement('temperature', 0.0, _MMMU_SENTENCE),), +} +PAGE_METADATA_DETAILS = { + 'version': 'benchmark_version', + 'runner': 'vals_runner', + 'mode': 'vals_mode', + 'family': 'vals_family', + 'archived': 'vals_archived', +} +ROW_OUTCOME_FIELDS = ( + 'token_totals', + 'usage', + 'task_results', + 'tie_breaker_score', +) @dataclass(frozen=True) @@ -75,6 +157,9 @@ class ValsMetric: model_id: str metrics: dict[str, Any] source_url: str + page_details: dict[str, Any] | None = None + page_fills: dict[str, float | int] | None = None + harness_label: str | None = None @dataclass(frozen=True) @@ -126,6 +211,73 @@ def handle_starttag( self.props.append(props) +class MethodologyParser(HTMLParser): + """Collect the text of the methodology section of a benchmark page.""" + + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.parts: list[str] = [] + self.found = False + self._article_depth = 0 + self._capturing = False + self._heading_open = False + self._skip_depth = 0 + + def handle_starttag( + self, tag: str, attrs: list[tuple[str, str | None]] + ) -> None: + attr_map = {name: value or '' for name, value in attrs} + if tag == 'article': + if self._article_depth: + self._article_depth += 1 + elif 'benchmark-details' in attr_map.get('class', '').split(): + self._article_depth = 1 + return + if not self._article_depth: + return + if tag == 'h2': + if self._capturing: + self._capturing = False + elif not self.found and attr_map.get('id') == 'methodology': + self.found = True + self._capturing = True + self._heading_open = True + return + if tag in METHODOLOGY_SKIPPED_TAGS: + self._skip_depth += 1 + elif tag in METHODOLOGY_BLOCK_TAGS: + self.parts.append(' ') + + def handle_endtag(self, tag: str) -> None: + if not self._article_depth: + return + if tag == 'article': + self._article_depth -= 1 + if not self._article_depth: + self._capturing = False + elif tag == 'h2': + self._heading_open = False + elif tag in METHODOLOGY_SKIPPED_TAGS: + self._skip_depth = max(0, self._skip_depth - 1) + elif tag in METHODOLOGY_BLOCK_TAGS: + self.parts.append(' ') + + def handle_data(self, data: str) -> None: + if self._capturing and not self._heading_open and not self._skip_depth: + self.parts.append(data) + + +def extract_methodology_text(page_html: str) -> str | None: + """Return the whitespace-normalised text under the methodology heading.""" + parser = MethodologyParser() + parser.feed(page_html) + parser.close() + if not parser.found: + return None + text = ' '.join(''.join(parser.parts).split()) + return text or None + + def fetch_text(url: str) -> str: request = Request(url, headers={'User-Agent': USER_AGENT}) try: @@ -211,11 +363,15 @@ def normalize_benchmark_page(page_html: str, source_url: str) -> dict[str, Any]: if not isinstance(metadata, dict) or not isinstance(tasks, dict): raise ValueError('BenchmarkView payload has invalid metadata/tasks') - return { + normalized = { 'metadata': metadata, 'tasks': tasks, 'source_url': source_url, } + methodology_text = extract_methodology_text(page_html) + if methodology_text is not None: + normalized['methodology_text'] = methodology_text + return normalized def extract_collection( @@ -273,7 +429,11 @@ def iter_vals_metrics_result( metrics: list[ValsMetric] = [] failures: list[SourceRecordFailure] = [] total_records = 0 - for benchmark_index, benchmark in enumerate(payload.get('benchmarks', [])): + slug_routes: dict[str, str] = {} + for benchmark_index, benchmark in sorted( + enumerate(payload.get('benchmarks', [])), + key=lambda item: _benchmark_order(item[1]), + ): if not isinstance(benchmark, dict): failures.append( SourceRecordFailure( @@ -311,6 +471,32 @@ def iter_vals_metrics_result( total_records += 1 continue benchmark_slug = str(raw_benchmark_slug) + first_route = slug_routes.setdefault(benchmark_slug, source_url) + if first_route != source_url: + failures.append( + SourceRecordFailure( + source_ref=source_url, + reason=( + f'Vals.ai route {source_url!r} serves benchmark slug ' + f'{benchmark_slug!r}, already converted from ' + f'{first_route!r}; route skipped' + ), + source_record={ + 'benchmark_slug': benchmark_slug, + 'source_url': source_url, + 'first_source_url': first_route, + }, + ) + ) + skipped = _skipped_row_failures( + source_url, + tasks, + f'Vals.ai route {source_url!r} skipped: benchmark slug ' + f'{benchmark_slug!r} already converted from {first_route!r}', + ) + failures.extend(skipped) + total_records += 1 + len(skipped) + continue benchmark_name = str(metadata.get('benchmark') or benchmark_slug) task_names = metadata.get('tasks') or {} if not isinstance(tasks, dict): @@ -324,6 +510,33 @@ def iter_vals_metrics_result( total_records += 1 continue + methodology_text = _optional_str(benchmark.get('methodology_text')) + statements = PAGE_STATEMENTS.get(benchmark_slug, ()) + missing_sentences = sorted( + { + statement.sentence + for statement in statements + if statement.sentence not in (methodology_text or '') + } + ) + if missing_sentences: + raise ValsMethodologyStatementMissing( + f'Vals.ai benchmark {benchmark_slug!r} ({source_url}): the ' + 'methodology section no longer contains the sentence the ' + f'page fill relies on: {missing_sentences!r}. Update ' + 'PAGE_STATEMENTS before publishing.' + ) + page_fills = { + statement.field: statement.value for statement in statements + } or None + page_details = { + detail_key: metadata.get(key) + for key, detail_key in PAGE_METADATA_DETAILS.items() + } + if methodology_text is not None: + page_details['vals_methodology_url'] = f'{source_url}#methodology' + harness_label = _optional_str(metadata.get('harness_label')) + for task_key, model_rows in tasks.items(): if not isinstance(model_rows, dict): failures.append( @@ -386,6 +599,9 @@ def iter_vals_metrics_result( model_id=str(model_id), metrics=row, source_url=source_url, + page_details=page_details, + page_fills=page_fills, + harness_label=harness_label, ) ) @@ -397,6 +613,40 @@ def iter_vals_metrics_result( ) +def _benchmark_order(benchmark: Any) -> tuple[bool, str]: + """Sort key preferring the route whose last segment is the embedded slug.""" + if not isinstance(benchmark, dict): + return (True, '') + route = str(benchmark.get('source_url') or BENCHMARKS_URL) + metadata = benchmark.get('metadata') + slug = ( + metadata.get('slug') or metadata.get('benchmark_id') + if isinstance(metadata, dict) + else None + ) + return (route.rstrip('/').rsplit('/', 1)[-1] != str(slug), route) + + +def _skipped_row_failures( + ref_prefix: str, + tasks: Any, + reason: str, +) -> list[SourceRecordFailure]: + """One failure per model/task row of a benchmark page that was skipped.""" + if not isinstance(tasks, dict): + return [] + return [ + SourceRecordFailure( + source_ref=f'{ref_prefix}/{task_key}/{model_id}', + reason=reason, + source_record=row, + ) + for task_key, model_rows in tasks.items() + if isinstance(model_rows, dict) + for model_id, row in model_rows.items() + ] + + def build_index( payload: dict[str, Any], ) -> dict[tuple[str, str], list[ValsMetric]]: @@ -533,6 +783,7 @@ def _make_bundle( 'industry': first.industry, 'leaderboard_page_url': first.source_url, 'extraction_method': 'static_astro_benchmark_view_props', + **(first.page_details or {}), } ), ), @@ -679,6 +930,8 @@ def make_result( 'verbosity': row.metrics.get('verbosity'), 'compute_effort': row.metrics.get('compute_effort'), 'provider': row.metrics.get('provider'), + 'harness': row.metrics.get('harness'), + **{field: row.metrics.get(field) for field in ROW_OUTCOME_FIELDS}, } ) @@ -738,26 +991,65 @@ def make_result( details=details, uncertainty=uncertainty, ), - generation_config=make_generation_config(row.metrics), + generation_config=make_generation_config( + row.metrics, + page_fills=row.page_fills, + harness_label=row.harness_label, + ), ) -def make_generation_config(metrics: dict[str, Any]) -> GenerationConfig | None: - explicit_generation_args = { - 'temperature': _optional_float(metrics.get('temperature')), - 'top_p': _optional_float(metrics.get('top_p')), - 'max_tokens': _optional_positive_int(metrics.get('max_output_tokens')), +def make_generation_config( + metrics: dict[str, Any], + *, + page_fills: dict[str, float | int] | None = None, + harness_label: str | None = None, +) -> GenerationConfig | None: + """Map one Vals.ai row to generation settings, filling nulls from the page. + + ``page_fills`` holds values the benchmark's methodology section states for + every model and fills only a field the row leaves null; a row value that is + present but unusable (unparseable, or a non-positive cap) stays unset. + ``_source`` records which one was used. + """ + row_fields = { + 'temperature': ('temperature', _optional_float), + 'max_tokens': ('max_output_tokens', _optional_positive_int), } - generation_args = None - if any(value is not None for value in explicit_generation_args.values()): - generation_args = GenerationArgs(**explicit_generation_args) + values: dict[str, Any] = {} + sources: dict[str, str] = {} + for field, (row_key, parse) in row_fields.items(): + raw_value = metrics.get(row_key) + if raw_value is None or raw_value is ASTRO_UNDEFINED: + page_value = (page_fills or {}).get(field) + if page_value is not None: + values[field] = page_value + sources[f'{field}_source'] = 'page_methodology' + continue + row_value = parse(raw_value) + if row_value is not None: + values[field] = row_value + sources[f'{field}_source'] = 'row' + + top_p = _optional_float(metrics.get('top_p')) + if top_p is not None: + values['top_p'] = top_p + reasoning = metrics.get('reasoning') + if isinstance(reasoning, bool): + values['reasoning'] = reasoning + harness = _optional_str(metrics.get('harness')) + if harness is not None: + values['agentic_eval_config'] = AgenticEvalConfig( + additional_details={_harness_detail_key(harness_label): harness} + ) + generation_args = GenerationArgs(**values) if values else None additional_details = _clean_details( { - 'reasoning': metrics.get('reasoning'), 'reasoning_effort': metrics.get('reasoning_effort'), 'verbosity': metrics.get('verbosity'), 'compute_effort': metrics.get('compute_effort'), + **sources, } ) if generation_args is not None or additional_details: @@ -768,6 +1060,11 @@ def make_generation_config(metrics: dict[str, Any]) -> GenerationConfig | None: return None +def _harness_detail_key(harness_label: str | None) -> str: + label = re.sub(r'[^a-z0-9]+', '_', (harness_label or '').lower()).strip('_') + return f'vals_{label or "harness"}' + + def make_source_data(row: ValsMetric) -> SourceDataPrivate | SourceDataUrl: details = _clean_details( { diff --git a/tests/data/vals_ai/mmmu_page.html b/tests/data/vals_ai/mmmu_page.html new file mode 100644 index 000000000..1920bea00 --- /dev/null +++ b/tests/data/vals_ai/mmmu_page.html @@ -0,0 +1,38 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

We adhered closely to the official MMMU Pro evaluation protocol with the following implementation details:

+
    +
  1. Prompt Structure: We used the chain-of-thought prompt from the original MMMU repository. Each question-answer set followed this format:
    2. +
+
Which of the following best explains the overall trend shown in the <image>?
+A. Migrations to areas of Central Asia for resettlement
+B. The spread of pathogens across the Silk Road
+C. Invasions by Mongol tribes
+D. Large-scale famine due to crop failures
+Answer the preceding multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of options. Think step by step before answering.
+
+
    +
  1. +

    Image Processing: All <image> tags were replaced with actual image bytes during model inference, replicating the original methodology exactly.

    +
    2. +
  2. +

    Standardization: We established a consistent baseline by using a default configuration of 8192 maximum output tokens for all models to ensure that outputs were not truncated. All models were ran with a temperature of 0.

    +
    3. +
  3. +

    Parsing Adaptation: We modified the answer extraction regex to handle markdown output from some models, ensuring reliable parsing across all responses.

    +
    4. +
  4. +

    Statistical Validity: This was a pass@1 evaluation (one attempt per question), with the large dataset size (1,700 questions). We found a standard deviation approximately 1% (calculated using the methodology from Adding Error Bars to Evals: A Statistical Approach to Language Model Evaluations)

    +
    5. +
  5. +

    Thinking Models Configuration: For models with “thinking” capabilities (Claude 3.7 Sonnet Thinking, o1, etc.), we set the maximum output token limit to 16,384 to accommodate extended reasoning.

    +
    6. +

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/reverse_eng_page.html b/tests/data/vals_ai/reverse_eng_page.html new file mode 100644 index 000000000..5d2f23523 --- /dev/null +++ b/tests/data/vals_ai/reverse_eng_page.html @@ -0,0 +1,15 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

SRE Bench pairs 19 in-house programs (averaging over 16,915.8 lines of code, spanning network protocols, firmware, games, file-format recovery, and malware) with in-house protection settings from a 27K-LoC anti-analysis suite of more than ten state-of-the-art protections, over half of which have no public implementations.

+SRE Bench development and evaluation pipeline: source code across languages and RE domains is compiled with protection layers into binaries, which agents reverse engineer in a workspace with auxiliary material and a deterministic grading server, with adversarial reward-hacking audits feeding back into development +

This yields 262 contamination-free instances with realistic difficulty. Each instance defines six scenario-specific, deterministically verifiable tasks that cover the core capabilities required for program understanding, for a total of 1,572 RE tasks.

+

Compared with existing RE-adjacent benchmarks — which rely on existing CTF challenges or toy programs with easy or textbook protection schemes — SRE Bench is RE-centric, contamination-controlled, and built from real-world-scale software with real-world protection complexity.

+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/sage_page.html b/tests/data/vals_ai/sage_page.html new file mode 100644 index 000000000..c193dbf77 --- /dev/null +++ b/tests/data/vals_ai/sage_page.html @@ -0,0 +1,23 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

The models are given student work samples and corresponding rubrics, expressed as a set of potential point deductions. We ask the models to determine which deductions apply to the given work.

+

All student work in the dataset consists of handwritten mathematical solutions, which adds an additional layer of complexity as models must be able to parse handwritten text and mathematical notation.

+

The evaluation process requires models to:

+
    +
  1. Parse and understand the rubric criteria
    2. +
  2. Analyze the student’s mathematical work and reasoning
    3. +
  3. Apply appropriate scoring based on the rubric guidelines
    4. +
  4. Provide clear justification for their scoring decisions
    5. +
+

This methodology tests not only the models’ mathematical understanding but also their ability to apply consistent evaluation standards—a critical skill for educational applications.

+

All models are evaluated with temperature 1, and produce at most 30K tokens (the length of a short book - more than enough to adequately grade student work).

+
+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/sage_page_statement_removed.html b/tests/data/vals_ai/sage_page_statement_removed.html new file mode 100644 index 000000000..fd9dd178d --- /dev/null +++ b/tests/data/vals_ai/sage_page_statement_removed.html @@ -0,0 +1,23 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

The models are given student work samples and corresponding rubrics, expressed as a set of potential point deductions. We ask the models to determine which deductions apply to the given work.

+

All student work in the dataset consists of handwritten mathematical solutions, which adds an additional layer of complexity as models must be able to parse handwritten text and mathematical notation.

+

The evaluation process requires models to:

+
    +
  1. Parse and understand the rubric criteria
    2. +
  2. Analyze the student’s mathematical work and reasoning
    3. +
  3. Apply appropriate scoring based on the rubric guidelines
    4. +
  4. Provide clear justification for their scoring decisions
    5. +
+

This methodology tests not only the models’ mathematical understanding but also their ability to apply consistent evaluation standards—a critical skill for educational applications.

+

All models are evaluated with default settings (the length of a short book - more than enough to adequately grade student work).

+
+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/srebench_page.html b/tests/data/vals_ai/srebench_page.html new file mode 100644 index 000000000..5d2f23523 --- /dev/null +++ b/tests/data/vals_ai/srebench_page.html @@ -0,0 +1,15 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

SRE Bench pairs 19 in-house programs (averaging over 16,915.8 lines of code, spanning network protocols, firmware, games, file-format recovery, and malware) with in-house protection settings from a 27K-LoC anti-analysis suite of more than ten state-of-the-art protections, over half of which have no public implementations.

+SRE Bench development and evaluation pipeline: source code across languages and RE domains is compiled with protection layers into binaries, which agents reverse engineer in a workspace with auxiliary material and a deterministic grading server, with adversarial reward-hacking audits feeding back into development +

This yields 262 contamination-free instances with realistic difficulty. Each instance defines six scenario-specific, deterministically verifiable tasks that cover the core capabilities required for program understanding, for a total of 1,572 RE tasks.

+

Compared with existing RE-adjacent benchmarks — which rely on existing CTF challenges or toy programs with easy or textbook protection schemes — SRE Bench is RE-centric, contamination-controlled, and built from real-world-scale software with real-world protection complexity.

+

Results

+

Not methodology.

+
+ diff --git a/tests/data/vals_ai/web_search_page.html b/tests/data/vals_ai/web_search_page.html new file mode 100644 index 000000000..fda3ce6bf --- /dev/null +++ b/tests/data/vals_ai/web_search_page.html @@ -0,0 +1,55 @@ + + + +
+

Key Takeaways

+

Trimmed fixture.

+

Methodology

+

Agents run on the same harness across both domains, with the web-search tool (native/Exa) as the variable under test. Two things distinguish these runs from the original FAB v2 and Legal Research Benchmark evaluations: the tool is swapped (native vs. Exa), and the domain-specific search tools those benchmarks provide (SEC EDGAR for finance, CourtListener for legal, and the like) are removed, leaving the web-search tool as the agent’s only means of gathering external information. Runs are orchestrated through Valkyrie with model calls routed through model-library.

+

Grading

+
    +
  • Legal Research: each task in the Legal Research Benchmark carries a rubric of required elements written by legal experts; rubrics range from 1 to 22 items per task (mean 9.47). All-pass is our primary metric: a task counts as correct only if every rubric item is satisfied. Grading is performed by a single LLM judge (GPT-5.4).
    • +
  • Finance Analysis: tasks use FAB v2’s dealbreaker-gated Partial Credit as the primary metric, category-balanced across the dataset’s nine question categories. A subset of checks in each question is flagged as dealbreakers—load-bearing facts or numbers required for a satisfactory answer—and failing any dealbreaker means the response scores 0% for that question regardless of everything else. All-pass (100% only if every check passes) is reported as a secondary metric. Grading uses the same three-judge jury as FAB v2: GPT-5.4, Gemini-3.1-Pro, and Claude Sonnet 4.6.
    • +
+

For the full rubrics, jury setup, and scoring rules behind any specific grading decision, refer to the original FAB v2 and Legal Research Benchmark methodologies.

+

Dataset

+
    +
  • Legal: 208 expert-authored legal research tasks, each tagged by the reasoning skill it tests (a task can test more than one): Statutory Interpretation, Application of Precedent to a Fact Pattern, Doctrinal Test Identification, Regulatory Framework Interpretation, Interaction of Multiple Legal Regimes, Cross-Jurisdiction Comparison, Timeline/Doctrinal Evolution, Validity of Precedent, Resolving Conflicting Precedent, and Consensus vs. Disagreement Across Courts.
    • +
  • Finance: 450 finance-analyst questions across nine categories, scored with FAB v2’s category-balanced methodology and reflecting real equity-research workflows: General Qualitative Analysis, General Quantitative Analysis, Market Analysis, Comparables, Precedents, Adjustments, Earnings Analysis, Disclosure Analysis, and Financial Modeling.
    • +
+

Statistical Analysis

+

To establish whether the two search tools differ statistically, we fit a mixed-effects model per domain, with search tool as a fixed effect (native as the reference level) and a random intercept per task absorbing task-level difficulty: a logistic model for legal’s All-pass metric (all_pass ~ search_tool + (1 | task_id)) and a linear model for finance’s primary metric (score ~ search_tool + (1 | task_id)). Each model pools the per-task, per-model observations underlying the Results charts above. The domain-level significance cited in the Key Takeaways comes from these models.

+

The fixed effect of search tool (Exa relative to native) in each model:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DomainOutcome (model)Exa effectStd. errorStatisticp
FinanceWeighted score, 0-100 (linear)+6.51 points0.98t = 6.62< 0.001
LegalAll-pass, 0/1 (logistic)−0.02 log-odds0.13z = −0.170.86

Results

+

Not methodology.

+
+ diff --git a/tests/test_vals_ai_adapter.py b/tests/test_vals_ai_adapter.py index d9425ece9..4d9851e4a 100644 --- a/tests/test_vals_ai_adapter.py +++ b/tests/test_vals_ai_adapter.py @@ -1,16 +1,44 @@ from __future__ import annotations +import json from pathlib import Path +import pytest + from every_eval_ever.adapters.vals_ai import adapter from every_eval_ever.eval_types import EvaluationLog from every_eval_ever.validate import validate_file -FIXTURE_PATH = ( - Path(__file__).parent / 'data' / 'vals_ai' / 'finance_agent_payload.json' +DATA_DIR = Path(__file__).parent / 'data' / 'vals_ai' +FIXTURE_PATH = DATA_DIR / 'finance_agent_payload.json' +SAGE_SENTENCE = ( + 'All models are evaluated with temperature 1, and produce at most 30K ' + 'tokens (the length of a short book - more than enough to adequately ' + 'grade student work).' ) +def page_benchmark(fixture: str, route: str) -> dict: + return adapter.normalize_benchmark_page( + (DATA_DIR / fixture).read_text(encoding='utf-8'), + f'https://www.vals.ai/benchmarks/{route}', + ) + + +def convert_pages(*benchmarks: dict): + return adapter.convert_logs( + {'benchmarks': list(benchmarks)}, + retrieved_timestamp='1234567890.0', + ) + + +def logs_by_raw_id(result) -> dict: + return { + bundle.log.model_info.additional_details['vals_model_id']: bundle.log + for bundle in result.records + } + + def sample_payload() -> dict: return { 'source_url': 'https://www.vals.ai/benchmarks', @@ -484,3 +512,325 @@ def test_export_paths_validate(tmp_path: Path): assert path.parent.parent.parent == output_dir report = validate_file(path) assert report.valid, report.errors + + +def test_sage_methodology_fills_null_row_values_and_names_the_source(): + result = convert_pages(page_benchmark('sage_page.html', 'sage')) + + assert not result.failures + logs = logs_by_raw_id(result) + assert set(logs) == { + 'anthropic/claude-sonnet-4-6', + 'openai/gpt-5.5', + 'openai/gpt-5.6-sol', + } + + full = logs['anthropic/claude-sonnet-4-6'].evaluation_results[0] + assert full.generation_config.generation_args.temperature == 1.0 + assert full.generation_config.generation_args.max_tokens == 30000 + assert full.generation_config.additional_details['temperature_source'] == ( + 'row' + ) + assert full.generation_config.additional_details['max_tokens_source'] == ( + 'row' + ) + + null_cap = logs['openai/gpt-5.5'].evaluation_results[0] + assert null_cap.score_details.details.get('max_output_tokens') is None + assert null_cap.generation_config.generation_args.max_tokens == 30000 + assert null_cap.generation_config.additional_details == { + 'reasoning_effort': 'xhigh', + 'temperature_source': 'row', + 'max_tokens_source': 'page_methodology', + } + + null_temperature = logs['openai/gpt-5.6-sol'].evaluation_results[0] + args = null_temperature.generation_config.generation_args + assert args.temperature == 1.0 + assert args.max_tokens == 30000 + details = null_temperature.generation_config.additional_details + assert details['temperature_source'] == 'page_methodology' + assert details['max_tokens_source'] == 'row' + + for log in logs.values(): + EvaluationLog.model_validate(log.model_dump()) + config = log.evaluation_results[0].generation_config + assert config.generation_args.eval_limits is None + + +def test_sage_methodology_is_linked_not_stored(): + benchmark = page_benchmark('sage_page.html', 'sage') + text = benchmark['methodology_text'] + + assert text.startswith( + 'The models are given student work samples and corresponding rubrics' + ) + assert text.endswith(SAGE_SENTENCE) + assert 'Parse and understand the rubric criteria Analyze' in text + assert 'Methodology' not in text + assert 'Trimmed fixture' not in text + assert 'Not methodology' not in text + + log = convert_pages(benchmark).records[0].log + source_details = log.source_metadata.additional_details + assert source_details['vals_methodology_url'] == ( + 'https://www.vals.ai/benchmarks/sage#methodology' + ) + assert 'vals_methodology_text' not in source_details + assert 'vals_methodology_sha256' not in source_details + assert source_details['benchmark_version'] == '1' + assert source_details['vals_runner'] == 'custom' + assert source_details['vals_mode'] == 'one-shot' + assert source_details['vals_family'] == 'sage' + assert source_details['vals_archived'] == 'false' + + +def test_missing_methodology_statement_fails_the_run(): + payload = sample_payload() + payload['benchmarks'].append( + page_benchmark('sage_page_statement_removed.html', 'sage') + ) + + with pytest.raises(adapter.ValsMethodologyStatementMissing) as excinfo: + adapter.convert_logs(payload, retrieved_timestamp='1234567890.0') + + message = str(excinfo.value) + assert "'sage'" in message + assert SAGE_SENTENCE in message + assert 'https://www.vals.ai/benchmarks/sage' in message + + +def test_statement_slug_replayed_without_methodology_fails_the_run(): + benchmark = page_benchmark('sage_page.html', 'sage') + del benchmark['methodology_text'] + + with pytest.raises(adapter.ValsMethodologyStatementMissing): + convert_pages(benchmark) + + +def test_mmmu_methodology_fills_temperature_but_not_caps(): + benchmark = page_benchmark('mmmu_page.html', 'mmmu') + text = benchmark['methodology_text'] + assert 'All models were ran with a temperature of 0.' in text + benchmark['tasks']['overall']['openai/o1-2024-12-17'][ + 'max_output_tokens' + ] = None + + logs = logs_by_raw_id(convert_pages(benchmark)) + + gemini = logs['google/gemini-2.0-flash-001'].evaluation_results[0] + assert gemini.generation_config.generation_args.temperature == 0.0 + assert gemini.generation_config.generation_args.max_tokens == 8192 + assert gemini.generation_config.additional_details == { + 'temperature_source': 'page_methodology', + 'max_tokens_source': 'row', + } + + o1 = logs['openai/o1-2024-12-17'].evaluation_results[0] + assert o1.generation_config.generation_args.temperature == 0.0 + assert o1.generation_config.generation_args.max_tokens is None + assert 'max_tokens_source' not in o1.generation_config.additional_details + + +def test_reasoning_is_typed_only_from_a_real_bool(): + def reasoning_of(row: dict): + config = adapter.make_generation_config(row) + if config is None or config.generation_args is None: + return None + return config.generation_args.reasoning + + assert reasoning_of({'reasoning': True}) is True + assert reasoning_of({'reasoning': False}) is False + assert reasoning_of({'reasoning': None, 'reasoning_effort': 'high'}) is None + assert reasoning_of({'reasoning': 'true', 'temperature': 1}) is None + + config = adapter.make_generation_config({'reasoning': True}) + assert config.additional_details is None + + +def test_harness_lands_in_agentic_eval_config(): + config = adapter.make_generation_config( + {'harness': 'OpenHands', 'temperature': 1} + ) + + args = config.generation_args + assert args.agentic_eval_config.additional_details == { + 'vals_harness': 'OpenHands' + } + assert args.eval_limits is None + + +def test_web_search_keeps_embedded_slug_and_page_harness_label(): + result = convert_pages(page_benchmark('web_search_page.html', 'web_search')) + + assert not result.failures + logs = logs_by_raw_id(result) + native = logs['anthropic/claude-fable-5'] + exa = logs['anthropic/claude-fable-5-exa'] + + assert native.evaluation_id.startswith('vals-ai/web_search_backends/') + details = native.source_metadata.additional_details + assert details['benchmark_slug'] == 'web_search_backends' + assert details['leaderboard_page_url'] == ( + 'https://www.vals.ai/benchmarks/web_search' + ) + assert native.evaluation_results[0].evaluation_name == ( + 'vals_ai.web_search_backends.overall' + ) + for log, tool in ((native, 'Native'), (exa, 'Exa')): + args = log.evaluation_results[0].generation_config.generation_args + assert args.agentic_eval_config.additional_details == { + 'vals_search_tool': tool + } + assert args.eval_limits is None + assert ( + log.evaluation_results[0].score_details.details['harness'] == tool + ) + + +def test_row_outcomes_are_json_encoded_in_score_details(): + payload = sample_payload() + row = payload['benchmarks'][0]['tasks']['overall']['openai/gpt-5.4'] + row['token_totals'] = {'input_tokens': 10, 'output_tokens': 2} + row['usage'] = {'trials': 3} + row['task_results'] = {'task-1': {'passed': True}} + row['tie_breaker_score'] = 53.5 + + bundles = adapter.make_logs(payload, retrieved_timestamp='1234567890.0') + finance = next( + bundle.log + for bundle in bundles + if bundle.log.model_info.id == 'openai/gpt-5.4' + ) + overall = next( + result + for result in finance.evaluation_results + if result.evaluation_name == 'vals_ai.finance_agent.overall' + ) + + details = overall.score_details.details + assert json.loads(details['token_totals']) == { + 'input_tokens': 10, + 'output_tokens': 2, + } + assert json.loads(details['usage']) == {'trials': 3} + assert json.loads(details['task_results']) == {'task-1': {'passed': True}} + assert details['tie_breaker_score'] == '53.5' + assert overall.generation_config.generation_args.eval_limits is None + + +def test_one_part_model_id_without_provider_fails_the_row(): + payload = sample_payload() + payload['benchmarks'][0]['tasks']['overall']['mystery-model'] = { + 'accuracy': 50.0, + } + + result = adapter.convert_logs(payload, retrieved_timestamp='1234567890.0') + + assert result.records + assert len(result.failures) == 1 + assert result.failures[0].source_ref == 'finance_agent/mystery-model' + assert 'must be known' in result.failures[0].reason + + +def _assert_one_route_kept(result, kept: str, skipped: str) -> None: + assert len(result.records) == 2 + for bundle in result.records: + details = bundle.log.source_metadata.additional_details + assert details['leaderboard_page_url'] == kept + assert len(bundle.log.evaluation_results) == 1 + failure = result.failures[0] + assert failure.source_ref == skipped + assert kept in failure.reason + assert "'srebench'" in failure.reason + assert failure.source_record == { + 'benchmark_slug': 'srebench', + 'source_url': skipped, + 'first_source_url': kept, + } + assert [row.source_ref for row in result.failures[1:]] == [ + f'{skipped}/overall/openai/gpt-6-astra', + f'{skipped}/overall/openai/gpt-5.6-sol', + ] + + +def test_routes_sharing_a_slug_keep_the_route_named_after_it(): + srebench = page_benchmark('srebench_page.html', 'srebench') + reverse_eng = page_benchmark('reverse_eng_page.html', 'reverse_eng') + assert srebench['metadata']['slug'] == 'srebench' + assert reverse_eng['metadata']['slug'] == 'srebench' + + for order in ((srebench, reverse_eng), (reverse_eng, srebench)): + _assert_one_route_kept( + convert_pages(*order), + kept='https://www.vals.ai/benchmarks/srebench', + skipped='https://www.vals.ai/benchmarks/reverse_eng', + ) + + +def test_routes_sharing_a_slug_fall_back_to_alphabetical_first(): + reverse_eng = page_benchmark('reverse_eng_page.html', 'reverse_eng') + sre_mirror = page_benchmark('srebench_page.html', 'sre_mirror') + + for order in ((sre_mirror, reverse_eng), (reverse_eng, sre_mirror)): + _assert_one_route_kept( + convert_pages(*order), + kept='https://www.vals.ai/benchmarks/reverse_eng', + skipped='https://www.vals.ai/benchmarks/sre_mirror', + ) + + +def test_row_failure_records_do_not_carry_methodology_text(): + benchmark = page_benchmark('sage_page.html', 'sage') + benchmark['tasks']['calculus']['mystery-model'] = {'accuracy': 50.0} + + result = convert_pages(benchmark) + + assert len(result.failures) == 1 + [record] = result.failures[0].source_record + assert record['benchmark_slug'] == 'sage' + assert record['page_details']['vals_methodology_url'] == ( + 'https://www.vals.ai/benchmarks/sage#methodology' + ) + assert benchmark['methodology_text'] not in json.dumps( + result.failures[0].model_dump() + ) + + +def test_present_but_unusable_row_values_are_not_page_filled(): + benchmark = page_benchmark('sage_page.html', 'sage') + row = benchmark['tasks']['calculus']['openai/gpt-5.5'] + row['max_output_tokens'] = 0 + row['temperature'] = 'abc' + + logs = logs_by_raw_id(convert_pages(benchmark)) + + result = logs['openai/gpt-5.5'].evaluation_results[0] + assert result.score_details.details['max_output_tokens'] == '0' + assert result.score_details.details['temperature'] == 'abc' + config = result.generation_config + assert config.generation_args is None + assert 'temperature_source' not in config.additional_details + assert 'max_tokens_source' not in config.additional_details + + +def test_skipped_benchmark_failures_add_up_to_source_rows(): + srebench = page_benchmark('srebench_page.html', 'srebench') + reverse_eng = page_benchmark('reverse_eng_page.html', 'reverse_eng') + finance = sample_payload()['benchmarks'][0] + benchmarks = [srebench, reverse_eng, finance] + source_rows = sum( + len(rows) + for benchmark in benchmarks + for rows in benchmark['tasks'].values() + ) + + result = adapter.iter_vals_metrics_result({'benchmarks': benchmarks}) + + assert [failure.source_ref for failure in result.failures] == [ + 'https://www.vals.ai/benchmarks/reverse_eng', + 'https://www.vals.ai/benchmarks/reverse_eng/overall/openai/gpt-6-astra', + 'https://www.vals.ai/benchmarks/reverse_eng/overall/openai/gpt-5.6-sol', + ] + assert result.total_records == source_rows + 1 + assert result.total_records == len(result.records) + len(result.failures) From b590176f8948011f415a82010cab74061606311a Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 04/16] arc_agi: write max_tokens and task counts from the published public-eval logs The adapter clones arcprize/arc_agi_v1_public_eval and arc_agi_v2_public_eval at pinned commits (git clone --no-checkout, sparse checkout of the leaderboard's model directories, git log from the same clone). --arc-runs-dir and --arc-history-dir replace the clone in offline runs. A score result is filled when the directory's task files and results.json reproduce the published score and cost at the row's precision, every attempt logs the same cap key and value, and no path in the directory was rewritten in the history. Written: generation_args.max_tokens, cap_key, request_log and scored_tasks. Cost results and other datasets are not changed. A directory that fails a check is recorded as request_args_unavailable= on the record and in the report. Co-Authored-By: Claude Fable 5.1 --- every_eval_ever/adapters/arc_agi/adapter.py | 294 +++++++- every_eval_ever/adapters/arc_agi/run_logs.py | 635 ++++++++++++++++++ tests/data/arc_agi/history/v2.json | 126 ++++ .../28a6681f.json | 320 +++++++++ .../58490d8a.json | 314 +++++++++ .../7b5033c1.json | 230 +++++++ .../results.json | 41 ++ tests/test_arc_agi_adapter.py | 573 +++++++++++++++- 8 files changed, 2505 insertions(+), 28 deletions(-) create mode 100644 every_eval_ever/adapters/arc_agi/run_logs.py create mode 100644 tests/data/arc_agi/history/v2.json create mode 100644 tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/28a6681f.json create mode 100644 tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/58490d8a.json create mode 100644 tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/7b5033c1.json create mode 100644 tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/results.json diff --git a/every_eval_ever/adapters/arc_agi/adapter.py b/every_eval_ever/adapters/arc_agi/adapter.py index a3d7dc400..91955a041 100644 --- a/every_eval_ever/adapters/arc_agi/adapter.py +++ b/every_eval_ever/adapters/arc_agi/adapter.py @@ -9,6 +9,13 @@ id -> display name). ``datasets.json`` names the ARC-AGI-1/2/3 splits. - The older ``/media/data/leaderboard/evaluations.json`` endpoint is gone (404 as of 2026-08-12); this adapter fetches the current files. +- Published run logs for the two public sets: the Hugging Face datasets + ``arcprize/arc_agi_v1_public_eval`` and ``arcprize/arc_agi_v2_public_eval``, + read at pinned commits (see ``run_logs.py``). A ``v1_Public_Eval`` or + ``v2_Public_Eval`` score whose chosen row's ``modelId`` names a run + directory that passes every gate gets the logged output cap, a uniform + logged temperature and the scored task count; a directory that fails a gate + is recorded as ``request_args_unavailable``. Each evaluation row has shape: { @@ -30,7 +37,8 @@ uv run python -m every_eval_ever.adapters.arc_agi.adapter \\ --output-dir data/arc-agi uv run python -m every_eval_ever.adapters.arc_agi.adapter \\ - --input-json /tmp/arc_agi_payload.json --output-dir /tmp/arc-smoke + --input-json /tmp/arc_agi_payload.json --output-dir /tmp/arc-smoke \\ + --arc-runs-dir /tmp/arc_public_eval """ from __future__ import annotations @@ -39,16 +47,28 @@ import json import math import re +import sys +import tempfile import time -from collections import defaultdict +from collections import Counter, defaultdict from pathlib import Path from typing import Any +from every_eval_ever.adapters.arc_agi.run_logs import ( + RUN_DATASETS, + RunLogs, + RunLogsUnavailable, + fetch_run_logs, + loads_stated, + local_run_logs, +) from every_eval_ever.eval_types import ( EvalLibrary, EvaluationLog, EvaluationResult, EvaluatorRelationship, + GenerationArgs, + GenerationConfig, MetricConfig, ModelInfo, ScoreDetails, @@ -66,7 +86,7 @@ save_evaluation_logs, save_failure_report, ) -from every_eval_ever.helpers.fetch import fetch_json +from every_eval_ever.helpers.fetch import fetch_text SOURCE_NAME = 'ARC Prize leaderboard' SOURCE_ORGANIZATION = 'ARC Prize Foundation' @@ -135,6 +155,28 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: 'future runs can replay offline with --input-json.' ), ) + parser.add_argument( + '--arc-runs-dir', + type=Path, + help=( + 'Read the published run logs from this directory, holding v1/ ' + 'and v2/ checkouts of the ARC public-eval datasets, instead of ' + 'cloning them.' + ), + ) + parser.add_argument( + '--arc-history-dir', + type=Path, + help=( + 'With --arc-runs-dir whose v1/ and v2/ are not git checkouts: ' + 'a directory of v1/ and v2/ clones supplying their commit history.' + ), + ) + parser.add_argument( + '--no-run-logs', + action='store_true', + help='Skip the run-log join; score results carry no request args.', + ) parser.add_argument( '--failure-report', type=Path, @@ -146,18 +188,34 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace: return parser.parse_args(argv) -def fetch_payload(base_url: str) -> dict[str, Any]: +PAYLOAD_FILES = ('evaluations', 'models', 'providers', 'datasets') + + +def fetch_payload_texts(base_url: str) -> dict[str, str]: + """Fetch the leaderboard's JSON files as their raw text.""" base = base_url.rstrip('/') - return { - 'evaluations': fetch_json(f'{base}/evaluations.json'), - 'models': fetch_json(f'{base}/models.json'), - 'providers': fetch_json(f'{base}/providers.json'), - 'datasets': fetch_json(f'{base}/datasets.json'), - } + return {name: fetch_text(f'{base}/{name}.json') for name in PAYLOAD_FILES} + + +def parse_payload_texts(texts: dict[str, str]) -> dict[str, Any]: + """Parse the fetched files, keeping each float's written precision.""" + return {name: loads_stated(text) for name, text in texts.items()} + + +def combined_payload_text( + texts: dict[str, str], base_url: str, fetched_at: str +) -> str: + """Return the --save-raw-json file: the fetched files embedded verbatim.""" + parts = [ + f' {json.dumps("source_base_url")}: {json.dumps(base_url)}', + f' {json.dumps("fetched_at")}: {json.dumps(fetched_at)}', + *(f' {json.dumps(name)}: {texts[name].strip()}' for name in texts), + ] + return '{\n' + ',\n'.join(parts) + '\n}\n' def load_payload_file(path: Path) -> dict[str, Any]: - payload = json.loads(path.read_text(encoding='utf-8')) + payload = loads_stated(path.read_text(encoding='utf-8')) if not isinstance(payload, dict) or 'evaluations' not in payload: raise ValueError( f'{path}: expected a combined payload with an "evaluations" key ' @@ -292,11 +350,52 @@ def _cost_result( ) +def run_log_fields( + run_logs: RunLogs | None, + dataset_id: str, + row: dict, + retrieved: float, +) -> tuple[GenerationConfig | None, dict[str, str]]: + """Return the score result's generation config and extra source details. + + Both are empty when the set publishes no logs or no run directory is + named after the row's ``modelId``. + """ + if run_logs is None: + return None, {} + outcome = run_logs.join(dataset_id, row, retrieved) + if outcome is None: + return None, {} + if outcome.run is None: + return ( + GenerationConfig( + additional_details={ + 'request_args_unavailable': str(outcome.rejection), + 'request_log': outcome.request_log, + } + ), + {}, + ) + run = outcome.run + details = {'cap_key': run.cap_key, 'request_log': outcome.request_log} + return ( + GenerationConfig( + generation_args=GenerationArgs( + max_tokens=run.cap_value, temperature=run.temperature + ), + additional_details=details, + ), + {'scored_tasks': str(run.task_count)}, + ) + + def make_results( rows_for_canonical: list[dict], developer_name: str, metric_bounds: dict[str, dict[str, float]], dataset_names: dict[str, str], + run_logs: RunLogs | None = None, + retrieved: float = math.inf, ) -> list[EvaluationResult]: results = [] by_dataset = defaultdict(list) @@ -309,12 +408,17 @@ def make_results( {r['modelId'] for r in by_dataset[dataset_id]} ) dataset_display_name = dataset_names.get(dataset_id) + generation_config, run_source_details = run_log_fields( + run_logs, dataset_id, row, retrieved + ) + score_source_data = make_source_data(dataset_id, dataset_display_name) + score_source_data.additional_details.update(run_source_details) results.append( EvaluationResult( evaluation_result_id=f'{dataset_id}::score', evaluation_name=dataset_id, - source_data=make_source_data(dataset_id, dataset_display_name), + source_data=score_source_data, metric_config=MetricConfig( metric_id='score', metric_name='ARC score', @@ -337,6 +441,7 @@ def make_results( ), }, ), + generation_config=generation_config, ) ) @@ -377,6 +482,7 @@ def make_log( model_entry: dict | None, provider_entry: dict | None, dataset_names: dict[str, str], + run_logs: RunLogs | None = None, ) -> EvaluationLog: primary_raw_model_id = choose_primary_raw_model_id( rows_for_canonical, developer_name @@ -437,7 +543,12 @@ def make_log( additional_details=model_details, ), evaluation_results=make_results( - rows_for_canonical, developer_name, metric_bounds, dataset_names + rows_for_canonical, + developer_name, + metric_bounds, + dataset_names, + run_logs, + float(retrieved_timestamp), ), ) @@ -451,6 +562,7 @@ def _valid_number(value: Any) -> bool: def convert_logs( payload: dict[str, Any], retrieved_timestamp: str | None = None, + run_logs: RunLogs | None = None, ) -> SourceConversionResult[tuple[EvaluationLog, str, str]]: timestamp = retrieved_timestamp or str(time.time()) models = {m['id']: m for m in payload.get('models') or []} @@ -578,6 +690,7 @@ def convert_logs( model_entry, provider_entry, dataset_names, + run_logs, ) bundles.append((log, developer_name, model_name)) @@ -605,36 +718,165 @@ def export( ) +def public_model_ids(payload: dict[str, Any]) -> dict[str, set[str]]: + """Return the displayed rows' ``modelId``s per set that publishes run logs.""" + model_ids: dict[str, set[str]] = defaultdict(set) + for row in payload.get('evaluations') or []: + if ( + isinstance(row, dict) + and row.get('display') is True + and row.get('datasetId') in RUN_DATASETS + and isinstance(row.get('modelId'), str) + ): + model_ids[row['datasetId']].add(row['modelId']) + return dict(model_ids) + + +#: Gates whose failure means the logs and the leaderboard disagree, as +#: opposed to logs that were never published complete. +REGRESSION_GATES = frozenset( + { + 'score_mismatch', + 'cost_mismatch', + 'task_score_mismatch', + 'task_attempts_mismatch', + 'task_set_mismatch', + 'run_score_mismatch', + 'cap_conflict', + 'cap_key_mixed', + 'pair_index_mismatch', + 'task_id_mismatch', + 'test_id_mismatch', + 'run_rewritten', + } +) + + +def _rejection_entry(outcome: Any) -> dict[str, Any]: + return { + 'model_id': outcome.model_id, + 'dataset_id': outcome.dataset_id, + 'directory': outcome.model_id, + 'gate': outcome.rejection, + 'detail': outcome.detail, + } + + +def run_log_report(run_logs: RunLogs) -> dict[str, Any]: + """Return the run-level join report: sources, gate counts, rows. + + ``gate_counts`` counts every joined score row by outcome (``filled`` or + the gate that rejected it) plus ``no_run_directory`` for rows whose + ``modelId`` names no directory. Rejections are split into + ``regressions`` (:data:`REGRESSION_GATES`) and ``unavailable``. + """ + counts = Counter( + 'filled' if outcome.run is not None else str(outcome.rejection) + for outcome in run_logs.outcomes + ) + if run_logs.unjoined: + counts['no_run_directory'] = len(run_logs.unjoined) + ordered = sorted( + run_logs.outcomes, key=lambda o: (o.dataset_id, o.model_id) + ) + return { + 'sources': { + dataset_id: { + 'dataset': source.dataset.repo_id, + 'revision': source.revision, + } + for dataset_id, source in sorted(run_logs.sources.items()) + }, + 'gate_counts': dict(sorted(counts.items())), + 'regressions': [ + _rejection_entry(o) + for o in ordered + if o.run is None and o.rejection in REGRESSION_GATES + ], + 'unavailable': [ + _rejection_entry(o) + for o in ordered + if o.run is None and o.rejection not in REGRESSION_GATES + ], + 'filled': [ + { + 'model_id': o.model_id, + 'dataset_id': o.dataset_id, + 'directory': o.model_id, + 'cap_key': o.run.cap_key, + 'max_tokens': o.run.cap_value, + 'temperature': o.run.temperature, + 'scored_tasks': o.run.task_count, + } + for o in ordered + if o.run is not None + ], + } + + +def summarize_joins(run_logs: RunLogs) -> str: + """Return a one-line count of run-log joins by outcome.""" + counts = Counter( + 'filled' if outcome.run is not None else str(outcome.rejection) + for outcome in run_logs.outcomes + ) + parts = ', '.join(f'{key} {counts[key]}' for key in sorted(counts)) + return f'ARC run-log joins: {len(run_logs.outcomes)} ({parts or "none"})' + + def run(args: argparse.Namespace) -> int: if args.input_json is not None: payload = load_payload_file(args.input_json) else: - payload = fetch_payload(args.base_url) + texts = fetch_payload_texts(args.base_url) + payload = parse_payload_texts(texts) if args.save_raw_json is not None: args.save_raw_json.parent.mkdir(parents=True, exist_ok=True) args.save_raw_json.write_text( - json.dumps( - { - 'source_base_url': args.base_url, - 'fetched_at': str(time.time()), - **payload, - }, - indent=2, - allow_nan=False, - ), + combined_payload_text(texts, args.base_url, str(time.time())), encoding='utf-8', ) - result = convert_logs(payload) + with tempfile.TemporaryDirectory(prefix='arc-run-logs-') as work_dir: + if args.no_run_logs: + run_logs = None + elif args.arc_runs_dir is not None: + try: + run_logs = local_run_logs( + args.arc_runs_dir, args.arc_history_dir + ) + except RunLogsUnavailable as exc: + raise SystemExit(f'ARC run logs: {exc}') from exc + else: + run_logs = fetch_run_logs( + public_model_ids(payload), Path(work_dir) + ) + result = convert_logs(payload, run_logs=run_logs) paths = export(result.records, args.output_dir) for path in paths: print(path) - if result.failures: + if run_logs is not None: + print(summarize_joins(run_logs)) + for entry in run_log_report(run_logs)['regressions']: + print( + 'ARC run-log regression: ' + f'{entry["model_id"]} {entry["dataset_id"]} {entry["gate"]}', + file=sys.stderr, + ) + if result.failures or run_logs is not None: report_path = save_failure_report( result, args.failure_report or default_failure_report_path(args.output_dir), ) - print(f'Failure report: {report_path}') + if run_logs is not None: + report = json.loads(report_path.read_text(encoding='utf-8')) + report['run_log_evidence'] = run_log_report(run_logs) + report_path.write_text( + json.dumps(report, indent=2, ensure_ascii=False, allow_nan=False) + + '\n', + encoding='utf-8', + ) + print(f'Adapter report: {report_path}') result.raise_if_incomplete() return len(paths) diff --git a/every_eval_ever/adapters/arc_agi/run_logs.py b/every_eval_ever/adapters/arc_agi/run_logs.py new file mode 100644 index 000000000..60cf4d777 --- /dev/null +++ b/every_eval_ever/adapters/arc_agi/run_logs.py @@ -0,0 +1,635 @@ +"""Join ARC Prize leaderboard score rows to the published run logs. + +ARC Prize publishes the outputs of its public-evaluation runs as two Hugging +Face datasets, one per public set. Each holds one directory per run, named +after the leaderboard ``modelId``, with one JSON file per task (a list of test +pairs, each mapping ``attempt_`` to the logged request and its outcome) and +a ``results.json`` summarising the run. + +A directory speaks for a leaderboard score only when it passes every gate in +:func:`read_run` and :func:`row_rejection`. What it then states (the output +cap every logged request carried, a temperature every request shared, the +number of tasks scored) is written onto that score result only. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime, timezone +from decimal import Decimal +from pathlib import Path +from typing import Any + +from every_eval_ever.helpers import raw_capture + + +@dataclass(frozen=True) +class RunDataset: + """One published run-output dataset, pinned to a commit.""" + + dataset_id: str + repo_id: str + revision: str + subdir: str + + @property + def url(self) -> str: + return f'https://huggingface.co/datasets/{self.repo_id}' + + +#: Leaderboard ``datasetId`` -> the dataset holding that set's run logs. Only +#: the public sets publish outputs; a directory fills its own set only. +RUN_DATASETS: dict[str, RunDataset] = { + 'v1_Public_Eval': RunDataset( + dataset_id='v1_Public_Eval', + repo_id='arcprize/arc_agi_v1_public_eval', + revision='3e9c9d1a8402aff82356815c106cecaea65cb7d9', + subdir='v1', + ), + 'v2_Public_Eval': RunDataset( + dataset_id='v2_Public_Eval', + repo_id='arcprize/arc_agi_v2_public_eval', + revision='026789c1c12a4c34580a32e84dcaf5630d7e8f31', + subdir='v2', + ), +} + +#: Output-cap spellings a logged request may carry. A run states exactly one, +#: under the same key and value, on every recorded request. +CAP_KEYS = ('max_tokens', 'max_completion_tokens', 'max_output_tokens') +RESULTS_FILE = 'results.json' +_ATTEMPT_KEY = re.compile(r'^attempt_\d+$') + + +class StatedFloat(float): + """A float parsed from JSON that keeps the text it was written as.""" + + text: str + + def __new__(cls, text: str) -> StatedFloat: + value = super().__new__(cls, text) + value.text = text + return value + + +def loads_stated(text: str) -> Any: + """Parse JSON keeping each float's written form (see :class:`StatedFloat`).""" + return json.loads(text, parse_float=StatedFloat) + + +def stated_decimal(value: float | int) -> Decimal: + """Return ``value`` as a Decimal at the precision it was written with. + + A :class:`StatedFloat` keeps its JSON text, so ``0.470`` stays three + decimals; any other number falls back to its shortest ``repr``. + """ + if isinstance(value, StatedFloat): + return Decimal(value.text) + if isinstance(value, int): + return Decimal(value) + return Decimal(repr(float(value))) + + +class RunRejected(Exception): + """A run directory failed a gate; ``reason`` names the gate.""" + + def __init__(self, reason: str, detail: str = '') -> None: + super().__init__(f'{reason}: {detail}' if detail else reason) + self.reason = reason + self.detail = detail + + +@dataclass(frozen=True) +class PublishedRun: + """One run as its logs state it, after the directory-level gates.""" + + directory: str + cap_key: str + cap_value: int + temperature: float | None + score: float + avg_cost_per_task: Any + latest_end: datetime + task_count: int + attempt_count: int + + +@dataclass(frozen=True) +class RunLogSource: + """A local checkout of one run dataset and, if known, its path history.""" + + dataset: RunDataset + root: Path + revision: str + history: dict[str, int] + + +@dataclass(frozen=True) +class JoinOutcome: + """What one score row's join produced.""" + + dataset_id: str + model_id: str + run: PublishedRun | None = None + rejection: str | None = None + detail: str = '' + request_log: str = '' + + +@dataclass +class RunLogs: + """Run-log sources keyed by leaderboard ``datasetId``, plus join outcomes.""" + + sources: dict[str, RunLogSource] + outcomes: list[JoinOutcome] = field(default_factory=list) + unjoined: list[tuple[str, str]] = field(default_factory=list) + _runs: dict[tuple[str, str], PublishedRun | RunRejected] = field( + default_factory=dict + ) + + def join( + self, dataset_id: str, row: dict[str, Any], retrieved: float + ) -> JoinOutcome | None: + """Join one chosen score row to its run directory. + + Returns ``None`` when the set publishes no logs or no directory is + named after the row's ``modelId``. + """ + source = self.sources.get(dataset_id) + model_id = row.get('modelId') + if source is None or not isinstance(model_id, str): + return None + directory = source.root / model_id + if ( + not model_id + or model_id.startswith('.') + or '/' in model_id + or not directory.is_dir() + ): + self.unjoined.append((dataset_id, model_id)) + return None + key = (dataset_id, model_id) + if key not in self._runs: + try: + self._runs[key] = read_run(directory, source.history) + except RunRejected as exc: + self._runs[key] = exc + read = self._runs[key] + request_log = f'{source.dataset.repo_id}@{source.revision}/{model_id}' + if isinstance(read, RunRejected): + outcome = JoinOutcome( + dataset_id, + model_id, + rejection=read.reason, + detail=read.detail, + request_log=request_log, + ) + else: + reason = row_rejection(read, row, retrieved) + outcome = JoinOutcome( + dataset_id, + model_id, + run=None if reason else read, + rejection=reason, + request_log=request_log, + ) + self.outcomes.append(outcome) + return outcome + + +def rounds_to(value: float, stated: float | int) -> bool: + """Whether ``value`` rounds to ``stated`` at the precision ``stated`` spells out.""" + target = stated_decimal(stated) + half = Decimal(1).scaleb(target.as_tuple().exponent) / 2 + return abs(Decimal(repr(float(value))) - target) <= half + + +def _sums_to(value: float, stated: float, terms: int) -> bool: + """Whether a sum of ``terms`` task scores equals one summed in another order.""" + spread = terms * sys.float_info.epsilon * max(abs(value), abs(stated), 1.0) + return abs(value - stated) <= spread + + +def parse_timestamp(value: Any) -> datetime | None: + """Parse an epoch number or ISO-8601 string to an aware UTC datetime.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return datetime.fromtimestamp(float(value), tz=timezone.utc) + if not isinstance(value, str) or not value: + return None + try: + return datetime.fromtimestamp(float(value), tz=timezone.utc) + except ValueError: + pass + try: + parsed = datetime.fromisoformat(value.replace('Z', '+00:00')) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + + +def _attempt_cap(kwargs: dict[str, Any], where: str) -> tuple[str, int]: + stated = [key for key in CAP_KEYS if key in kwargs] + if not stated: + raise RunRejected('cap_absent', where) + if len(stated) > 1: + raise RunRejected('cap_ambiguous', f'{where}: {", ".join(stated)}') + value = kwargs[stated[0]] + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise RunRejected('cap_not_an_integer', f'{where}: {value!r}') + return stated[0], value + + +def read_history(repo_dir: Path, revision: str) -> dict[str, int]: + """Count the commits up to ``revision`` that touched each path. + + Reads commits and trees only, so it works in a blobless clone without + fetching file contents. + """ + log = _git( + '-C', + str(repo_dir), + 'log', + '--name-only', + '--no-renames', + '--pretty=format:', + revision, + '--', + ) + return dict(Counter(line for line in log.splitlines() if line)) + + +def read_run(directory: Path, history: dict[str, int]) -> PublishedRun: + """Reconstruct one run from its directory, or raise :class:`RunRejected`. + + Gates: no path the directory ever held was touched by a second commit + according to ``history``; every recorded attempt names this directory + as its ``test_id``, its file's task id, and its pair index; every recorded + request states one output cap under one key with one value; the task set, + each task's score and attempt count, the total attempts and the aggregate + all recompute from the task files to ``results.json``. + + A pair counts as solved when any attempt is correct, and a task scores the + fraction of its pairs solved. An attempt recorded as ``null`` rejects the + run: its request arguments, and so its cap, are not logged. + """ + results_path = directory / RESULTS_FILE + if not results_path.is_file(): + raise RunRejected('no_results_file') + try: + published = json.loads(results_path.read_text(encoding='utf-8')) + except ValueError as exc: + raise RunRejected('results_file_unreadable', str(exc)) from exc + task_results = ( + published.get('task_results') if isinstance(published, dict) else None + ) + if not isinstance(task_results, dict): + raise RunRejected('results_file_without_task_results') + + prefix = f'{directory.name}/' + rewritten = sorted( + path + for path, commits in history.items() + if path.startswith(prefix) and commits > 1 + ) + if rewritten: + raise RunRejected('run_rewritten', ', '.join(rewritten[:5])) + + task_paths = sorted( + path + for path in directory.glob('*.json') + if path.is_file() and path.name != RESULTS_FILE + ) + if not task_paths: + raise RunRejected('no_task_files') + + scores: dict[str, float] = {} + attempts_per_task: dict[str, int] = {} + cap: tuple[str, int] | None = None + temperatures: list[Any] = [] + latest_end: datetime | None = None + + for path in task_paths: + task_id = path.stem + try: + pairs = json.loads(path.read_text(encoding='utf-8')) + except ValueError as exc: + raise RunRejected( + 'task_file_unreadable', f'{path.name}: {exc}' + ) from exc + if not isinstance(pairs, list) or not pairs: + raise RunRejected('task_file_not_a_pair_list', path.name) + solved = [] + task_attempts = 0 + for index, pair in enumerate(pairs): + if not isinstance(pair, dict) or not pair: + raise RunRejected('task_file_not_a_pair_list', path.name) + correct = False + for key, attempt in pair.items(): + where = f'{path.name}: {key}' + if not _ATTEMPT_KEY.match(key): + raise RunRejected('attempt_key_unknown', where) + task_attempts += 1 + if not isinstance(attempt, dict): + raise RunRejected('attempt_not_recorded', where) + metadata = attempt.get('metadata') + if not isinstance(metadata, dict): + raise RunRejected('attempt_without_metadata', where) + if metadata.get('task_id') != task_id: + raise RunRejected( + 'task_id_mismatch', + f'{where}: {metadata.get("task_id")!r}', + ) + if metadata.get('test_id') != directory.name: + raise RunRejected( + 'test_id_mismatch', + f'{where}: {metadata.get("test_id")!r}', + ) + pair_index = metadata.get('pair_index') + if ( + isinstance(pair_index, bool) + or not isinstance(pair_index, int) + or pair_index != index + ): + raise RunRejected( + 'pair_index_mismatch', + f'{where}: {pair_index!r} at {index}', + ) + end = parse_timestamp(metadata.get('end_timestamp')) + if end is None: + raise RunRejected('attempt_without_end_time', where) + latest_end = end if latest_end is None else max(latest_end, end) + kwargs = metadata.get('kwargs') + if not isinstance(kwargs, dict): + raise RunRejected('attempt_without_kwargs', where) + attempt_cap = _attempt_cap(kwargs, where) + if cap is None: + cap = attempt_cap + elif attempt_cap[0] != cap[0]: + raise RunRejected( + 'cap_key_mixed', f'{cap[0]} vs {attempt_cap[0]}' + ) + elif attempt_cap[1] != cap[1]: + raise RunRejected( + 'cap_conflict', f'{cap[1]} vs {attempt_cap[1]}' + ) + temperatures.append(kwargs.get('temperature')) + if not isinstance(attempt.get('correct'), bool): + raise RunRejected('correct_not_recorded', where) + correct = correct or attempt['correct'] + solved.append(correct) + scores[task_id] = sum(solved) / len(solved) + attempts_per_task[task_id] = task_attempts + + if cap is None or latest_end is None: + raise RunRejected('no_recorded_attempts') + if set(scores) != set(task_results): + missing = len(set(task_results) - set(scores)) + extra = len(set(scores) - set(task_results)) + raise RunRejected( + 'task_set_mismatch', + f'{missing} not published as files, {extra} not in results.json', + ) + for task_id, score in sorted(scores.items()): + stated = task_results[task_id] + stated = stated if isinstance(stated, dict) else {} + stated_score = stated.get('score') + if not _is_number(stated_score) or not rounds_to(score, stated_score): + raise RunRejected( + 'task_score_mismatch', f'{task_id}: {stated_score!r}' + ) + stated_attempts = stated.get('attempts') + if ( + stated_attempts is not None + and stated_attempts != attempts_per_task[task_id] + ): + raise RunRejected( + 'task_attempts_mismatch', + f'{task_id}: {stated_attempts!r} vs {attempts_per_task[task_id]}', + ) + total_attempts = sum(attempts_per_task.values()) + for name, recomputed, terms in ( + ('total_tasks', len(scores), 0), + ('total_attempts', total_attempts, 0), + ('score', sum(scores.values()), len(scores)), + ): + stated = published.get(name) + matches = _is_number(stated) and ( + _sums_to(float(recomputed), float(stated), terms) + if terms + else float(recomputed) == float(stated) + ) + if not matches: + raise RunRejected( + 'run_score_mismatch', f'{name}: {stated!r} vs {recomputed}' + ) + + temperature = None + if ( + temperatures + and all(_is_number(t) for t in temperatures) + and len({float(t) for t in temperatures}) == 1 + ): + temperature = float(temperatures[0]) + + return PublishedRun( + directory=directory.name, + cap_key=cap[0], + cap_value=cap[1], + temperature=temperature, + score=sum(scores.values()) / len(scores), + avg_cost_per_task=published.get('avg_cost_per_task'), + latest_end=latest_end, + task_count=len(scores), + attempt_count=total_attempts, + ) + + +def row_rejection( + run: PublishedRun, row: dict[str, Any], retrieved: float +) -> str | None: + """Name the gate a leaderboard row fails against a run, or ``None``. + + The run's recomputed score must round to the row's ``score`` and its + ``avg_cost_per_task`` to the row's ``costPerTask``, each at the precision + the row spells out, and the run must have ended before retrieval. + """ + if not _is_number(row.get('score')): + return 'score_not_stated' + if not rounds_to(run.score, row['score']): + return 'score_mismatch' + if not _is_number(row.get('costPerTask')): + return 'cost_not_stated' + if not _is_number(run.avg_cost_per_task) or not rounds_to( + run.avg_cost_per_task, row['costPerTask'] + ): + return 'cost_mismatch' + if run.latest_end.timestamp() > retrieved: + return 'run_ended_after_retrieval' + return None + + +def _is_number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def _git_head(path: Path) -> str | None: + try: + completed = subprocess.run( + ['git', '-C', str(path), 'rev-parse', 'HEAD'], + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return None + return completed.stdout.strip() or None + + +def _is_git_root(path: Path) -> bool: + return (path / '.git').exists() + + +class RunLogsUnavailable(RuntimeError): + """Local run logs cannot be read under the same gates as a live run.""" + + +def load_history_file(path: Path, revision: str) -> dict[str, int]: + """Read a path-history file: ``{"revision": sha, "paths": {path: commits}}``. + + The file is what :func:`read_history` returns for ``revision``, saved as + JSON; its ``revision`` must equal the one the run logs are read at. + """ + try: + data = json.loads(Path(path).read_text(encoding='utf-8')) + except (OSError, ValueError) as exc: + raise RunLogsUnavailable(f'{path}: unreadable history file: {exc}') + if not isinstance(data, dict) or data.get('revision') != revision: + raise RunLogsUnavailable( + f'{path}: history is not for revision {revision}' + ) + paths = data.get('paths') + if not isinstance(paths, dict) or not all( + isinstance(k, str) and isinstance(v, int) for k, v in paths.items() + ): + raise RunLogsUnavailable(f'{path}: "paths" must map path to count') + return paths + + +def local_run_logs(runs_dir: Path, history_dir: Path | None = None) -> RunLogs: + """Use run logs already on disk under ``runs_dir/``. + + A subdirectory that is itself a git checkout supplies its own commit and + history. Otherwise the commit is taken to be the pinned one and its + history must come from ``history_dir``: a clone at ``/`` or a file + ``.json`` (see :func:`load_history_file`). A subdirectory with no + history raises :class:`RunLogsUnavailable`, so an offline run applies the + same gates as a live one. A missing subdirectory leaves that set unread. + """ + sources = {} + for dataset_id, dataset in RUN_DATASETS.items(): + root = Path(runs_dir) / dataset.subdir + if not root.is_dir(): + print( + f'ARC run logs: {root} not found; {dataset_id} not joined', + file=sys.stderr, + ) + continue + revision = dataset.revision + if _is_git_root(root): + revision = _git_head(root) or dataset.revision + history = read_history(root, revision) + elif history_dir is None: + raise RunLogsUnavailable( + f'{root} is not a git checkout, so its history is unknown; ' + 'pass --arc-history-dir with a clone or history file for ' + f'{dataset.subdir}' + ) + elif _is_git_root(Path(history_dir) / dataset.subdir): + history = read_history(Path(history_dir) / dataset.subdir, revision) + elif (Path(history_dir) / f'{dataset.subdir}.json').is_file(): + history = load_history_file( + Path(history_dir) / f'{dataset.subdir}.json', revision + ) + else: + raise RunLogsUnavailable( + f'{history_dir} holds neither a {dataset.subdir}/ clone nor ' + f'{dataset.subdir}.json' + ) + sources[dataset_id] = RunLogSource(dataset, root, revision, history) + return RunLogs(sources) + + +def _git(*args: str, cwd: Path | None = None, stdin: str | None = None) -> str: + env = {**os.environ, 'GIT_LFS_SKIP_SMUDGE': '1', 'GIT_TERMINAL_PROMPT': '0'} + try: + completed = subprocess.run( + ['git', *args], + cwd=cwd, + input=stdin, + capture_output=True, + text=True, + check=True, + env=env, + ) + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f'git {" ".join(args)} failed ({exc.returncode}): ' + f'{(exc.stderr or "").strip()}' + ) from exc + return completed.stdout + + +def fetch_run_logs( + model_ids: dict[str, set[str]], + work_dir: Path, + datasets: dict[str, RunDataset] | None = None, +) -> RunLogs: + """Clone each run dataset and check out its wanted directories at the + pinned commit. + + One full ``--no-checkout`` clone per dataset (a single pack, so a pinned + commit that is no longer HEAD still checks out), a cone-mode sparse + checkout of ``model_ids[datasetId]`` that exist at the commit, then a + detached checkout of it. The same clone's history backs the + rewritten-path gate. + """ + sources = {} + for dataset_id, dataset in (datasets or RUN_DATASETS).items(): + dest = Path(work_dir) / dataset.subdir + _git('clone', '--quiet', '--no-checkout', dataset.url, str(dest)) + present = set( + _git( + 'ls-tree', '-d', '--name-only', dataset.revision, cwd=dest + ).splitlines() + ) + wanted = sorted(model_ids.get(dataset_id, set()) & present) + _git( + 'sparse-checkout', + 'set', + '--cone', + '--stdin', + cwd=dest, + stdin=''.join(f'{name}\n' for name in wanted), + ) + _git('checkout', '--quiet', '--detach', dataset.revision, cwd=dest) + raw_capture.record_git_checkout( + dataset.url, + dest, + ref=dataset.revision, + label=f'{dataset.repo_id} run logs ({len(wanted)} directories)', + ) + sources[dataset_id] = RunLogSource( + dataset, + dest, + dataset.revision, + read_history(dest, dataset.revision), + ) + return RunLogs(sources) diff --git a/tests/data/arc_agi/history/v2.json b/tests/data/arc_agi/history/v2.json new file mode 100644 index 000000000..15bf4442a --- /dev/null +++ b/tests/data/arc_agi/history/v2.json @@ -0,0 +1,126 @@ +{ + "revision": "026789c1c12a4c34580a32e84dcaf5630d7e8f31", + "paths": { + "claude-opus-4-5-20251101-thinking-16k/0934a4d8.json": 1, + "claude-opus-4-5-20251101-thinking-16k/135a2760.json": 1, + "claude-opus-4-5-20251101-thinking-16k/136b0064.json": 1, + "claude-opus-4-5-20251101-thinking-16k/13e47133.json": 1, + "claude-opus-4-5-20251101-thinking-16k/142ca369.json": 1, + "claude-opus-4-5-20251101-thinking-16k/16b78196.json": 1, + "claude-opus-4-5-20251101-thinking-16k/16de56c4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/1818057f.json": 1, + "claude-opus-4-5-20251101-thinking-16k/195c6913.json": 1, + "claude-opus-4-5-20251101-thinking-16k/1ae2feb7.json": 1, + "claude-opus-4-5-20251101-thinking-16k/20270e3b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/20a9e565.json": 1, + "claude-opus-4-5-20251101-thinking-16k/21897d95.json": 1, + "claude-opus-4-5-20251101-thinking-16k/221dfab4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/247ef758.json": 1, + "claude-opus-4-5-20251101-thinking-16k/269e22fb.json": 1, + "claude-opus-4-5-20251101-thinking-16k/271d71e2.json": 1, + "claude-opus-4-5-20251101-thinking-16k/28a6681f.json": 1, + "claude-opus-4-5-20251101-thinking-16k/291dc1e1.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2b83f449.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2ba387bc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2c181942.json": 1, + "claude-opus-4-5-20251101-thinking-16k/2d0172a1.json": 1, + "claude-opus-4-5-20251101-thinking-16k/31f7f899.json": 1, + "claude-opus-4-5-20251101-thinking-16k/332f06d7.json": 1, + "claude-opus-4-5-20251101-thinking-16k/35ab12c3.json": 1, + "claude-opus-4-5-20251101-thinking-16k/36a08778.json": 1, + "claude-opus-4-5-20251101-thinking-16k/38007db0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/3a25b0d8.json": 1, + "claude-opus-4-5-20251101-thinking-16k/3dc255db.json": 1, + "claude-opus-4-5-20251101-thinking-16k/3e6067c3.json": 1, + "claude-opus-4-5-20251101-thinking-16k/409aa875.json": 1, + "claude-opus-4-5-20251101-thinking-16k/446ef5d2.json": 1, + "claude-opus-4-5-20251101-thinking-16k/45a5af55.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4a21e3da.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4c3d4a41.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4c416de3.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4c7dc4dd.json": 1, + "claude-opus-4-5-20251101-thinking-16k/4e34c42c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/53fb4810.json": 1, + "claude-opus-4-5-20251101-thinking-16k/5545f144.json": 1, + "claude-opus-4-5-20251101-thinking-16k/581f7754.json": 1, + "claude-opus-4-5-20251101-thinking-16k/58490d8a.json": 1, + "claude-opus-4-5-20251101-thinking-16k/58f5dbd5.json": 1, + "claude-opus-4-5-20251101-thinking-16k/5961cc34.json": 1, + "claude-opus-4-5-20251101-thinking-16k/5dbc8537.json": 1, + "claude-opus-4-5-20251101-thinking-16k/62593bfd.json": 1, + "claude-opus-4-5-20251101-thinking-16k/64efde09.json": 1, + "claude-opus-4-5-20251101-thinking-16k/65b59efc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/67e490f4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/6e453dd6.json": 1, + "claude-opus-4-5-20251101-thinking-16k/6e4f6532.json": 1, + "claude-opus-4-5-20251101-thinking-16k/6ffbe589.json": 1, + "claude-opus-4-5-20251101-thinking-16k/71e489b6.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7491f3cf.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7666fa5d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/78332cb0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b0280bc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b3084d4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b5033c1.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7b80bb43.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7c66cb00.json": 1, + "claude-opus-4-5-20251101-thinking-16k/7ed72f31.json": 1, + "claude-opus-4-5-20251101-thinking-16k/800d221b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/80a900e0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8698868d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/88bcf3b4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/88e364bc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/89565ca0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/898e7135.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8b7bacbf.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8b9c3697.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8e5c0c38.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8f215267.json": 1, + "claude-opus-4-5-20251101-thinking-16k/8f3a5a89.json": 1, + "claude-opus-4-5-20251101-thinking-16k/9385bd28.json": 1, + "claude-opus-4-5-20251101-thinking-16k/97d7923e.json": 1, + "claude-opus-4-5-20251101-thinking-16k/981571dc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/9aaea919.json": 1, + "claude-opus-4-5-20251101-thinking-16k/9bbf930d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a251c730.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a25697e4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a32d8b75.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a395ee82.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a47bf94d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/a6f40cea.json": 1, + "claude-opus-4-5-20251101-thinking-16k/aa4ec2a5.json": 1, + "claude-opus-4-5-20251101-thinking-16k/abc82100.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b0039139.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b10624e5.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b5ca7ac4.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b6f77b65.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b99e7126.json": 1, + "claude-opus-4-5-20251101-thinking-16k/b9e38dc0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/bf45cf4b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/c4d067a0.json": 1, + "claude-opus-4-5-20251101-thinking-16k/c7f57c3e.json": 1, + "claude-opus-4-5-20251101-thinking-16k/cb2d8a2c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/cbebaa4b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/d35bdbdc.json": 1, + "claude-opus-4-5-20251101-thinking-16k/d59b0160.json": 1, + "claude-opus-4-5-20251101-thinking-16k/d8e07eb2.json": 1, + "claude-opus-4-5-20251101-thinking-16k/da515329.json": 1, + "claude-opus-4-5-20251101-thinking-16k/db0c5428.json": 1, + "claude-opus-4-5-20251101-thinking-16k/db695cfb.json": 1, + "claude-opus-4-5-20251101-thinking-16k/dbff022c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/dd6b8c4b.json": 1, + "claude-opus-4-5-20251101-thinking-16k/de809cff.json": 1, + "claude-opus-4-5-20251101-thinking-16k/dfadab01.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e12f9a14.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e3721c99.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e376de54.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e8686506.json": 1, + "claude-opus-4-5-20251101-thinking-16k/e87109e9.json": 1, + "claude-opus-4-5-20251101-thinking-16k/edb79dae.json": 1, + "claude-opus-4-5-20251101-thinking-16k/eee78d87.json": 1, + "claude-opus-4-5-20251101-thinking-16k/f560132c.json": 1, + "claude-opus-4-5-20251101-thinking-16k/f931b4a8.json": 1, + "claude-opus-4-5-20251101-thinking-16k/faa9f03d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/fc7cae8d.json": 1, + "claude-opus-4-5-20251101-thinking-16k/results.json": 1 + } +} diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/28a6681f.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/28a6681f.json new file mode 100644 index 000000000..8d0b4b003 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/28a6681f.json @@ -0,0 +1,320 @@ +[ + { + "attempt_1": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5, + 0, + 5 + ], + [ + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5 + ], + [ + 3, + 3, + 0, + 1, + 1, + 1, + 5, + 5, + 0, + 5 + ], + [ + 3, + 3, + 3, + 1, + 1, + 5, + 5, + 5, + 0, + 5 + ], + [ + 3, + 3, + 3, + 3, + 5, + 5, + 5, + 5, + 0, + 5 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T16:13:14.383973+00:00", + "end_timestamp": "2025-11-21T16:15:31.704129+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2297, + "completion_tokens": 11646, + "total_tokens": 13943, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 11646, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.011485, + "completion_cost": 0.29115, + "reasoning_cost": 0.0, + "total_cost": 0.30263500000000004 + }, + "task_id": "28a6681f", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": false + }, + "attempt_2": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 9, + 9, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 9, + 9, + 9, + 0, + 0, + 5 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5, + 1, + 5 + ], + [ + 3, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 5 + ], + [ + 3, + 3, + 0, + 0, + 0, + 0, + 5, + 5, + 1, + 5 + ], + [ + 3, + 3, + 3, + 0, + 0, + 5, + 5, + 5, + 1, + 5 + ], + [ + 3, + 3, + 3, + 3, + 5, + 5, + 5, + 5, + 1, + 5 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T16:15:31.705226+00:00", + "end_timestamp": "2025-11-21T16:17:25.495164+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2297, + "completion_tokens": 10390, + "total_tokens": 12687, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 10390, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.011485, + "completion_cost": 0.25975000000000004, + "reasoning_cost": 0.0, + "total_cost": 0.27123500000000006 + }, + "task_id": "28a6681f", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": false + } + } +] diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/58490d8a.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/58490d8a.json new file mode 100644 index 000000000..c866e2bb5 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/58490d8a.json @@ -0,0 +1,314 @@ +[ + { + "attempt_1": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 2, + 0, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:14:14.600064+00:00", + "end_timestamp": "2025-11-21T17:15:27.470540+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 6488, + "completion_tokens": 7239, + "total_tokens": 13727, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7239, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.032440000000000004, + "completion_cost": 0.180975, + "reasoning_cost": 0.0, + "total_cost": 0.213415 + }, + "task_id": "58490d8a", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": true + }, + "attempt_2": { + "answer": [ + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 1, + 0, + 1, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 2, + 0, + 2, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 8, + 0, + 0, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + [ + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0, + 3, + 0 + ], + [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:15:27.474466+00:00", + "end_timestamp": "2025-11-21T17:16:52.086340+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 6488, + "completion_tokens": 7541, + "total_tokens": 14029, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7541, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.032440000000000004, + "completion_cost": 0.188525, + "reasoning_cost": 0.0, + "total_cost": 0.220965 + }, + "task_id": "58490d8a", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": true + } + } +] diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/7b5033c1.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/7b5033c1.json new file mode 100644 index 000000000..3d81da5a5 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/7b5033c1.json @@ -0,0 +1,230 @@ +[ + { + "attempt_1": { + "answer": [ + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 3 + ], + [ + 3 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:32:14.686501+00:00", + "end_timestamp": "2025-11-21T17:33:34.203070+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 7453, + "total_tokens": 10074, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7453, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.013105, + "completion_cost": 0.18632500000000002, + "reasoning_cost": 0.0, + "total_cost": 0.19943000000000002 + }, + "task_id": "7b5033c1", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": true + }, + "attempt_2": { + "answer": [ + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 1 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 3 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 8 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ] + ], + "metadata": { + "model": "claude-opus-4-5-20251101", + "provider": "anthropic", + "start_timestamp": "2025-11-21T17:33:34.208119+00:00", + "end_timestamp": "2025-11-21T17:34:59.171003+00:00", + "kwargs": { + "max_tokens": 64000, + "stream": true, + "thinking": { + "type": "enabled", + "budget_tokens": 16000 + } + }, + "usage": { + "prompt_tokens": 2621, + "completion_tokens": 7616, + "total_tokens": 10237, + "completion_tokens_details": { + "reasoning_tokens": 0, + "accepted_prediction_tokens": 7616, + "rejected_prediction_tokens": 0 + } + }, + "cost": { + "prompt_cost": 0.013105, + "completion_cost": 0.1904, + "reasoning_cost": 0.0, + "total_cost": 0.20350500000000002 + }, + "task_id": "7b5033c1", + "pair_index": 0, + "test_id": "claude-opus-4-5-20251101-thinking-16k" + }, + "correct": false + } + } +] diff --git a/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/results.json b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/results.json new file mode 100644 index 000000000..5acc494d3 --- /dev/null +++ b/tests/data/arc_agi/v2/claude-opus-4-5-20251101-thinking-16k/results.json @@ -0,0 +1,41 @@ +{ + "score": 2.0, + "total_tasks": 3, + "total_cost": 1.4111850000000001, + "total_attempts": 6, + "avg_cost_per_task": 0.47039500000000006, + "avg_cost_per_attempt": 0.23519750000000003, + "avg_output_tokens_per_task": 17295.0, + "avg_total_tokens_per_task": 24899.0, + "avg_duration_per_task": 191.02396566666667, + "task_results": { + "28a6681f": { + "score": 0.0, + "cost": 0.5738700000000001, + "attempts": 2, + "output_tokens": 22036, + "total_tokens": 26630, + "duration": 251.110094, + "num_attempts_with_empty_list": 0 + }, + "58490d8a": { + "score": 1.0, + "cost": 0.43438, + "attempts": 2, + "output_tokens": 14780, + "total_tokens": 27756, + "duration": 157.48235, + "num_attempts_with_empty_list": 0 + }, + "7b5033c1": { + "score": 1.0, + "cost": 0.40293500000000004, + "attempts": 2, + "output_tokens": 15069, + "total_tokens": 20311, + "duration": 164.479453, + "num_attempts_with_empty_list": 0 + } + }, + "num_attempts_with_empty_list": 0 +} diff --git a/tests/test_arc_agi_adapter.py b/tests/test_arc_agi_adapter.py index 806377207..5e2f85afc 100644 --- a/tests/test_arc_agi_adapter.py +++ b/tests/test_arc_agi_adapter.py @@ -1,12 +1,29 @@ -"""Unit tests for the ARC Prize leaderboard adapter.""" +"""Unit tests for the ARC Prize leaderboard adapter. + +The run-log fixture under ``tests/data/arc_agi/v2/`` is three task files +(``7b5033c1``, ``58490d8a``, ``28a6681f``) from the directory +``claude-opus-4-5-20251101-thinking-16k`` of the Hugging Face dataset +``arcprize/arc_agi_v2_public_eval`` at commit +``026789c1c12a4c34580a32e84dcaf5630d7e8f31``, with each attempt's +``choices`` and ``reasoning_summary`` removed. Its ``results.json`` keeps +those three ``task_results`` entries verbatim and recomputes the run totals +from them (score 2.0 of 3 tasks, average cost per task 0.470395). +``tests/data/arc_agi/history/v2.json`` is that directory's path history at +the same commit (``git log --name-only --no-renames`` over a blobless clone, +restricted to the directory): every path committed once. +""" from __future__ import annotations import json +import shutil +import subprocess +from pathlib import Path import pytest -from every_eval_ever.adapters.arc_agi import adapter +from every_eval_ever.adapters.arc_agi import adapter, run_logs +from every_eval_ever.adapters.arc_agi.run_logs import StatedFloat from every_eval_ever.eval_types import EvaluationLog @@ -258,6 +275,7 @@ def test_run_writes_records_and_replays_offline(tmp_path): str(payload_path), '--output-dir', str(out_dir), + '--no-run-logs', ] ) written = adapter.run(args) @@ -267,3 +285,554 @@ def test_run_writes_records_and_replays_offline(tmp_path): record = json.loads(files[0].read_text(encoding='utf-8')) assert record['schema_version'] assert record['evaluation_id'].startswith('arc-agi/') + + +RUN_FIXTURE = Path(__file__).parent / 'data' / 'arc_agi' +RUN_ID = 'claude-opus-4-5-20251101-thinking-16k' +ALIAS_ID = 'anthropic-claude-opus-4-5-20251101-thinking-16k' +V2 = run_logs.RUN_DATASETS['v2_Public_Eval'] +RETRIEVED = '1790000000.0' +FIXTURE_HISTORY = run_logs.load_history_file( + RUN_FIXTURE / 'history' / 'v2.json', V2.revision +) + + +def run_log_payload(**v2_row: object) -> dict: + row = { + 'datasetId': 'v2_Public_Eval', + 'modelId': RUN_ID, + 'score': 0.6667, + 'costPerTask': 0.47, + 'display': True, + } + row.update(v2_row) + return { + 'datasets': [], + 'providers': [{'id': 'Anthropic', 'displayName': 'Anthropic'}], + 'models': [ + {'id': model_id, 'providerId': 'Anthropic'} + for model_id in (RUN_ID, ALIAS_ID) + ], + 'evaluations': [ + row, + { + 'datasetId': 'v1_Public_Eval', + 'modelId': RUN_ID, + 'score': 0.6667, + 'costPerTask': 0.47, + 'display': True, + }, + { + 'datasetId': 'v2_Semi_Private', + 'modelId': RUN_ID, + 'score': 0.6667, + 'costPerTask': 0.47, + 'display': True, + }, + ], + } + + +@pytest.fixture +def runs_dir(tmp_path: Path) -> Path: + root = tmp_path / 'runs' + shutil.copytree(RUN_FIXTURE, root) + return root + + +def run_dir(root: Path, name: str = RUN_ID) -> Path: + return root / 'v2' / name + + +def edit_attempts(directory: Path, edit, task_id: str | None = None) -> None: + """Apply ``edit(metadata, task_id, attempt_key)`` to every recorded attempt.""" + for path in sorted(directory.glob('*.json')): + if path.name == 'results.json' or task_id not in (None, path.stem): + continue + pairs = json.loads(path.read_text(encoding='utf-8')) + for pair in pairs: + for key, attempt in pair.items(): + if attempt is not None: + edit(attempt['metadata'], path.stem, key) + path.write_text(json.dumps(pairs), encoding='utf-8') + + +def convert_with_logs( + payload: dict, + root: Path, + history: dict[str, int] = FIXTURE_HISTORY, + retrieved: str = RETRIEVED, +) -> tuple[EvaluationLog, run_logs.RunLogs]: + logs = run_logs.RunLogs( + { + dataset_id: run_logs.RunLogSource( + dataset, root / dataset.subdir, dataset.revision, history + ) + for dataset_id, dataset in run_logs.RUN_DATASETS.items() + } + ) + result = adapter.convert_logs( + payload, retrieved_timestamp=retrieved, run_logs=logs + ) + result.raise_if_incomplete() + [(log, _, _)] = result.records + return log, logs + + +def result_by_id(log: EvaluationLog, result_id: str): + return next( + r for r in log.evaluation_results if r.evaluation_result_id == result_id + ) + + +def test_accepted_run_fills_the_score_result(runs_dir): + log, logs = convert_with_logs(run_log_payload(), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + config = score.generation_config + assert config.generation_args.max_tokens == 64000 + assert config.generation_args.temperature is None + assert config.additional_details == { + 'cap_key': 'max_tokens', + 'request_log': f'{V2.repo_id}@{V2.revision}/{RUN_ID}', + } + assert score.source_data.additional_details['scored_tasks'] == '3' + assert [(o.dataset_id, o.rejection) for o in logs.outcomes] == [ + ('v2_Public_Eval', None) + ] + + +def test_cost_result_and_other_sets_are_untouched(runs_dir): + log, _ = convert_with_logs(run_log_payload(), runs_dir) + for result_id in ( + 'v2_Public_Eval::cost_per_task', + 'v1_Public_Eval::score', + 'v1_Public_Eval::cost_per_task', + 'v2_Semi_Private::score', + 'v2_Semi_Private::cost_per_task', + ): + result = result_by_id(log, result_id) + assert result.generation_config is None, result_id + assert 'scored_tasks' not in result.source_data.additional_details + + +def test_v2_run_never_fills_v1_even_when_v1_has_no_directory(runs_dir): + (runs_dir / 'v1').mkdir() + log, logs = convert_with_logs(run_log_payload(), runs_dir) + assert result_by_id(log, 'v1_Public_Eval::score').generation_config is None + assert {o.dataset_id for o in logs.outcomes} == {'v2_Public_Eval'} + assert logs.unjoined == [('v1_Public_Eval', RUN_ID)] + assert adapter.run_log_report(logs)['gate_counts'] == { + 'filled': 1, + 'no_run_directory': 1, + } + + +def _set_cap(metadata, task_id, key): + if (task_id, key) == ('28a6681f', 'attempt_2'): + metadata['kwargs']['max_tokens'] = 32000 + + +def _mix_cap_key(metadata, task_id, key): + if (task_id, key) == ('28a6681f', 'attempt_2'): + metadata['kwargs']['max_output_tokens'] = metadata['kwargs'].pop( + 'max_tokens' + ) + + +def _wrong_test_id(metadata, task_id, key): + if (task_id, key) == ('58490d8a', 'attempt_1'): + metadata['test_id'] = 'some-other-run' + + +@pytest.mark.parametrize( + ('row', 'mutate', 'gate'), + [ + ({'score': 0.6}, None, 'score_mismatch'), + ({'costPerTask': 0.48}, None, 'cost_mismatch'), + ({}, lambda d: edit_attempts(d, _set_cap), 'cap_conflict'), + ({}, lambda d: edit_attempts(d, _mix_cap_key), 'cap_key_mixed'), + ({}, lambda d: (d / 'results.json').unlink(), 'no_results_file'), + ({}, lambda d: edit_attempts(d, _wrong_test_id), 'test_id_mismatch'), + ], + ids=[ + 'score', + 'cost', + 'cap_conflict', + 'mixed_cap_keys', + 'no_results', + 'test_id', + ], +) +def test_failed_gate_leaves_the_result_unfilled(runs_dir, row, mutate, gate): + if mutate is not None: + mutate(run_dir(runs_dir)) + log, _ = convert_with_logs(run_log_payload(**row), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config.generation_args is None + assert score.generation_config.additional_details == { + 'request_args_unavailable': gate, + 'request_log': f'{V2.repo_id}@{V2.revision}/{RUN_ID}', + } + assert 'scored_tasks' not in score.source_data.additional_details + cost = result_by_id(log, 'v2_Public_Eval::cost_per_task') + assert cost.generation_config is None + + +def test_score_is_compared_at_the_row_precision(runs_dir): + log, _ = convert_with_logs(run_log_payload(score=0.667), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config.generation_args.max_tokens == 64000 + + +def test_run_ending_after_retrieval_is_rejected(runs_dir): + log, _ = convert_with_logs( + run_log_payload(), runs_dir, retrieved='1700000000.0' + ) + details = result_by_id( + log, 'v2_Public_Eval::score' + ).generation_config.additional_details + assert details['request_args_unavailable'] == 'run_ended_after_retrieval' + + +def test_rewritten_path_in_history_is_rejected(runs_dir): + history = {f'{RUN_ID}/results.json': 1, f'{RUN_ID}/7b5033c1.json': 2} + log, _ = convert_with_logs(run_log_payload(), runs_dir, history=history) + details = result_by_id( + log, 'v2_Public_Eval::score' + ).generation_config.additional_details + assert details['request_args_unavailable'] == 'run_rewritten' + + +def test_temperature_only_when_every_attempt_states_it(runs_dir): + def set_temperature(metadata, task_id, key): + metadata['kwargs']['temperature'] = 1.0 + + edit_attempts(run_dir(runs_dir), set_temperature) + log, _ = convert_with_logs(run_log_payload(), runs_dir) + args = result_by_id(log, 'v2_Public_Eval::score').generation_config + assert args.generation_args.temperature == 1.0 + + def drop_one(metadata, task_id, key): + if key == 'attempt_2': + metadata['kwargs'].pop('temperature') + + edit_attempts(run_dir(runs_dir), drop_one, task_id='28a6681f') + log, _ = convert_with_logs(run_log_payload(), runs_dir) + args = result_by_id(log, 'v2_Public_Eval::score').generation_config + assert args.generation_args.temperature is None + assert args.generation_args.max_tokens == 64000 + + +def test_unrecorded_attempt_rejects_the_run(runs_dir): + path = run_dir(runs_dir) / '28a6681f.json' + pairs = json.loads(path.read_text(encoding='utf-8')) + pairs[0]['attempt_2'] = None + path.write_text(json.dumps(pairs), encoding='utf-8') + log, _ = convert_with_logs(run_log_payload(), runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config.generation_args is None + assert ( + score.generation_config.additional_details['request_args_unavailable'] + == 'attempt_not_recorded' + ) + assert 'scored_tasks' not in score.source_data.additional_details + + +@pytest.mark.parametrize( + ('stated', 'filled'), [('0.470', True), ('0.4700', False)] +) +def test_cost_is_compared_at_its_written_precision(runs_dir, stated, filled): + results_path = run_dir(runs_dir) / 'results.json' + results = json.loads(results_path.read_text(encoding='utf-8')) + results['avg_cost_per_task'] = 0.4704 + results_path.write_text(json.dumps(results), encoding='utf-8') + log, _ = convert_with_logs( + run_log_payload(costPerTask=StatedFloat(stated)), runs_dir + ) + config = result_by_id(log, 'v2_Public_Eval::score').generation_config + if filled: + assert config.generation_args.max_tokens == 64000 + else: + assert config.generation_args is None + assert config.additional_details['request_args_unavailable'] == ( + 'cost_mismatch' + ) + + +def test_payload_parsing_keeps_written_precision(tmp_path): + texts = { + 'evaluations': '[{"modelId": "m", "score": 0.4700, "costPerTask": 1}]', + 'models': '[]', + 'providers': '[]', + 'datasets': '[]', + } + path = tmp_path / 'raw.json' + path.write_text( + adapter.combined_payload_text(texts, 'https://example.test', '1.0'), + encoding='utf-8', + ) + [row] = adapter.load_payload_file(path)['evaluations'] + assert row['score'].text == '0.4700' + assert adapter.parse_payload_texts(texts)['evaluations'][0][ + 'score' + ].text == ('0.4700') + + +def test_join_uses_the_chosen_rows_model_id(runs_dir): + payload = run_log_payload() + alias_row = dict(payload['evaluations'][0], modelId=ALIAS_ID) + payload['evaluations'].append(alias_row) + log, _ = convert_with_logs(payload, runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.score_details.details['raw_model_id'] == RUN_ID + assert score.generation_config.generation_args.max_tokens == 64000 + + run_dir(runs_dir).rename(run_dir(runs_dir, ALIAS_ID)) + edit_attempts( + run_dir(runs_dir, ALIAS_ID), + lambda metadata, task_id, key: metadata.update(test_id=ALIAS_ID), + ) + log, logs = convert_with_logs(payload, runs_dir) + score = result_by_id(log, 'v2_Public_Eval::score') + assert score.generation_config is None + assert logs.outcomes == [] + + +def test_output_is_deterministic_for_a_fixed_directory(runs_dir): + first, _ = convert_with_logs(run_log_payload(), runs_dir) + second, _ = convert_with_logs(run_log_payload(), runs_dir) + assert first.model_dump_json() == second.model_dump_json() + + +def test_run_reads_a_local_runs_dir(tmp_path, runs_dir): + payload_path = tmp_path / 'payload.json' + payload_path.write_text(json.dumps(run_log_payload()), encoding='utf-8') + out_dir = tmp_path / 'data' / 'arc-agi' + args = adapter.parse_args( + [ + '--input-json', + str(payload_path), + '--output-dir', + str(out_dir), + '--arc-runs-dir', + str(runs_dir), + '--arc-history-dir', + str(runs_dir / 'history'), + ] + ) + assert adapter.run(args) == 1 + report = json.loads( + (tmp_path / 'adapter_reports' / 'arc-agi_failures.json').read_text( + encoding='utf-8' + ) + ) + assert report['failed_record_count'] == 0 + evidence = report['run_log_evidence'] + assert evidence['sources'] == { + 'v2_Public_Eval': {'dataset': V2.repo_id, 'revision': V2.revision} + } + assert evidence['gate_counts'] == {'filled': 1} + assert evidence['regressions'] == [] + assert evidence['unavailable'] == [] + assert evidence['filled'] == [ + { + 'model_id': RUN_ID, + 'dataset_id': 'v2_Public_Eval', + 'directory': RUN_ID, + 'cap_key': 'max_tokens', + 'max_tokens': 64000, + 'temperature': None, + 'scored_tasks': 3, + } + ] + [path] = out_dir.glob('*/*/*.json') + record = json.loads(path.read_text(encoding='utf-8')) + [score] = [ + r + for r in record['evaluation_results'] + if r['evaluation_result_id'] == 'v2_Public_Eval::score' + ] + assert score['generation_config']['generation_args'] == { + 'max_tokens': 64000 + } + + +def test_public_model_ids_cover_only_sets_with_run_logs(): + payload = run_log_payload() + payload['evaluations'].append( + {'datasetId': 'v2_Public_Eval', 'modelId': 'hidden', 'display': False} + ) + assert adapter.public_model_ids(payload) == { + 'v1_Public_Eval': {RUN_ID}, + 'v2_Public_Eval': {RUN_ID}, + } + + +def _git(*args: str, cwd: Path) -> str: + return subprocess.run( + [ + 'git', + '-c', + 'user.name=t', + '-c', + 'user.email=t@example.com', + *args, + ], + cwd=cwd, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +@pytest.fixture +def dataset_repo(tmp_path: Path) -> Path: + if shutil.which('git') is None: + pytest.skip('git not available') + repo = tmp_path / 'origin' + shutil.copytree(RUN_FIXTURE / 'v2', repo) + other = repo / 'unrelated-run' + other.mkdir() + (other / 'results.json').write_text('{}', encoding='utf-8') + _git('init', '--quiet', cwd=repo) + _git('add', '.', cwd=repo) + _git('commit', '--quiet', '-m', 'publish', cwd=repo) + return repo + + +def test_fetch_clones_and_checks_out_wanted_dirs_at_the_pinned_revision( + tmp_path, dataset_repo, monkeypatch +): + revision = _git('rev-parse', 'HEAD', cwd=dataset_repo) + path = dataset_repo / RUN_ID / '7b5033c1.json' + path.write_text(path.read_text(encoding='utf-8') + '\n', encoding='utf-8') + (dataset_repo / 'late.txt').write_text('later', encoding='utf-8') + _git('add', '.', cwd=dataset_repo) + _git('commit', '--quiet', '-m', 'later', cwd=dataset_repo) + monkeypatch.setattr( + run_logs.RunDataset, + 'url', + property(lambda self: dataset_repo.as_uri()), + ) + dataset = run_logs.RunDataset('v2_Public_Eval', 'test/v2', revision, 'v2') + logs = run_logs.fetch_run_logs( + {'v2_Public_Eval': {RUN_ID, 'not-published'}}, + tmp_path / 'work', + datasets={'v2_Public_Eval': dataset}, + ) + source = logs.sources['v2_Public_Eval'] + assert source.revision == revision + assert _git('rev-parse', 'HEAD', cwd=source.root) == revision + assert (source.root / RUN_ID / 'results.json').is_file() + assert not (source.root / 'unrelated-run').exists() + assert not (source.root / 'late.txt').exists() + assert source.history == { + f'{RUN_ID}/{name}': 1 + for name in sorted(p.name for p in (dataset_repo / RUN_ID).iterdir()) + } | {'unrelated-run/results.json': 1} + + +def test_git_failure_carries_stderr(tmp_path): + if shutil.which('git') is None: + pytest.skip('git not available') + with pytest.raises(RuntimeError, match='not a git repository|fatal'): + run_logs.read_history(tmp_path, 'HEAD') + + +def test_local_git_checkout_supplies_revision_and_history( + tmp_path, dataset_repo +): + runs = tmp_path / 'local' + runs.mkdir() + dataset_repo.rename(runs / 'v2') + path = runs / 'v2' / RUN_ID / '7b5033c1.json' + path.write_text(path.read_text(encoding='utf-8') + '\n', encoding='utf-8') + _git('commit', '--quiet', '-am', 'rewrite', cwd=runs / 'v2') + logs = run_logs.local_run_logs(runs) + source = logs.sources['v2_Public_Eval'] + assert source.revision == _git('rev-parse', 'HEAD', cwd=runs / 'v2') + assert source.history[f'{RUN_ID}/7b5033c1.json'] == 2 + result = adapter.convert_logs( + run_log_payload(), retrieved_timestamp=RETRIEVED, run_logs=logs + ) + [(log, _, _)] = result.records + details = result_by_id( + log, 'v2_Public_Eval::score' + ).generation_config.additional_details + assert details['request_args_unavailable'] == 'run_rewritten' + + +def test_gate_rejection_is_reported_without_failing_the_run( + tmp_path, runs_dir, capsys +): + payload_path = tmp_path / 'payload.json' + payload_path.write_text( + json.dumps(run_log_payload(score=0.6)), encoding='utf-8' + ) + args = adapter.parse_args( + [ + '--input-json', + str(payload_path), + '--output-dir', + str(tmp_path / 'data' / 'arc-agi'), + '--arc-runs-dir', + str(runs_dir), + '--arc-history-dir', + str(runs_dir / 'history'), + ] + ) + assert adapter.run(args) == 1 + evidence = json.loads( + (tmp_path / 'adapter_reports' / 'arc-agi_failures.json').read_text( + encoding='utf-8' + ) + )['run_log_evidence'] + assert evidence['gate_counts'] == {'score_mismatch': 1} + assert evidence['unavailable'] == [] + assert 'ARC run-log regression: ' in capsys.readouterr().err + assert evidence['regressions'] == [ + { + 'model_id': RUN_ID, + 'dataset_id': 'v2_Public_Eval', + 'directory': RUN_ID, + 'gate': 'score_mismatch', + 'detail': '', + } + ] + assert evidence['filled'] == [] + + +def test_plain_runs_dir_without_history_fails(tmp_path, runs_dir): + payload_path = tmp_path / 'payload.json' + payload_path.write_text(json.dumps(run_log_payload()), encoding='utf-8') + args = adapter.parse_args( + [ + '--input-json', + str(payload_path), + '--output-dir', + str(tmp_path / 'data' / 'arc-agi'), + '--arc-runs-dir', + str(runs_dir), + ] + ) + with pytest.raises(SystemExit, match='--arc-history-dir'): + adapter.run(args) + assert not (tmp_path / 'data').exists() + + +def test_history_file_must_match_the_revision(tmp_path): + path = tmp_path / 'v2.json' + path.write_text( + json.dumps({'revision': 'other', 'paths': {}}), encoding='utf-8' + ) + with pytest.raises(run_logs.RunLogsUnavailable, match='revision'): + run_logs.load_history_file(path, V2.revision) + + +def test_permanent_rejections_are_listed_as_unavailable(tmp_path, runs_dir): + (run_dir(runs_dir) / 'results.json').unlink() + log, logs = convert_with_logs(run_log_payload(), runs_dir) + report = adapter.run_log_report(logs) + assert list(report)[2:4] == ['regressions', 'unavailable'] + assert report['regressions'] == [] + assert [e['gate'] for e in report['unavailable']] == ['no_results_file'] From 5f720061c14d74f8e81a9c4a03fa591ddd903ac0 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 05/16] hle: remove the fixed temperature 0.0; read temperature from the row notes temperature was 0.0 on every result. Scale's statement is "temperature 0.0 when configurable", and no row says whether its model was configurable. temperature is now null unless the row note matches one of two sentence forms: "Sampled at temperature = , top_p = ." or "Sampled at temperature ". The statement is written as source_generation_policy and the note as source_row_note. The judge prompt text no longer names a temperature. Co-Authored-By: Claude Fable 5.1 --- every_eval_ever/adapters/hle/adapter.py | 67 ++++++++++++---- tests/test_hle_adapter.py | 101 ++++++++++++++++++++++-- 2 files changed, 146 insertions(+), 22 deletions(-) diff --git a/every_eval_ever/adapters/hle/adapter.py b/every_eval_ever/adapters/hle/adapter.py index 2dea092ce..df08619da 100644 --- a/every_eval_ever/adapters/hle/adapter.py +++ b/every_eval_ever/adapters/hle/adapter.py @@ -85,12 +85,24 @@ DEFAULT_OUTPUT_DIR = 'data/hle' JUDGE_MODEL_ID = 'openai/o3-mini-2025-01-31' JUDGE_PROMPT_DESCRIPTION = ( - "Scale SEAL evaluates Humanity's Last Exam at temperature 0.0. The " - 'judge model o3-mini-2025-01-31 acts as an automatic answer extractor ' + 'The judge model o3-mini-2025-01-31 acts as an automatic answer extractor ' 'and grader against ground-truth solutions for each of the 2,500 ' 'frozen questions.' ) DATASET_TOTAL_QUESTIONS = 2500 +SOURCE_GENERATION_POLICY = ( + 'Each model on the leaderboard is evaluated on all public questions of ' + "Humanity's Last Exam with temperature 0.0 when configurable or stated " + 'otherwise.' +) +_NUMBER = r'\d+(?:\.\d+)?' +ROW_NOTE_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile( + rf'Sampled at temperature = (?P{_NUMBER}), ' + rf'top_p = (?P{_NUMBER})\.' + ), + re.compile(rf'Sampled at temperature (?P{_NUMBER})\.?'), +) # Map the leaderboard's lowercase company slug to the canonical developer # slug used elsewhere in the EEE data tree (matches helpers/developer.py @@ -143,7 +155,7 @@ def created_at(self) -> str | None: return str(value) if value else None @property - def contamination_message(self) -> str | None: + def row_note(self) -> str | None: value = self.raw.get('contaminationMessage') return str(value) if value else None @@ -305,13 +317,39 @@ def make_judge_model_info() -> ModelInfo: ) -def make_generation_config() -> GenerationConfig: - """Per Scale SEAL's published methodology, the model under evaluation - runs at temperature 0.0. Per the schema, that belongs on - ``EvaluationResult.generation_config.generation_args.temperature`` - (eval.schema.json L400) — not in source/metric ``additional_details``. +def parse_row_note(note: str) -> GenerationArgs | None: + """Return the sampling settings stated by a per-row note. + + Only notes that fully match one of ``ROW_NOTE_PATTERNS`` yield values; + any other note yields ``None``. + """ + stripped = note.strip() + for pattern in ROW_NOTE_PATTERNS: + match = pattern.fullmatch(stripped) + if match is None: + continue + values = { + key: float(value) + for key, value in match.groupdict().items() + if value is not None + } + return GenerationArgs(**values) + return None + + +def make_generation_config(row: LeaderboardRow) -> GenerationConfig | None: + """Build the generation config from the row's own note, if any. + + Sampling values are set only when the note states them in a reviewed + sentence form; the note itself is kept verbatim as ``source_row_note``. """ - return GenerationConfig(generation_args=GenerationArgs(temperature=0.0)) + note = row.row_note + if note is None: + return None + return GenerationConfig( + generation_args=parse_row_note(note), + additional_details={'source_row_note': note}, + ) def make_source_data() -> SourceDataUrl: @@ -383,7 +421,7 @@ def make_accuracy_result( }, ), score_details=ScoreDetails(**score_details_kwargs), - generation_config=make_generation_config(), + generation_config=make_generation_config(row), ) @@ -417,7 +455,7 @@ def make_calibration_result( ), ), score_details=ScoreDetails(score=row.calibration_error), - generation_config=make_generation_config(), + generation_config=make_generation_config(row), ) @@ -441,9 +479,9 @@ def make_log( 'raw_model_display_name': row.model_display, 'rank': str(row.raw.get('rank')), } - contamination = row.contamination_message - if contamination: - additional_details['contamination_message'] = contamination + note = row.row_note + if note: + additional_details['source_row_note'] = note deprecated = row.raw.get('deprecated') if deprecated is not None: additional_details['deprecated'] = 'true' if deprecated else 'false' @@ -464,6 +502,7 @@ def make_log( 'leaderboard_url': LEADERBOARD_URL, 'hle_home_url': HLE_HOME_URL, 'hle_dataset_hf_url': HLE_DATASET_HF_URL, + 'source_generation_policy': SOURCE_GENERATION_POLICY, }, ), eval_library=EvalLibrary( diff --git a/tests/test_hle_adapter.py b/tests/test_hle_adapter.py index 7d33e7a78..b85091cae 100644 --- a/tests/test_hle_adapter.py +++ b/tests/test_hle_adapter.py @@ -159,17 +159,102 @@ def test_metric_config_uses_percent_scale_with_judge(): assert judge.model_info.id == adapter.JUDGE_MODEL_ID -def test_generation_config_carries_model_temperature(): - """Per reviewer feedback on HF datastore PR #125: model-under-eval - temperature belongs on generation_config.generation_args.temperature - (schema L400), not in source/metric additional_details. - """ +def _log_for_note(note: str | None) -> EvaluationLog: + row = { + 'model': 'test-model', + 'rank': 1, + 'score': 10.0, + 'company': 'openai', + 'calibrationError': 40, + } + if note is not None: + row['contaminationMessage'] = note + log = adapter.make_logs([row], retrieved_timestamp='123.0')[0][0] + assert len(log.evaluation_results) == 2 + return log + + +def test_no_row_note_leaves_temperature_null_and_records_policy(): + log = _log_for_note(None) + for result in log.evaluation_results: + config = result.generation_config + assert config is None or config.generation_args is None + assert ( + log.source_metadata.additional_details['source_generation_policy'] + == adapter.SOURCE_GENERATION_POLICY + ) + assert 'temperature 0.0 when configurable' in ( + adapter.SOURCE_GENERATION_POLICY + ) + assert 'source_row_note' not in log.model_info.additional_details + + +def test_row_note_with_temperature_and_top_p_is_parsed(): + note = 'Sampled at temperature = 1.0, top_p = 0.95.' + log = _log_for_note(note) + for result in log.evaluation_results: + args = result.generation_config.generation_args + assert args.temperature == 1.0 + assert args.top_p == 0.95 + assert ( + result.generation_config.additional_details['source_row_note'] + == note + ) + assert log.model_info.additional_details['source_row_note'] == note + + +def test_row_note_with_temperature_only_is_parsed(): + log = _log_for_note(' Sampled at temperature 0.7 ') + for result in log.evaluation_results: + args = result.generation_config.generation_args + assert args.temperature == 0.7 + assert args.top_p is None + + +def test_default_temperature_note_parses_nothing(): + note = 'Sampled at 32K Tokens, temp = null (default temp)' + log = _log_for_note(note) + for result in log.evaluation_results: + assert result.generation_config.generation_args is None + assert ( + result.generation_config.additional_details['source_row_note'] + == note + ) + + +def test_budget_and_effort_notes_are_kept_but_not_parsed(): + for note in ( + 'Thinking budget: 16,000 tokens.', + "Sampled at reasoning_effort: 'high'.", + ): + log = _log_for_note(note) + for result in log.evaluation_results: + assert result.generation_config.generation_args is None + assert ( + result.generation_config.additional_details['source_row_note'] + == note + ) + + +def test_note_mentioning_temperature_in_other_prose_parses_nothing(): + for note in ( + 'Sampled at temperature 0.7 with a custom system prompt.', + 'Note: Sampled at temperature = 1.0, top_p = 0.95.', + 'Sampled at temperature = 1.0, top_p = 0.95', + 'The provider ignores temperature settings for this model.', + ): + log = _log_for_note(note) + for result in log.evaluation_results: + assert result.generation_config.generation_args is None + + +def test_judge_prompt_carries_no_temperature(): + assert 'temperature' not in adapter.JUDGE_PROMPT_DESCRIPTION.lower() bundles = adapter.make_logs(sample_rows(), retrieved_timestamp='123.0') for log, _, _ in bundles: for result in log.evaluation_results: - assert result.generation_config is not None - assert result.generation_config.generation_args is not None - assert result.generation_config.generation_args.temperature == 0.0 + prompt = result.metric_config.llm_scoring.input_prompt + assert 'temperature' not in prompt.lower() def test_judge_and_temperature_not_duplicated_in_additional_details(): From 2091386294d4c4bc79271ab4ccabe619ad43e56e Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 06/16] mmlu_pro: write prompt_style only from the model name prompt_style was "5-shot CoT" on every result. The leaderboard has no per-row shot count, and rows named "(0-shot)" contradict the label. prompt_style is now written only when the model name contains "(N-shot)". Co-Authored-By: Claude Fable 5.1 --- every_eval_ever/adapters/mmlu_pro/adapter.py | 64 +++++++----- tests/test_mmlu_pro_adapter.py | 101 +++++++++++++++++++ 2 files changed, 142 insertions(+), 23 deletions(-) diff --git a/every_eval_ever/adapters/mmlu_pro/adapter.py b/every_eval_ever/adapters/mmlu_pro/adapter.py index 66bd0b28c..51521f4d8 100644 --- a/every_eval_ever/adapters/mmlu_pro/adapter.py +++ b/every_eval_ever/adapters/mmlu_pro/adapter.py @@ -12,8 +12,9 @@ Each CSV row carries a model's overall accuracy plus 14 per-subject accuracies (Biology, Business, Chemistry, Computer Science, Economics, Engineering, Health, History, Law, Math, Philosophy, Physics, Psychology, -Other), all reported as proportions in [0, 1] from a 5-shot CoT setup -described in Wang et al. 2024. +Other), all reported as proportions in [0, 1]. The leaderboard has no +prompt-setup column; ``prompt_style`` is recorded only when the model name +carries an ``(N-shot)`` marker, e.g. ``Athene-V2-Chat (0-shot)``. For every model the adapter emits one ``EvaluationLog`` with 15 ``EvaluationResult`` entries: ``mmlu_pro/overall`` and one @@ -158,6 +159,8 @@ 'rho': 'microsoft', } +SHOT_MARKER = re.compile(r'\((\d+)-shot\)', re.IGNORECASE) + def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( @@ -251,21 +254,29 @@ def normalize_developer(model_name: str) -> str: return get_developer(model_name) -def make_source_data() -> SourceDataHf: +def prompt_style_from_name(model_name: str) -> str | None: + """Return the ``N-shot`` label stated by a ``(N-shot)`` name marker.""" + match = SHOT_MARKER.search(model_name) + return f'{int(match.group(1))}-shot' if match else None + + +def make_source_data(prompt_style: str | None = None) -> SourceDataHf: + additional_details = { + 'results_csv_url': RESULTS_CSV_URL, + 'leaderboard_space_url': LEADERBOARD_SPACE_URL, + 'benchmark_hf_repo': BENCHMARK_HF_REPO, + 'paper_url': PAPER_URL, + 'github_url': GITHUB_URL, + 'dataset_total_questions': str(DATASET_TOTAL_QUESTIONS), + } + if prompt_style: + additional_details['prompt_style'] = prompt_style return SourceDataHf( dataset_name='MMLU-Pro leaderboard submissions (TIGER-Lab)', source_type='hf_dataset', hf_repo=RESULTS_HF_REPO, hf_split='train', - additional_details={ - 'results_csv_url': RESULTS_CSV_URL, - 'leaderboard_space_url': LEADERBOARD_SPACE_URL, - 'benchmark_hf_repo': BENCHMARK_HF_REPO, - 'paper_url': PAPER_URL, - 'github_url': GITHUB_URL, - 'dataset_total_questions': str(DATASET_TOTAL_QUESTIONS), - 'prompt_style': '5-shot CoT', - }, + additional_details=additional_details, ) @@ -274,7 +285,11 @@ def make_metric_config( metric_id: str, metric_name: str, description: str, + prompt_style: str | None = None, ) -> MetricConfig: + additional_details = {'aggregation': 'accuracy_over_subset'} + if prompt_style: + additional_details['prompt_style'] = prompt_style return MetricConfig( evaluation_description=description, metric_id=metric_id, @@ -285,24 +300,27 @@ def make_metric_config( score_type=ScoreType.continuous, min_score=0.0, max_score=1.0, - additional_details={ - 'aggregation': 'accuracy_over_subset', - 'prompt_style': '5-shot CoT', - }, + additional_details=additional_details, ) def make_evaluation_result( - *, result_id: str, name: str, description: str, score: float + *, + result_id: str, + name: str, + description: str, + score: float, + prompt_style: str | None = None, ) -> EvaluationResult: return EvaluationResult( evaluation_result_id=result_id, evaluation_name=name, - source_data=make_source_data(), + source_data=make_source_data(prompt_style), metric_config=make_metric_config( metric_id=result_id, metric_name=name, description=description, + prompt_style=prompt_style, ), score_details=ScoreDetails(score=score), ) @@ -354,6 +372,7 @@ def make_log( raw_data_source = row.get('Data Source', '').strip() data_source = normalize_data_source(raw_data_source) size_b = parse_size(row.get('Model Size(B)', '')) + prompt_style = prompt_style_from_name(model_name) results: list[EvaluationResult] = [ make_evaluation_result( @@ -361,9 +380,10 @@ def make_log( name='MMLU-Pro (overall)', description=( 'Overall accuracy across the ~12,000-question MMLU-Pro ' - 'benchmark, evaluated 5-shot with chain-of-thought.' + 'benchmark.' ), score=overall, + prompt_style=prompt_style, ) ] for subject in SUBJECTS: @@ -375,11 +395,9 @@ def make_log( make_evaluation_result( result_id=f'mmlu_pro/{slug}', name=f'MMLU-Pro ({subject})', - description=( - f'Accuracy on the MMLU-Pro {subject} subset, ' - 'evaluated 5-shot with chain-of-thought.' - ), + description=f'Accuracy on the MMLU-Pro {subject} subset.', score=subject_score, + prompt_style=prompt_style, ) ) diff --git a/tests/test_mmlu_pro_adapter.py b/tests/test_mmlu_pro_adapter.py index 0c337126c..fcdc2bb7a 100644 --- a/tests/test_mmlu_pro_adapter.py +++ b/tests/test_mmlu_pro_adapter.py @@ -385,3 +385,104 @@ def test_source_data_is_hf_dataset_pointing_at_results_repo(): overall = log.evaluation_results[0] assert overall.source_data.source_type == 'hf_dataset' assert overall.source_data.hf_repo == adapter.RESULTS_HF_REPO + + +def _prompt_styles(log: EvaluationLog) -> set[str | None]: + styles: set[str | None] = set() + for result in log.evaluation_results: + styles.add( + (result.source_data.additional_details or {}).get('prompt_style') + ) + styles.add( + (result.metric_config.additional_details or {}).get('prompt_style') + ) + return styles + + +def test_rows_without_shot_marker_carry_no_prompt_style(): + bundles = adapter.make_logs(sample_rows(), retrieved_timestamp='123.0') + sources = { + (b[0].source_metadata.additional_details or {}).get( + 'leaderboard_data_source' + ) + for b in bundles + } + assert sources == {'TIGER-Lab', 'Self-Reported'} + for log, _, _ in bundles: + assert _prompt_styles(log) == {None} + + +def test_zero_shot_name_marker_sets_prompt_style(): + row = { + 'Models': 'Athene-V2-Chat (0-shot)', + 'Data Source': 'TIGER-Lab', + 'Model Size(B)': '72', + 'Overall': '0.7311', + 'Math': '0.8', + } + + [(log, _, _)] = adapter.make_logs([row], retrieved_timestamp='123.0') + + assert len(log.evaluation_results) == 2 + for result in log.evaluation_results: + assert result.source_data.additional_details['prompt_style'] == '0-shot' + assert ( + result.metric_config.additional_details['prompt_style'] == '0-shot' + ) + + +def test_descriptions_make_no_shot_claim(): + bundles = adapter.make_logs(sample_rows(), retrieved_timestamp='123.0') + descriptions = [ + result.metric_config.evaluation_description + for log, _, _ in bundles + for result in log.evaluation_results + ] + assert descriptions + for description in descriptions: + assert '5-shot' not in description + assert 'chain-of-thought' not in description + assert '5-shot' not in (adapter.__doc__ or '') + + +def test_blank_data_source_is_recorded_as_unknown(): + row = { + 'Models': 'GPT-4o (2024-05-13)', + 'Data Source': '', + 'Model Size(B)': 'unk', + 'Overall': '0.7255', + } + + [(log, _, _)] = adapter.make_logs([row], retrieved_timestamp='123.0') + + assert ( + log.source_metadata.additional_details['leaderboard_data_source'] + == 'unknown' + ) + assert 'leaderboard_data_source' not in ( + log.model_info.additional_details or {} + ) + + +def test_blank_data_source_keeps_evaluation_id_shape(): + row = { + 'Models': 'GPT-4o (2024-05-13)', + 'Data Source': '', + 'Model Size(B)': 'unk', + 'Overall': '0.7255', + } + + [(log, _, _)] = adapter.make_logs([row], retrieved_timestamp='123.0') + + assert log.evaluation_id == ( + 'mmlu-pro/openai_gpt-4o-2024-05-13/unknown/123.0' + ) + + +def test_evaluation_id_keeps_data_source_segment(): + [(log, _, _)] = adapter.make_logs( + [sample_rows()[0]], retrieved_timestamp='123.0' + ) + assert log.evaluation_id == ( + 'mmlu-pro/openai_gpt-4o-2024-05-13/tiger-lab/123.0' + ) From 231e8d26a9fd9c8219e6ed0d462c9572eb76ba7e Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 07/16] flat rebuild: mark superseded captures in the index A corrected or re-scraped record is published as a new file and the old file stays, so consumers read both. Rows in entries.jsonl now have evaluation_id, retrieved_timestamp, cron_adapter, cron_run_date, result_count, superseded_by and superseded_reason. flat/retired.jsonl ({object_uuid, reason, replaced_by?}) is applied first; retired rows are excluded from the rule below. For rows with type_of_addition=cron, rows with the same (collection, cron_adapter, evaluation_id without its last segment) form a series when that segment equals retrieved_timestamp. Every row except the newest gets superseded_by = the newest row's uuid and superseded_reason = newer_capture. A series is reported and not marked when two rows share a cron_run_date, or when the newest row has fewer results than an older row. A collection that still has rows without evaluation_id is skipped. --bootstrap-ids downloads the aggregate objects once with snapshot_download to fill the new fields; later runs fill up to 500 rows each. data/ and flat/objects/ are not modified. Co-Authored-By: Claude Fable 5.1 --- every_eval_ever/cron/README.md | 41 +++ every_eval_ever/cron/flat_rebuild.py | 492 +++++++++++++++++++++++++- tests/test_flat_rebuild.py | 511 ++++++++++++++++++++++++++- 3 files changed, 1033 insertions(+), 11 deletions(-) diff --git a/every_eval_ever/cron/README.md b/every_eval_ever/cron/README.md index 5c9fef4b3..86ef40e63 100644 --- a/every_eval_ever/cron/README.md +++ b/every_eval_ever/cron/README.md @@ -428,6 +428,47 @@ Unreadable retention entries are kept with a warning naming the snapshot and failure; unscanned snapshots are listed in the summary. Unchanged same-size source files still use the agreed size-based change detection. +### Supersession + +Every row carries `evaluation_id`, `retrieved_timestamp`, `cron_adapter` +and `cron_run_date` (read from the record; the cron fields only when its +`type_of_addition` is `cron`) and `superseded_by` / `superseded_reason`, +recomputed on every run from two rules, in this order: + +- **Retired.** `flat/retired.jsonl` in the datastore, maintained by pull + request, lists `{"object_uuid", "reason", "replaced_by"?}` per line: + `superseded_by` = `replaced_by`, or `"retired"` without one, and reason + `retired`. A malformed line, a duplicate UUID or a self-replacement fails + the run; an entry whose UUID is not in the index is skipped and listed as + stale in the summary, and one whose `replaced_by` is not in the index is + applied and listed as dangling. +- **Newer capture.** Only cron-ingested rows join a series, keyed by + collection, `cron_adapter` and `evaluation_id` minus its trailing + segment, and only when that segment is an epoch number equal to + `retrieved_timestamp`. Among the series' rows that are not retired, the + newest by `retrieved_timestamp` (then UUID) is current; the others get + `superseded_by` = its UUID and reason `newer_capture`. Retiring the + newest capture therefore makes the previous one current again. A series + whose newest row has fewer `evaluation_results` than an older one + (`result_count`, also stored on the row) is left unmarked and listed as + regressed, for a maintainer to settle with a retire entry. A series + where two rows share a `cron_run_date` is ambiguous and left unmarked, and + a collection whose rows still have ids pending is skipped; both are + listed in the summary. Hand-submitted and converted records are only ever + superseded by the retire list. + +Consumers keep rows where `superseded_by` is null. Nothing is deleted: +`data/` and `flat/objects/` are untouched, and the fields enter the +manifest core hash through the entries hash, so a supersession change +publishes a new snapshot while an unchanged rebuild stays a no-op. + +Rows from snapshots older than these fields serialize without +`evaluation_id` until filled. The first fill is a manual run with +`--bootstrap-ids`: one bulk download of the aggregate objects +(`flat/objects/**/*.json`, about 1.4 GB) into a temporary directory, read +locally. Afterwards `--backfill-limit` (default 500, `0` disables) fetches +any stragglers one by one per run. + ### Reviewer comments and changes The review found that companion reattachment could overwrite historical diff --git a/every_eval_ever/cron/flat_rebuild.py b/every_eval_ever/cron/flat_rebuild.py index 8e401e727..3af4addfa 100644 --- a/every_eval_ever/cron/flat_rebuild.py +++ b/every_eval_ever/cron/flat_rebuild.py @@ -13,6 +13,8 @@ behind ``--allow-bootstrap``); 4. rebuild rows, collection indexes and the manifest, preserving published bytes, accepting reserialization and reporting conflicting UUID reuse; + mark superseded rows (see ``apply_supersession``), backfilling the + ``evaluation_id`` of inherited rows that predate the field; 5. retire collection indexes whose collection no longer exists under ``data/`` (moved to ``flat/indexes/retired/``), trim snapshots past the retention window, and commit everything in bounded batches with the @@ -37,13 +39,16 @@ import hashlib import json import os +import re import sys +import tempfile import time import warnings from collections import defaultdict from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta +from decimal import Decimal from pathlib import Path, PurePosixPath from typing import Any, Callable, Sequence from uuid import UUID @@ -68,6 +73,7 @@ RETIRE_PREFIX = 'flat/indexes/retired' BY_LEGACY_PATH = 'flat/indexes/by_legacy_path.jsonl' MANIFESTS_PREFIX = 'flat/manifests' +RETIRED_LIST_PATH = 'flat/retired.jsonl' DATA_PREFIX = 'data/' #: Fields of a manifest excluded from its content hash, mirroring #: ``manifest_core`` in ``tools/build_flat_datastore.py``. @@ -80,6 +86,13 @@ #: decide between trim and pin; unscanned ones are conservatively kept. DEFAULT_SCAN_BUDGET = 20 DOWNLOAD_WORKERS = 8 +#: Inherited rows whose ids are fetched one by one per run by default; a +#: first fill uses ``--bootstrap-ids`` instead. +DEFAULT_BACKFILL_LIMIT = 500 +BACKFILL_CHUNK = 500 +OBJECT_PATTERNS = ('flat/objects/**/*.json',) +SUPERSEDED_REASONS = frozenset({'newer_capture', 'retired'}) +_NUMERIC = re.compile(r'\d+(?:\.\d+)?') #: Excluded paths a maintainer has acknowledged; see ``exit_code_for``. ACKNOWLEDGED_CONFLICTS = Path(__file__).with_name( 'flat_acknowledged_conflicts.txt' @@ -105,8 +118,34 @@ class Row: instance_level_path: str | None = None instance_sha: str | None = None instance_level_size_bytes: int | None = None + evaluation_id: str | None = None + retrieved_timestamp: str | None = None + #: Set only for records whose provenance says ``type_of_addition`` is + #: ``cron``. + cron_adapter: str | None = None + cron_run_date: str | None = None + #: ``len(evaluation_results)``; None when the record has no such list. + result_count: int | None = None + #: False for an inherited row written before the record ids were + #: indexed; such a row serializes without them until backfilled. + ids_recorded: bool = True + superseded_by: str | None = None + superseded_reason: str | None = None def __post_init__(self) -> None: + if (self.superseded_by is None) != (self.superseded_reason is None): + raise FlatRebuildError( + f'{self.object_uuid}: superseded_by and superseded_reason ' + 'are set together' + ) + if ( + self.superseded_reason is not None + and self.superseded_reason not in SUPERSEDED_REASONS + ): + raise FlatRebuildError( + f'{self.object_uuid}: unknown superseded_reason ' + f'{self.superseded_reason!r}' + ) if self.instance_level_available: if ( self.instance_level_path is None @@ -137,6 +176,14 @@ def to_dict(self) -> dict[str, Any]: row['instance_level_path'] = self.instance_level_path row['instance_sha'] = self.instance_sha row['instance_level_size_bytes'] = self.instance_level_size_bytes + if self.ids_recorded: + row['evaluation_id'] = self.evaluation_id + row['retrieved_timestamp'] = self.retrieved_timestamp + row['cron_adapter'] = self.cron_adapter + row['cron_run_date'] = self.cron_run_date + row['result_count'] = self.result_count + row['superseded_by'] = self.superseded_by + row['superseded_reason'] = self.superseded_reason return row @classmethod @@ -153,9 +200,66 @@ def from_dict(cls, row: dict[str, Any]) -> 'Row': instance_level_path=row.get('instance_level_path'), instance_sha=row.get('instance_sha'), instance_level_size_bytes=row.get('instance_level_size_bytes'), + evaluation_id=row.get('evaluation_id'), + retrieved_timestamp=row.get('retrieved_timestamp'), + cron_adapter=row.get('cron_adapter'), + cron_run_date=row.get('cron_run_date'), + result_count=row.get('result_count'), + ids_recorded='evaluation_id' in row, + superseded_by=row.get('superseded_by'), + superseded_reason=row.get('superseded_reason'), ) +@dataclass(frozen=True) +class RecordIds: + """The identity and provenance fields a row copies from its record.""" + + evaluation_id: str | None = None + retrieved_timestamp: str | None = None + cron_adapter: str | None = None + cron_run_date: str | None = None + result_count: int | None = None + + def apply(self, row: 'Row') -> 'Row': + return replace( + row, + evaluation_id=self.evaluation_id, + retrieved_timestamp=self.retrieved_timestamp, + cron_adapter=self.cron_adapter, + cron_run_date=self.cron_run_date, + result_count=self.result_count, + ids_recorded=True, + ) + + +@dataclass(frozen=True) +class Supersession: + """Rows with supersession applied, plus what the rules declined.""" + + rows: tuple['Row', ...] + #: Series keys left unmarked because two captures share a run date. + ambiguous_series: tuple[str, ...] = () + #: Series left unmarked because the newest capture has fewer results + #: than an older one. + regressed_series: tuple[str, ...] = () + #: Collections skipped by the automatic rule while ids are pending. + pending_collections: tuple[str, ...] = () + #: Retire-list entries naming a uuid absent from the index. + stale_retired: tuple[str, ...] = () + #: Retire-list entries whose ``replaced_by`` is absent from the index. + dangling_replaced_by: tuple[str, ...] = () + + +@dataclass(frozen=True) +class RetiredEntry: + """One ``flat/retired.jsonl`` line: a record a maintainer retired.""" + + object_uuid: str + reason: str + replaced_by: str | None = None + + @dataclass(frozen=True) class ManifestInfo: """One snapshot directory under ``flat/manifests/``.""" @@ -233,6 +337,15 @@ class RebuildReport: manifest_core_sha256: str | None = None verified_files: int = 0 verified_reserialized: int = 0 + ids_backfilled: int = 0 + ids_pending: int = 0 + superseded_captures: int = 0 + superseded_retired: int = 0 + ambiguous_series: tuple[str, ...] = () + regressed_series: tuple[str, ...] = () + pending_collections: tuple[str, ...] = () + stale_retired: tuple[str, ...] = () + dangling_replaced_by: tuple[str, ...] = () commits: int = 0 uploads: int = 0 duration_seconds: float = 0.0 @@ -672,6 +785,7 @@ def build_rows( schema_version = schema_version_of(data, path, errors) if schema_version is None: continue + ids = record_ids(data) sha256 = sha256_bytes(data) old = old_by_uuid.get(object_uuid) if old is not None: @@ -698,6 +812,8 @@ def build_rows( excluded.add(samples_path_for(path)) if row is None: continue + if not row.ids_recorded: + row = ids.apply(row) if ( row.instance_level_available and not old.instance_level_available @@ -706,14 +822,16 @@ def build_rows( upload_samples.add(object_uuid) rows[object_uuid] = row continue - row = Row( - object_uuid=object_uuid, - object_path=object_path_for(object_uuid, suffix='.json'), - sha256=sha256, - size_bytes=len(data), - legacy_path=path, - benchmark=benchmark, - eval_schema_version=schema_version, + row = ids.apply( + Row( + object_uuid=object_uuid, + object_path=object_path_for(object_uuid, suffix='.json'), + sha256=sha256, + size_bytes=len(data), + legacy_path=path, + benchmark=benchmark, + eval_schema_version=schema_version, + ) ) if listing is not None and row.object_path in listing: old_object = _fetch_existing( @@ -890,6 +1008,246 @@ def schema_version_of(data: bytes, path: str, errors: list[str]) -> str | None: return version +def record_ids(data: bytes) -> RecordIds: + """Read the fields a row copies from its record; absent ones are None.""" + try: + loaded = json.loads(data) + except json.JSONDecodeError: + return RecordIds() + if not isinstance(loaded, dict): + return RecordIds() + + def text(mapping: Any, key: str) -> str | None: + value = mapping.get(key) if isinstance(mapping, dict) else None + return value if isinstance(value, str) else None + + source = loaded.get('source_metadata') + details = ( + source.get('additional_details') if isinstance(source, dict) else None + ) + cron = text(details, 'type_of_addition') == 'cron' + results = loaded.get('evaluation_results') + return RecordIds( + evaluation_id=text(loaded, 'evaluation_id'), + retrieved_timestamp=text(loaded, 'retrieved_timestamp'), + cron_adapter=text(details, 'cron_adapter') if cron else None, + cron_run_date=text(details, 'cron_run_date') if cron else None, + result_count=len(results) if isinstance(results, list) else None, + ) + + +def backfill_ids( + rows: Sequence[Row], + fetch: Callable[[Sequence[str]], dict[str, bytes]], + *, + limit: int | None, + chunk: int = BACKFILL_CHUNK, +) -> tuple[list[Row], int, int]: + """Fill record ids of inherited rows from their published objects. + + At most ``limit`` rows (``None``: all), in ``legacy_path`` order, are + fetched, ``chunk`` objects at a time. Returns ``(rows, filled, + pending)``. + """ + missing = sorted( + (row for row in rows if not row.ids_recorded), + key=lambda row: row.legacy_path, + ) + todo = missing if limit is None else missing[: max(limit, 0)] + filled: dict[str, Row] = {} + for start in range(0, len(todo), chunk): + batch = todo[start : start + chunk] + contents = fetch([row.object_path for row in batch]) + for row in batch: + filled[row.object_uuid] = record_ids( + contents[row.object_path] + ).apply(row) + return ( + [filled.get(row.object_uuid, row) for row in rows], + len(filled), + len(missing) - len(filled), + ) + + +def local_object_reader( + root: Path, +) -> Callable[[Sequence[str]], dict[str, bytes]]: + """Read published objects from a local ``snapshot_download`` tree.""" + + def fetch(paths: Sequence[str]) -> dict[str, bytes]: + contents: dict[str, bytes] = {} + for path in paths: + local = root / path + if not local.is_file(): + raise FlatRebuildError( + f'{path}: missing from the bulk object download' + ) + contents[path] = local.read_bytes() + return contents + + return fetch + + +def parse_retired(data: bytes) -> list[RetiredEntry]: + """Parse ``flat/retired.jsonl``; malformed lines raise.""" + entries: list[RetiredEntry] = [] + problems: list[str] = [] + seen: set[str] = set() + for number, line in enumerate(data.decode('utf-8').splitlines(), 1): + if not line.strip(): + continue + try: + loaded = json.loads(line) + except json.JSONDecodeError as exc: + problems.append(f'line {number}: invalid JSON ({exc})') + continue + if not isinstance(loaded, dict): + problems.append(f'line {number}: expected a JSON object') + continue + object_uuid = loaded.get('object_uuid') + reason = loaded.get('reason') + replaced_by = loaded.get('replaced_by') + if not isinstance(object_uuid, str) or not object_uuid: + problems.append(f'line {number}: object_uuid is required') + continue + if not isinstance(reason, str) or not reason.strip(): + problems.append(f'line {number}: reason is required') + continue + if replaced_by is not None and not isinstance(replaced_by, str): + problems.append(f'line {number}: replaced_by must be a uuid') + continue + if replaced_by == object_uuid: + problems.append(f'line {number}: {object_uuid} replaces itself') + continue + if object_uuid in seen: + problems.append(f'line {number}: {object_uuid} is listed twice') + continue + seen.add(object_uuid) + entries.append(RetiredEntry(object_uuid, reason, replaced_by)) + if problems: + raise FlatRebuildError( + f'{RETIRED_LIST_PATH} is malformed:\n' + '\n'.join(problems) + ) + return entries + + +def _number(text: str | None) -> Decimal | None: + if text is None or not _NUMERIC.fullmatch(text): + return None + return Decimal(text) + + +def series_key(row: Row) -> tuple[str, str, str] | None: + """Return ``(collection, cron_adapter, id prefix)``, or None. + + Only cron-ingested rows with a run date and a result count form series, and only when the + ``evaluation_id``'s last ``/`` segment is a plain (possibly fractional) + epoch number equal to the record's ``retrieved_timestamp``. + """ + if ( + row.evaluation_id is None + or row.cron_adapter is None + or row.cron_run_date is None + or row.result_count is None + ): + return None + prefix, separator, tail = row.evaluation_id.rpartition('/') + number = _number(tail) + if ( + not separator + or not prefix + or number is None + or number != _number(row.retrieved_timestamp) + ): + return None + return row.benchmark, row.cron_adapter, prefix + + +def apply_supersession( + rows: Sequence[Row], retired: Sequence[RetiredEntry] = () +) -> Supersession: + """Recompute ``superseded_by`` for every row; order is preserved. + + Explicit rule first: each retired entry points at its ``replaced_by`` + or the literal ``'retired'``, with reason ``retired``; entries naming a + uuid absent from ``rows`` are skipped and reported as stale. + + Automatic rule: rows sharing a ``series_key`` form a series. Among the + series' rows that are not retired, the newest capture (by + ``retrieved_timestamp``, then uuid) is current and every other one + points at it with reason ``newer_capture``. A series in which two rows + share a ``cron_run_date`` is ambiguous and left unmarked, as is one + whose newest capture has fewer results than an older one (reported as + regressed for a human to settle with a retire entry). A collection with + rows whose ids are still pending is skipped. + """ + current = { + row.object_uuid: replace( + row, superseded_by=None, superseded_reason=None + ) + for row in rows + } + stale: list[str] = [] + dangling: list[str] = [] + for entry in retired: + if entry.object_uuid not in current: + stale.append(entry.object_uuid) + continue + if entry.replaced_by is not None and entry.replaced_by not in current: + dangling.append(f'{entry.object_uuid} -> {entry.replaced_by}') + current[entry.object_uuid] = replace( + current[entry.object_uuid], + superseded_by=entry.replaced_by or 'retired', + superseded_reason='retired', + ) + pending = {row.benchmark for row in rows if not row.ids_recorded} + series: dict[tuple[str, str, str], list[Row]] = defaultdict(list) + for row in current.values(): + key = series_key(row) + if ( + key is not None + and row.benchmark not in pending + and row.superseded_reason is None + ): + series[key].append(row) + ambiguous: list[str] = [] + regressed: list[str] = [] + for key, members in sorted(series.items()): + if len(members) < 2: + continue + run_dates = [row.cron_run_date for row in members] + if len(set(run_dates)) != len(run_dates): + ambiguous.append('/'.join(key)) + continue + newest = max( + members, + key=lambda row: (_number(row.retrieved_timestamp), row.object_uuid), + ) + older = [row for row in members if row is not newest] + fuller = max(older, key=lambda row: (row.result_count, row.object_uuid)) + if fuller.result_count > newest.result_count: + regressed.append( + f'{"/".join(key)}: newest {newest.object_uuid} has ' + f'{newest.result_count} result(s), {fuller.object_uuid} has ' + f'{fuller.result_count}' + ) + continue + for row in older: + current[row.object_uuid] = replace( + row, + superseded_by=newest.object_uuid, + superseded_reason='newer_capture', + ) + return Supersession( + rows=tuple(current[row.object_uuid] for row in rows), + ambiguous_series=tuple(ambiguous), + regressed_series=tuple(regressed), + pending_collections=tuple(sorted(pending)), + stale_retired=tuple(stale), + dangling_replaced_by=tuple(dangling), + ) + + def plan_retire( listing: dict[str, int], live_benchmarks: set[str], *, today: str ) -> list[tuple[str, str]]: @@ -1256,6 +1614,8 @@ def orchestrate( allow_bootstrap: bool = False, dry_run: bool = False, verify: bool = False, + backfill_limit: int = DEFAULT_BACKFILL_LIMIT, + bootstrap_ids: bool = False, now: datetime | None = None, ) -> RebuildReport: """Plan and, unless dry, publish one flat rebuild. Returns the report.""" @@ -1293,6 +1653,41 @@ def orchestrate( } ) contents = download_bytes(api, repo_id, needed) + removed_uuids = {row.object_uuid for row in diff.removed_rows} + missing_objects = sorted( + row.object_path + for row in old_rows + if not row.ids_recorded + and row.object_uuid not in removed_uuids + and row.object_path not in listing + ) + if missing_objects: + raise FlatRebuildError( + f'{len(missing_objects)} inherited object(s) are missing, so ' + 'their ids cannot be backfilled:\n' + '\n'.join(missing_objects) + ) + inherited = [ + row for row in old_rows if row.object_uuid not in removed_uuids + ] + if bootstrap_ids and any(not row.ids_recorded for row in inherited): + with tempfile.TemporaryDirectory() as bulk: + api.snapshot_download( + repo_id=repo_id, + repo_type='dataset', + allow_patterns=list(OBJECT_PATTERNS), + local_dir=bulk, + ) + kept, report.ids_backfilled, report.ids_pending = backfill_ids( + inherited, local_object_reader(Path(bulk)), limit=None + ) + else: + kept, report.ids_backfilled, report.ids_pending = backfill_ids( + inherited, + lambda paths: download_bytes(api, repo_id, paths), + limit=backfill_limit, + ) + backfilled = {row.object_uuid: row for row in kept} + build_input = [backfilled.get(row.object_uuid, row) for row in old_rows] object_cache: dict[str, bytes] = {} def existing_object(object_path: str) -> bytes | None: @@ -1303,7 +1698,7 @@ def existing_object(object_path: str) -> bytes | None: object_cache.update(download_bytes(api, repo_id, [object_path])) return object_cache[object_path] - build = build_rows(old_rows, diff, contents, existing_object, listing) + build = build_rows(build_input, diff, contents, existing_object, listing) if build.errors: raise FlatRebuildError( f'{len(build.errors)} record(s) cannot be flattened:\n' @@ -1311,7 +1706,24 @@ def existing_object(object_path: str) -> bytes | None: ) report.conflicts = build.conflicts report.excluded_paths = build.excluded_paths - rows = build.rows + retired: list[RetiredEntry] = [] + if RETIRED_LIST_PATH in listing: + retired = parse_retired( + download_bytes(api, repo_id, [RETIRED_LIST_PATH])[RETIRED_LIST_PATH] + ) + supersession = apply_supersession(build.rows, retired) + rows = supersession.rows + report.ambiguous_series = supersession.ambiguous_series + report.regressed_series = supersession.regressed_series + report.pending_collections = supersession.pending_collections + report.stale_retired = supersession.stale_retired + report.dangling_replaced_by = supersession.dangling_replaced_by + report.superseded_captures = sum( + 1 for row in rows if row.superseded_reason == 'newer_capture' + ) + report.superseded_retired = sum( + 1 for row in rows if row.superseded_reason == 'retired' + ) if verify: pending_objects = frozenset( [ @@ -1510,8 +1922,42 @@ def summary_lines( f'{len(report.manifests_trimmed)} trimmed, ' f'{len(report.manifests_pinned)} pinned, ' f'{len(report.manifests_unscanned)} unscanned', + f'- superseded: {report.superseded_captures} by a newer capture, ' + f'{report.superseded_retired} retired', f'- manifest core sha256: `{report.manifest_core_sha256}`', ] + if report.ids_backfilled or report.ids_pending: + lines.append( + f'- evaluation_id backfill: {report.ids_backfilled} filled, ' + f'{report.ids_pending} pending' + ) + if report.pending_collections: + lines.append( + f'- automatic supersession skipped in ' + f'{len(report.pending_collections)} collection(s) with ids ' + 'pending' + ) + if report.ambiguous_series: + lines.append( + f'- **ambiguous series: {len(report.ambiguous_series)}** ' + '(two captures share a cron run date; left unmarked)' + ) + if report.regressed_series: + lines.append( + f'- **regressed series: {len(report.regressed_series)}** ' + '(newest capture has fewer results than an older one; left ' + 'unmarked)' + ) + if report.stale_retired: + lines.append( + f'- **stale retire entries: {len(report.stale_retired)}** ' + f'uuid(s) in {RETIRED_LIST_PATH} are not in the index' + ) + if report.dangling_replaced_by: + lines.append( + f'- **dangling replaced_by: {len(report.dangling_replaced_by)}** ' + f'{RETIRED_LIST_PATH} entries point at uuids not in the index' + ) if report.verified_files: lines.append( f'- deep verification: {report.verified_files} data file(s) ' @@ -1551,6 +1997,11 @@ def summary_lines( for label, paths in ( ('Excluded records', report.conflicts), ('Stale acknowledgements', stale), + ('Ambiguous series', report.ambiguous_series), + ('Regressed series', report.regressed_series), + ('Collections with ids pending', report.pending_collections), + ('Stale retire entries', report.stale_retired), + ('Dangling replaced_by', report.dangling_replaced_by), ('Retired indexes', report.indexes_retired), ('Trimmed snapshots', report.manifests_trimmed), ('Pinned snapshots', report.manifests_pinned), @@ -1622,6 +2073,25 @@ def main(argv: list[str] | None = None) -> int: 'loudly. Slow; intended for a periodic sweep, not every run.' ), ) + parser.add_argument( + '--backfill-limit', + type=int, + default=DEFAULT_BACKFILL_LIMIT, + help=( + 'Max inherited rows per run whose ids are fetched one by one ' + 'from their published object (default: %(default)s; 0 ' + 'disables).' + ), + ) + parser.add_argument( + '--bootstrap-ids', + action='store_true', + help=( + 'Fill the ids of every inherited row from one bulk download of ' + 'the aggregate objects (about 1.4 GB) instead of per-file ' + 'fetches.' + ), + ) parser.add_argument( '--acknowledged-conflicts', type=Path, @@ -1644,6 +2114,8 @@ def main(argv: list[str] | None = None) -> int: allow_bootstrap=args.allow_bootstrap, dry_run=args.dry_run, verify=args.verify, + backfill_limit=args.backfill_limit, + bootstrap_ids=args.bootstrap_ids, ) except FlatRebuildError as exc: if os.environ.get('GITHUB_ACTIONS'): diff --git a/tests/test_flat_rebuild.py b/tests/test_flat_rebuild.py index 0b1452a4d..8fab9b663 100644 --- a/tests/test_flat_rebuild.py +++ b/tests/test_flat_rebuild.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from pathlib import Path, PurePosixPath from typing import Any @@ -1318,3 +1318,512 @@ def test_verify_bootstrap_allows_planned_new_objects(tmp_path): assert not report.conflicts assert report.verified_files == 0 assert row.object_path in api.files + + +# -- supersession ------------------------------------------------------------ + + +def captured_record( + uuid: str, + evaluation_id: str | None, + retrieved: str | None, + *, + adapter: str | None = 'hle', + run_date: str | None = None, +) -> bytes: + payload = json.loads(record_bytes(uuid, 'x')) + payload.pop('evaluation_id') + if evaluation_id is not None: + payload['evaluation_id'] = evaluation_id + if retrieved is not None: + payload['retrieved_timestamp'] = retrieved + if adapter is not None: + payload['source_metadata'] = { + 'additional_details': { + 'type_of_addition': 'cron', + 'cron_adapter': adapter, + 'cron_run_date': run_date or f'day-{retrieved}', + } + } + return json.dumps(payload).encode() + + +def captured_row( + uuid: str, + benchmark: str, + evaluation_id: str | None, + retrieved: str | None = None, + *, + adapter: str | None = 'hle', + run_date: str | None = None, + result_count: int = 1, +) -> fr.Row: + return replace( + make_row(uuid, benchmark), + evaluation_id=evaluation_id, + result_count=result_count, + retrieved_timestamp=retrieved, + cron_adapter=adapter, + cron_run_date=(run_date or f'day-{retrieved}') if adapter else None, + ) + + +def capture(uuid: str, stamp: str, **kwargs: Any) -> fr.Row: + return captured_row(uuid, 'hle', f'hle/m/{stamp}', stamp, **kwargs) + + +def supersede(rows, retired=()) -> dict[str, fr.Row]: + return by_uuid(fr.apply_supersession(rows, retired).rows) + + +def by_uuid(rows) -> dict[str, fr.Row]: + return {row.object_uuid: row for row in rows} + + +def test_older_capture_is_superseded_by_newest() -> None: + rows = supersede([capture('a', '100.5'), capture('b', '200')]) + assert rows['a'].superseded_by == 'b' + assert rows['a'].superseded_reason == 'newer_capture' + assert rows['b'].superseded_by is None + assert rows['b'].to_dict()['superseded_by'] is None + + +def test_timestamp_tie_breaks_on_uuid() -> None: + rows = supersede( + [capture('c', '500'), capture('a', '500.0', run_date='d2')] + ) + assert rows['c'].superseded_by is None + assert rows['a'].superseded_by == 'c' + + +def test_record_ids_reads_cron_provenance_only_for_cron_records() -> None: + ids = fr.record_ids(captured_record('a', 'hle/m/1', '1', run_date='d')) + assert ids == fr.RecordIds('hle/m/1', '1', 'hle', 'd', 1) + manual = json.loads(captured_record('a', 'hle/m/1', '1')) + manual['source_metadata']['additional_details']['type_of_addition'] = ( + 'manual' + ) + ids = fr.record_ids(json.dumps(manual).encode()) + assert (ids.cron_adapter, ids.cron_run_date) == (None, None) + + +def test_rows_without_cron_provenance_are_never_grouped() -> None: + rows = fr.apply_supersession( + [capture('a', '100', adapter=None), capture('b', '200', adapter=None)] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_series_are_keyed_by_adapter() -> None: + rows = fr.apply_supersession( + [capture('a', '100'), capture('b', '200', adapter='other')] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_tail_must_equal_retrieved_timestamp() -> None: + rows = fr.apply_supersession( + [ + captured_row('a', 'hle', 'hle/m/100', '150'), + captured_row('b', 'hle', 'hle/m/200', '200'), + ] + ).rows + assert all(row.superseded_by is None for row in rows) + + +@pytest.mark.parametrize( + 'evaluation_id', ['hle/m/run-a', 'hle/m/1e9', '12345', None] +) +def test_ids_without_timestamp_segment_form_no_series(evaluation_id) -> None: + tail = (evaluation_id or '').rpartition('/')[2] or None + rows = fr.apply_supersession( + [ + captured_row('a', 'hle', evaluation_id, tail, run_date='d1'), + captured_row('b', 'hle', evaluation_id, tail, run_date='d2'), + ] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_series_do_not_cross_collections() -> None: + rows = fr.apply_supersession( + [ + captured_row('a', 'hle', 'shared/m/100', '100'), + captured_row('b', 'mmlu_pro', 'shared/m/200', '200'), + ] + ).rows + assert all(row.superseded_by is None for row in rows) + + +def test_same_run_date_makes_series_ambiguous() -> None: + result = fr.apply_supersession( + [ + capture('a', '100', run_date='2026-08-13'), + capture('b', '200', run_date='2026-08-13'), + capture('c', '300', run_date='2026-08-14'), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert result.ambiguous_series == ('hle/hle/hle/m',) + + +def test_newest_with_fewer_results_leaves_series_unmarked() -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_count=5), + capture('b', '200', result_count=3), + capture('c', '300', result_count=2), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert result.regressed_series == ( + 'hle/hle/hle/m: newest c has 2 result(s), a has 5', + ) + report = fr.RebuildReport( + repo_id='org/ds', regressed_series=result.regressed_series + ) + lines = fr.summary_lines(report) + assert any('regressed series: 1' in line for line in lines) + + +@pytest.mark.parametrize('newest_count', [3, 4]) +def test_newest_with_equal_or_more_results_supersedes(newest_count) -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_count=3), + capture('b', '200', result_count=newest_count), + ] + ) + assert by_uuid(result.rows)['a'].superseded_by == 'b' + assert result.regressed_series == () + + +def test_retired_fuller_row_does_not_block_series() -> None: + rows = supersede( + [ + capture('a', '100', result_count=9), + capture('b', '200', result_count=2), + capture('c', '300', result_count=2), + ], + [fr.RetiredEntry('a', 'extra rows were wrong')], + ) + assert rows['b'].superseded_by == 'c' + + +def test_collection_with_pending_ids_is_skipped() -> None: + pending = replace(make_row('p', 'hle'), ids_recorded=False) + result = fr.apply_supersession( + [ + capture('a', '100'), + capture('b', '200'), + pending, + captured_row('c', 'gsm8k', 'gsm8k/m/1', '1'), + captured_row('d', 'gsm8k', 'gsm8k/m/2', '2'), + ] + ) + rows = by_uuid(result.rows) + assert rows['a'].superseded_by is None + assert rows['c'].superseded_by == 'd' + assert result.pending_collections == ('hle',) + + +def test_retire_list_marks_rows_with_or_without_replacement() -> None: + rows = supersede( + [ + captured_row('c', 'hle', 'hle/other/1', '1'), + capture('b', '200'), + captured_row('d', 'hle', None, adapter=None), + ], + [ + fr.RetiredEntry('b', 'wrong temperature', replaced_by='c'), + fr.RetiredEntry('d', 'pre-cron, no successor'), + ], + ) + assert (rows['b'].superseded_by, rows['b'].superseded_reason) == ( + 'c', + 'retired', + ) + assert (rows['d'].superseded_by, rows['d'].superseded_reason) == ( + 'retired', + 'retired', + ) + assert rows['c'].superseded_by is None + + +def test_retiring_newest_capture_restores_previous() -> None: + rows = supersede( + [capture('a', '100'), capture('b', '200'), capture('c', '50')], + [fr.RetiredEntry('b', 'wrong temperature')], + ) + assert rows['a'].superseded_by is None + assert rows['b'].superseded_reason == 'retired' + assert rows['c'].superseded_by == 'a' + + +def test_retiring_older_capture_leaves_newest_current() -> None: + rows = supersede( + [capture('a', '100'), capture('b', '200')], + [fr.RetiredEntry('a', 'bad capture', replaced_by='b')], + ) + assert rows['b'].superseded_by is None + assert (rows['a'].superseded_by, rows['a'].superseded_reason) == ( + 'b', + 'retired', + ) + + +def test_fully_retired_series_has_no_current_row() -> None: + rows = fr.apply_supersession( + [capture('a', '100'), capture('b', '200')], + [fr.RetiredEntry('a', 'x'), fr.RetiredEntry('b', 'y')], + ).rows + assert all(row.superseded_reason == 'retired' for row in rows) + + +def test_stale_retire_entry_is_reported_and_skipped() -> None: + result = fr.apply_supersession( + [capture('a', '100')], + [ + fr.RetiredEntry('zzz', 'gone'), + fr.RetiredEntry('a', 'successor left', replaced_by='yyy'), + ], + ) + assert result.stale_retired == ('zzz',) + assert result.rows[0].superseded_by == 'yyy' + assert result.dangling_replaced_by == ('a -> yyy',) + + +def test_dangling_replaced_by_is_listed_in_summary() -> None: + report = fr.RebuildReport( + repo_id='org/ds', dangling_replaced_by=('a -> yyy',) + ) + lines = fr.summary_lines(report) + assert any('dangling replaced_by: 1' in line for line in lines) + assert '- `a -> yyy`' in lines + + +def test_supersession_is_recomputed_not_inherited() -> None: + stale = replace( + captured_row('a', 'hle', None), + superseded_by='retired', + superseded_reason='retired', + ) + assert fr.apply_supersession([stale]).rows[0].superseded_by is None + + +def test_unknown_superseded_reason_is_a_rebuild_error() -> None: + row = capture('a', '1').to_dict() + row.update(superseded_by='b', superseded_reason='bogus') + with pytest.raises(fr.FlatRebuildError, match='bogus'): + fr.Row.from_dict(row) + + +@pytest.mark.parametrize( + 'text', + [ + b'not json\n', + b'{"reason": "no uuid"}\n', + b'{"object_uuid": "a"}\n', + b'{"object_uuid": "a", "reason": "x", "replaced_by": "a"}\n', + b'{"object_uuid": "a", "reason": "x"}\n{"object_uuid": "a", "reason": "y"}\n', + ], +) +def test_malformed_retire_list_errors(text) -> None: + with pytest.raises(fr.FlatRebuildError, match='malformed'): + fr.parse_retired(text) + + +def test_row_ids_round_trip_and_legacy_rows_are_unrecorded() -> None: + row = replace( + capture('a', '1'), + superseded_by='b', + superseded_reason='newer_capture', + ) + assert fr.Row.from_dict(row.to_dict()) == row + legacy = row.to_dict() + for key in ( + 'evaluation_id', + 'retrieved_timestamp', + 'cron_adapter', + 'cron_run_date', + 'result_count', + ): + del legacy[key] + restored = fr.Row.from_dict(legacy) + assert not restored.ids_recorded + assert 'evaluation_id' not in restored.to_dict() + + +def seed_captures(files: dict[str, bytes], benchmark: str, stamps) -> list: + uuids = [] + for stamp in stamps: + uuid = str(uuid4()) + files[f'data/{benchmark}/dev/model/{uuid}.json'] = captured_record( + uuid, f'{benchmark}/dev/model/{stamp}', stamp + ) + uuids.append(uuid) + return uuids + + +def published_rows(api: FakeApi, path: str) -> dict[str, dict[str, Any]]: + return { + row['object_uuid']: row + for row in map(json.loads, api.files[path].decode().splitlines()) + } + + +def test_orchestrate_marks_supersession_and_stays_idempotent( + tmp_path: Path, +) -> None: + files: dict[str, bytes] = {} + old, new = seed_captures(files, 'hle', ['100', '200']) + api = FakeApi(files, tmp_path) + fr.orchestrate(api, 'org/ds', allow_bootstrap=True, now=NOW) + index = published_rows(api, f'{fr.INDEXES_PREFIX}/hle.jsonl') + assert index[old]['superseded_by'] == new + assert index[old]['evaluation_id'] == 'hle/dev/model/100' + assert index[old]['cron_adapter'] == 'hle' + assert index[old]['result_count'] == 1 + assert index[new]['superseded_by'] is None + latest = json.loads(api.files[fr.LATEST_MANIFEST_PATH]) + assert published_rows(api, latest['entries_path']) == index + + commits = len(api.commits) + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert report.noop + assert report.superseded_captures == 1 + assert len(api.commits) == commits + + +def test_orchestrate_applies_retire_list_and_republishes( + tmp_path: Path, +) -> None: + files: dict[str, bytes] = {} + (only,) = seed_captures(files, 'hle', ['100']) + api = FakeApi(files, tmp_path) + fr.orchestrate(api, 'org/ds', allow_bootstrap=True, now=NOW) + core_before = json.loads(api.files[fr.LATEST_MANIFEST_PATH])[ + 'manifest_core_sha256' + ] + + api.files[fr.RETIRED_LIST_PATH] = ( + json.dumps({'object_uuid': only, 'reason': 'wrong prompt label'}) + '\n' + ).encode() + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert not report.noop + assert report.superseded_retired == 1 + latest = json.loads(api.files[fr.LATEST_MANIFEST_PATH]) + assert latest['manifest_core_sha256'] != core_before + row = published_rows(api, f'{fr.INDEXES_PREFIX}/hle.jsonl')[only] + assert (row['superseded_by'], row['superseded_reason']) == ( + 'retired', + 'retired', + ) + assert f'data/hle/dev/model/{only}.json' in api.files + assert fr.orchestrate(api, 'org/ds', now=NOW).noop + + +def test_orchestrate_reports_stale_retire_entry(tmp_path: Path) -> None: + files: dict[str, bytes] = {} + seed_captures(files, 'hle', ['100']) + api = FakeApi(files, tmp_path) + fr.orchestrate(api, 'org/ds', allow_bootstrap=True, now=NOW) + api.files[fr.RETIRED_LIST_PATH] = ( + b'{"object_uuid": "0000", "reason": "typo"}\n' + ) + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert report.noop + assert report.stale_retired == ('0000',) + assert any( + 'stale retire entries: 1' in line for line in fr.summary_lines(report) + ) + + +def seed_legacy_snapshot(files: dict[str, bytes], uuids) -> None: + """Publish rows for ``uuids`` in the shape written before ids existed.""" + legacy_rows = [] + for uuid in uuids: + data = files[f'data/hle/dev/model/{uuid}.json'] + row = replace( + make_row(uuid, 'hle'), + sha256=fr.sha256_bytes(data), + size_bytes=len(data), + ) + files[row.object_path] = data + legacy = row.to_dict() + for key in ( + 'evaluation_id', + 'retrieved_timestamp', + 'cron_adapter', + 'cron_run_date', + 'result_count', + 'superseded_by', + 'superseded_reason', + ): + legacy.pop(key, None) + legacy_rows.append(legacy) + manifest = fr.manifest_for( + [fr.Row.from_dict(row) for row in legacy_rows], + created_at=RECENT.isoformat(), + ) + files[manifest['entries_path']] = fr.jsonl_text(legacy_rows).encode() + seed_pointer(files, manifest) + + +def test_orchestrate_backfills_ids_in_bounded_passes(tmp_path: Path) -> None: + files: dict[str, bytes] = {} + uuids = seed_captures(files, 'hle', ['100', '200', '300']) + seed_legacy_snapshot(files, uuids) + api = FakeApi(files, tmp_path) + + report = fr.orchestrate(api, 'org/ds', now=NOW, backfill_limit=2) + assert (report.ids_backfilled, report.ids_pending) == (2, 1) + assert report.pending_collections == ('hle',) + assert report.superseded_captures == 0 + report = fr.orchestrate(api, 'org/ds', now=NOW, backfill_limit=2) + assert (report.ids_backfilled, report.ids_pending) == (1, 0) + assert report.superseded_captures == 2 + index = published_rows(api, f'{fr.INDEXES_PREFIX}/hle.jsonl') + assert {row['superseded_by'] for row in index.values()} == { + uuids[2], + None, + } + report = fr.orchestrate(api, 'org/ds', now=NOW, backfill_limit=2) + assert report.noop + assert report.ids_backfilled == 0 + + +def test_bootstrap_ids_reads_one_bulk_download(tmp_path: Path) -> None: + files: dict[str, bytes] = {} + uuids = seed_captures(files, 'hle', ['100', '200']) + seed_legacy_snapshot(files, uuids) + api = FakeApi(files, tmp_path) + calls: list[dict[str, Any]] = [] + + def snapshot_download(**kwargs: Any) -> str: + calls.append(kwargs) + root = Path(kwargs['local_dir']) + for path, data in api.files.items(): + if path.startswith('flat/objects/') and path.endswith('.json'): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_bytes(data) + return str(root) + + api.snapshot_download = snapshot_download + fetched: list[str] = [] + original = api.hf_hub_download + + def tracking(repo_id: str, repo_type: str, filename: str) -> str: + fetched.append(filename) + return original(repo_id, repo_type, filename) + + api.hf_hub_download = tracking + report = fr.orchestrate( + api, 'org/ds', now=NOW, bootstrap_ids=True, backfill_limit=0 + ) + assert len(calls) == 1 + assert calls[0]['allow_patterns'] == ['flat/objects/**/*.json'] + assert calls[0]['repo_type'] == 'dataset' + assert not any(path.startswith('flat/objects/') for path in fetched) + assert (report.ids_backfilled, report.ids_pending) == (2, 0) + assert report.superseded_captures == 1 From f7aa9c38969f68701be0d44a454fb06a7626cef4 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Mon, 28 Sep 2026 20:40:10 +0100 Subject: [PATCH 08/16] catalog: hfopenllm_v2 inputs and arc_agi note hfopenllm_v2 lists pyarrow in with_packages (needed to read the contents parquet) and its note names the pinned inputs. The arc_agi note describes the clones; a live run took 24 s, so the default timeout is kept. Co-Authored-By: Claude Fable 5.1 --- every_eval_ever/adapters/catalog.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/every_eval_ever/adapters/catalog.py b/every_eval_ever/adapters/catalog.py index 2592f9d8f..6087ba9f3 100644 --- a/every_eval_ever/adapters/catalog.py +++ b/every_eval_ever/adapters/catalog.py @@ -250,7 +250,11 @@ def _helm(key: str, leaderboard: str, weekday: int) -> AdapterSpec: notes=( 'Repointed 2026-08-12 to the JSON files behind ' 'arcprize.org/leaderboard; the old ' - '/media/data/leaderboard/evaluations.json endpoint is gone.' + '/media/data/leaderboard/evaluations.json endpoint is gone. ' + 'Also clones the two arcprize public-eval log datasets (full ' + 'no-checkout clones, about 130 MB of packs, then a sparse ' + 'checkout at the pinned commits); a live run measured 24 s, so ' + 'the default timeout stands.' ), ), AdapterSpec( @@ -315,6 +319,7 @@ def _helm(key: str, leaderboard: str, weekday: int) -> AdapterSpec: weekday=4, timeout_minutes=45, runnable=False, + with_packages=('pyarrow',), unrunnable_reason=( 'the Open LLM Leaderboard is no longer maintained upstream ' '(space discussion 1135); the archive is frozen, so a scheduled ' @@ -322,7 +327,11 @@ def _helm(key: str, leaderboard: str, weekday: int) -> AdapterSpec: ), notes=( 'The adapter still works for a one-off manual conversion of the ' - 'frozen archive (around 4,576 models).' + 'frozen archive (4,576 model and precision rows): it reads the ' + 'leaderboard table from the open-llm-leaderboard/contents ' + 'parquet and the harness dumps from open-llm-leaderboard/results, ' + 'both at pinned revisions (pyarrow, from with_packages, or `uv run --with pyarrow`). The Space API is ' + 'no longer served and is only read with --source-api.' ), ), AdapterSpec( From 18f66b23dffdd585aa03f46ab581bacc509e57d7 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 15:35:41 +0100 Subject: [PATCH 09/16] Require result identity coverage for automatic supersession --- every_eval_ever/cron/README.md | 18 ++-- every_eval_ever/cron/flat_rebuild.py | 84 +++++++++++++--- tests/test_flat_rebuild.py | 143 ++++++++++++++++++++++++++- 3 files changed, 224 insertions(+), 21 deletions(-) diff --git a/every_eval_ever/cron/README.md b/every_eval_ever/cron/README.md index 86ef40e63..3a98e41d9 100644 --- a/every_eval_ever/cron/README.md +++ b/every_eval_ever/cron/README.md @@ -449,9 +449,15 @@ recomputed on every run from two rules, in this order: newest by `retrieved_timestamp` (then UUID) is current; the others get `superseded_by` = its UUID and reason `newer_capture`. Retiring the newest capture therefore makes the previous one current again. A series - whose newest row has fewer `evaluation_results` than an older one - (`result_count`, also stored on the row) is left unmarked and listed as - regressed, for a maintainer to settle with a retire entry. A series + whose newest row drops any older result identity is left unmarked and + listed as regressed, for a maintainer to settle with a retire entry. + `result_identities` stores a sorted multiset of SHA-256 hashes of result + metadata: all result fields except `score_details`, `evaluation_timestamp` + and the run-local `evaluation_result_id`. Thus benchmark names, metrics, + dataset/split, generation settings and duplicate multiplicity must all be + covered. A metadata correction may require reviewed retirement; changed + scores alone do not. Unreadable identities also leave the series unmarked. + `result_count` remains diagnostic and never establishes coverage. A series where two rows share a `cron_run_date` is ambiguous and left unmarked, and a collection whose rows still have ids pending is skipped; both are listed in the summary. Hand-submitted and converted records are only ever @@ -462,10 +468,10 @@ Consumers keep rows where `superseded_by` is null. Nothing is deleted: manifest core hash through the entries hash, so a supersession change publishes a new snapshot while an unchanged rebuild stays a no-op. -Rows from snapshots older than these fields serialize without -`evaluation_id` until filled. The first fill is a manual run with +Rows from snapshots older than either record ids or `result_identities` +serialize without identity fields until filled. The first fill is a manual run with `--bootstrap-ids`: one bulk download of the aggregate objects -(`flat/objects/**/*.json`, about 1.4 GB) into a temporary directory, read +(`flat/objects/**/*.json`, size depends on the datastore) into a temporary directory, read locally. Afterwards `--backfill-limit` (default 500, `0` disables) fetches any stragglers one by one per run. diff --git a/every_eval_ever/cron/flat_rebuild.py b/every_eval_ever/cron/flat_rebuild.py index 3af4addfa..d1d1057e2 100644 --- a/every_eval_ever/cron/flat_rebuild.py +++ b/every_eval_ever/cron/flat_rebuild.py @@ -44,7 +44,7 @@ import tempfile import time import warnings -from collections import defaultdict +from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta @@ -126,6 +126,8 @@ class Row: cron_run_date: str | None = None #: ``len(evaluation_results)``; None when the record has no such list. result_count: int | None = None + #: Sorted metadata hashes, preserving duplicate result identities. + result_identities: tuple[str, ...] | None = None #: False for an inherited row written before the record ids were #: indexed; such a row serializes without them until backfilled. ids_recorded: bool = True @@ -182,6 +184,11 @@ def to_dict(self) -> dict[str, Any]: row['cron_adapter'] = self.cron_adapter row['cron_run_date'] = self.cron_run_date row['result_count'] = self.result_count + row['result_identities'] = ( + list(self.result_identities) + if self.result_identities is not None + else None + ) row['superseded_by'] = self.superseded_by row['superseded_reason'] = self.superseded_reason return row @@ -205,7 +212,14 @@ def from_dict(cls, row: dict[str, Any]) -> 'Row': cron_adapter=row.get('cron_adapter'), cron_run_date=row.get('cron_run_date'), result_count=row.get('result_count'), - ids_recorded='evaluation_id' in row, + result_identities=( + tuple(row['result_identities']) + if row.get('result_identities') is not None + else None + ), + ids_recorded=( + 'evaluation_id' in row and 'result_identities' in row + ), superseded_by=row.get('superseded_by'), superseded_reason=row.get('superseded_reason'), ) @@ -220,6 +234,8 @@ class RecordIds: cron_adapter: str | None = None cron_run_date: str | None = None result_count: int | None = None + #: Sorted metadata hashes, preserving duplicate result identities. + result_identities: tuple[str, ...] | None = None def apply(self, row: 'Row') -> 'Row': return replace( @@ -229,6 +245,7 @@ def apply(self, row: 'Row') -> 'Row': cron_adapter=self.cron_adapter, cron_run_date=self.cron_run_date, result_count=self.result_count, + result_identities=self.result_identities, ids_recorded=True, ) @@ -240,8 +257,8 @@ class Supersession: rows: tuple['Row', ...] #: Series keys left unmarked because two captures share a run date. ambiguous_series: tuple[str, ...] = () - #: Series left unmarked because the newest capture has fewer results - #: than an older one. + #: Series left unmarked because the newest capture does not cover + #: every older result identity, or identities cannot be read. regressed_series: tuple[str, ...] = () #: Collections skipped by the automatic rule while ids are pending. pending_collections: tuple[str, ...] = () @@ -1008,6 +1025,34 @@ def schema_version_of(data: bytes, path: str, errors: list[str]) -> str | None: return version +def result_identities(results: Any) -> tuple[str, ...] | None: + """Hash result metadata, excluding scores and capture-local identifiers. + + Duplicate multiplicity is preserved; result order is ignored. Malformed + result lists have no usable identity coverage. + """ + if not isinstance(results, list): + return None + identities = [] + for result in results: + if not isinstance(result, dict) or not isinstance( + result.get('evaluation_name'), str + ): + return None + metadata = { + key: value + for key, value in result.items() + if key + not in { + 'score_details', + 'evaluation_timestamp', + 'evaluation_result_id', + } + } + identities.append(sha256_bytes(stable_json_bytes(metadata))) + return tuple(sorted(identities)) + + def record_ids(data: bytes) -> RecordIds: """Read the fields a row copies from its record; absent ones are None.""" try: @@ -1033,6 +1078,7 @@ def text(mapping: Any, key: str) -> str | None: cron_adapter=text(details, 'cron_adapter') if cron else None, cron_run_date=text(details, 'cron_run_date') if cron else None, result_count=len(results) if isinstance(results, list) else None, + result_identities=result_identities(results), ) @@ -1177,9 +1223,10 @@ def apply_supersession( ``retrieved_timestamp``, then uuid) is current and every other one points at it with reason ``newer_capture``. A series in which two rows share a ``cron_run_date`` is ambiguous and left unmarked, as is one - whose newest capture has fewer results than an older one (reported as - regressed for a human to settle with a retire entry). A collection with - rows whose ids are still pending is skipped. + whose newest capture does not contain every older result identity, + including duplicate multiplicity (reported as regressed for a human to + settle with a retire entry). Unreadable identities also prevent automatic + supersession. A collection with rows whose ids are still pending is skipped. """ current = { row.object_uuid: replace( @@ -1224,12 +1271,25 @@ def apply_supersession( key=lambda row: (_number(row.retrieved_timestamp), row.object_uuid), ) older = [row for row in members if row is not newest] - fuller = max(older, key=lambda row: (row.result_count, row.object_uuid)) - if fuller.result_count > newest.result_count: + if any(row.result_identities is None for row in members): + regressed.append(f'{"/".join(key)}: unreadable result identities') + continue + newest_identities = Counter(newest.result_identities) + missing = [ + ( + row.object_uuid, + Counter(row.result_identities) - newest_identities, + ) + for row in older + ] + lost = [ + (uuid, sum(counts.values())) for uuid, counts in missing if counts + ] + if lost: regressed.append( - f'{"/".join(key)}: newest {newest.object_uuid} has ' - f'{newest.result_count} result(s), {fuller.object_uuid} has ' - f'{fuller.result_count}' + f'{"/".join(key)}: newest {newest.object_uuid} drops result ' + 'identities from ' + + ', '.join(f'{uuid} ({count})' for uuid, count in sorted(lost)) ) continue for row in older: diff --git a/tests/test_flat_rebuild.py b/tests/test_flat_rebuild.py index 8fab9b663..06fc7b2b8 100644 --- a/tests/test_flat_rebuild.py +++ b/tests/test_flat_rebuild.py @@ -1213,7 +1213,9 @@ def test_moved_companion_conflict_excludes_samples_path(): path: record_bytes(row.object_uuid, 'gsm8k'), sample_path: b'{"x":2000}\n', }, - lambda object_path: published if object_path == row.instance_level_path else None, + lambda object_path: ( + published if object_path == row.instance_level_path else None + ), ) assert not result.errors assert len(result.conflicts) == 1 @@ -1357,11 +1359,17 @@ def captured_row( adapter: str | None = 'hle', run_date: str | None = None, result_count: int = 1, + result_identities: tuple[str, ...] | None = None, ) -> fr.Row: return replace( make_row(uuid, benchmark), evaluation_id=evaluation_id, result_count=result_count, + result_identities=( + result_identities + if result_identities is not None + else tuple(f'result-{i}' for i in range(result_count)) + ), retrieved_timestamp=retrieved, cron_adapter=adapter, cron_run_date=(run_date or f'day-{retrieved}') if adapter else None, @@ -1398,7 +1406,18 @@ def test_timestamp_tie_breaks_on_uuid() -> None: def test_record_ids_reads_cron_provenance_only_for_cron_records() -> None: ids = fr.record_ids(captured_record('a', 'hle/m/1', '1', run_date='d')) - assert ids == fr.RecordIds('hle/m/1', '1', 'hle', 'd', 1) + assert ids == fr.RecordIds( + 'hle/m/1', + '1', + 'hle', + 'd', + 1, + fr.result_identities( + json.loads(captured_record('a', 'hle/m/1', '1'))[ + 'evaluation_results' + ] + ), + ) manual = json.loads(captured_record('a', 'hle/m/1', '1')) manual['source_metadata']['additional_details']['type_of_addition'] = ( 'manual' @@ -1477,7 +1496,7 @@ def test_newest_with_fewer_results_leaves_series_unmarked() -> None: ) assert all(row.superseded_by is None for row in result.rows) assert result.regressed_series == ( - 'hle/hle/hle/m: newest c has 2 result(s), a has 5', + 'hle/hle/hle/m: newest c drops result identities from a (3), b (1)', ) report = fr.RebuildReport( repo_id='org/ds', regressed_series=result.regressed_series @@ -1498,6 +1517,122 @@ def test_newest_with_equal_or_more_results_supersedes(newest_count) -> None: assert result.regressed_series == () +@pytest.mark.parametrize( + 'newer', [('bench-a', 'bench-c'), ('bench-a', 'bench-c', 'bench-d')] +) +def test_equal_or_larger_capture_cannot_drop_result_identity(newer) -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_identities=('bench-a', 'bench-b')), + capture('b', '200', result_identities=newer), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert len(result.regressed_series) == 1 + + +def test_duplicate_result_multiplicity_is_preserved() -> None: + result = fr.apply_supersession( + [ + capture('a', '100', result_identities=('x', 'x')), + capture('b', '200', result_identities=('x', 'y', 'z')), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert len(result.regressed_series) == 1 + + +def test_result_identity_order_and_score_changes_do_not_block_supersession() -> ( + None +): + old = json.loads(captured_record('a', 'hle/m/100', '100')) + old['evaluation_results'].append( + { + 'evaluation_name': 'other', + 'metric_config': {'metric_id': 'accuracy'}, + 'score_details': {'score': 0.5}, + } + ) + new = json.loads(captured_record('b', 'hle/m/200', '200')) + new['evaluation_results'] = list(reversed(old['evaluation_results'])) + for result in new['evaluation_results']: + result['score_details'] = { + 'score': 0.7, + 'uncertainty': {'num_samples': 20}, + } + result['evaluation_result_id'] = 'new-capture-result' + result['evaluation_timestamp'] = '200' + rows = supersede( + [ + fr.record_ids(json.dumps(old).encode()).apply(make_row('a', 'hle')), + fr.record_ids(json.dumps(new).encode()).apply(make_row('b', 'hle')), + ] + ) + assert rows['a'].superseded_by == 'b' + + +@pytest.mark.parametrize( + 'field,value', + [ + ('evaluation_name', 'new-benchmark-version'), + ('metric_config', {'metric_id': 'f1'}), + ( + 'source_data', + {'source_type': 'hf', 'dataset_name': 'bench', 'split': 'test'}, + ), + ('generation_config', {'temperature': 0.5}), + ], +) +def test_result_identity_distinguishes_published_metadata(field, value) -> None: + old = json.loads(captured_record('a', 'hle/m/100', '100')) + new = json.loads(captured_record('b', 'hle/m/200', '200')) + new['evaluation_results'][0][field] = value + result = fr.apply_supersession( + [ + fr.record_ids(json.dumps(old).encode()).apply(make_row('a', 'hle')), + fr.record_ids(json.dumps(new).encode()).apply(make_row('b', 'hle')), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert len(result.regressed_series) == 1 + + +@pytest.mark.parametrize('results', [None, {}, [None], [{}]]) +def test_unreadable_result_identity_cannot_establish_coverage(results) -> None: + assert fr.result_identities(results) is None + result = fr.apply_supersession( + [ + capture('a', '100'), + replace(capture('b', '200'), result_identities=None), + ] + ) + assert all(row.superseded_by is None for row in result.rows) + assert result.regressed_series == ( + 'hle/hle/hle/m: unreadable result identities', + ) + + +def test_count_only_index_rows_are_backfilled_before_supersession() -> None: + old = capture('a', '100').to_dict() + del old['result_identities'] + row = fr.Row.from_dict(old) + assert not row.ids_recorded + calls = [] + + def fetch(paths): + calls.extend(paths) + return {row.object_path: captured_record('a', 'hle/m/100', '100')} + + result = fr.apply_supersession([row, capture('b', '200')]) + assert result.pending_collections == ('hle',) + assert all(row.superseded_by is None for row in result.rows) + filled, count, pending = fr.backfill_ids([row], fetch, limit=1) + assert calls == [row.object_path] + assert (count, pending) == (1, 0) + assert filled[0].ids_recorded + assert filled[0].result_identities is not None + + def test_retired_fuller_row_does_not_block_series() -> None: rows = supersede( [ @@ -1647,6 +1782,7 @@ def test_row_ids_round_trip_and_legacy_rows_are_unrecorded() -> None: 'cron_adapter', 'cron_run_date', 'result_count', + 'result_identities', ): del legacy[key] restored = fr.Row.from_dict(legacy) @@ -1757,6 +1893,7 @@ def seed_legacy_snapshot(files: dict[str, bytes], uuids) -> None: 'cron_adapter', 'cron_run_date', 'result_count', + 'result_identities', 'superseded_by', 'superseded_reason', ): From f6a91d3b99cd434d4d14c20fe5acf49277ad9c95 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 15:37:31 +0100 Subject: [PATCH 10/16] Read Vals page defaults and complete table payloads --- every_eval_ever/adapters/vals_ai/adapter.py | 45 +++++++++- tests/data/vals_ai/saved_pages/README.md | 12 +++ .../data/vals_ai/saved_pages/medcode.html.gz | Bin 0 -> 111976 bytes .../vals_ai/saved_pages/medscribe.html.gz | Bin 0 -> 98297 bytes tests/data/vals_ai/saved_pages/mmmu.html.gz | Bin 0 -> 24346 bytes tests/data/vals_ai/saved_pages/sage.html.gz | Bin 0 -> 63815 bytes tests/test_vals_ai_adapter.py | 79 +++++++++++++++++- 7 files changed, 129 insertions(+), 7 deletions(-) create mode 100644 tests/data/vals_ai/saved_pages/README.md create mode 100644 tests/data/vals_ai/saved_pages/medcode.html.gz create mode 100644 tests/data/vals_ai/saved_pages/medscribe.html.gz create mode 100644 tests/data/vals_ai/saved_pages/mmmu.html.gz create mode 100644 tests/data/vals_ai/saved_pages/sage.html.gz diff --git a/every_eval_ever/adapters/vals_ai/adapter.py b/every_eval_ever/adapters/vals_ai/adapter.py index c6f76f8cb..ae43ab515 100644 --- a/every_eval_ever/adapters/vals_ai/adapter.py +++ b/every_eval_ever/adapters/vals_ai/adapter.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Any from urllib.error import URLError +from urllib.parse import urljoin from urllib.request import Request, urlopen from every_eval_ever.eval_types import ( @@ -114,6 +115,11 @@ class PageStatement: 'tokens.' ) _MMMU_SENTENCE = 'All models were ran with a temperature of 0.' +_MMMU_CAP_SENTENCE = ( + 'We established a consistent baseline by using a default configuration ' + 'of 8192 maximum output tokens for all models to ensure that outputs ' + 'were not truncated.' +) PAGE_STATEMENTS: dict[str, tuple[PageStatement, ...]] = { 'sage': ( @@ -128,7 +134,10 @@ class PageStatement: PageStatement('temperature', 1.0, _MED_SENTENCE), PageStatement('max_tokens', 30000, _MED_SENTENCE), ), - 'mmmu': (PageStatement('temperature', 0.0, _MMMU_SENTENCE),), + 'mmmu': ( + PageStatement('temperature', 0.0, _MMMU_SENTENCE), + PageStatement('max_tokens', 8192, _MMMU_CAP_SENTENCE), + ), } PAGE_METADATA_DETAILS = { 'version': 'benchmark_version', @@ -356,8 +365,29 @@ def parse_astro_props(raw_props: str) -> dict[str, Any]: return json.loads(unescape(raw_props)) -def normalize_benchmark_page(page_html: str, source_url: str) -> dict[str, Any]: - view = extract_benchmark_view(page_html) +def extract_benchmark_view_url(page_html: str, source_url: str) -> str | None: + """Return the full table payload URL named by the benchmark loader.""" + parser = AstroIslandParser() + parser.feed(page_html) + for raw_props in parser.props: + props = decode_astro_value(parse_astro_props(raw_props)) + if isinstance(url := props.get('benchmarkViewUrl'), str): + return urljoin(source_url, url) + return None + + +def normalize_benchmark_page( + page_html: str, + source_url: str, + *, + benchmark_view: dict[str, Any] | None = None, +) -> dict[str, Any]: + view = ( + benchmark_view + if benchmark_view is not None + else extract_benchmark_view(page_html) + ) + view = view.get('default') or view metadata = view.get('metadata') or {} tasks = view.get('tasks') or {} if not isinstance(metadata, dict) or not isinstance(tasks, dict): @@ -391,7 +421,14 @@ def extract_collection( source_url = f'{base_url.rstrip("/")}/benchmarks/{slug}' try: page_html = fetch_text(source_url) - benchmarks.append(normalize_benchmark_page(page_html, source_url)) + view_url = extract_benchmark_view_url(page_html, source_url) + # Graph props are projections and may omit settings and subtasks. + view = json.loads(fetch_text(view_url)) if view_url else None + benchmarks.append( + normalize_benchmark_page( + page_html, source_url, benchmark_view=view + ) + ) except Exception as exc: source_failures.append( SourceRecordFailure( diff --git a/tests/data/vals_ai/saved_pages/README.md b/tests/data/vals_ai/saved_pages/README.md new file mode 100644 index 000000000..5a0cccdbe --- /dev/null +++ b/tests/data/vals_ai/saved_pages/README.md @@ -0,0 +1,12 @@ +# Saved Vals methodology pages + +Losslessly compressed HTML supplied for offline verification. Each file is an exact +copy of the corresponding page saved on 2026-10-21; runtime code does not read these +fixtures. The methodology tests verify the source sentences, not snapshot row counts. + +| Page | SHA-256 of decompressed HTML | +| --- | --- | +| https://www.vals.ai/benchmarks/sage | `fb3f238f596cc1402e066d95d95a2cb71936fb2f0350ba3a3376dba78714ca6b` | +| https://www.vals.ai/benchmarks/medcode | `e64e0ee3a4fdac087e0580855eace09192332b5e91b82771e5429cb84a649852` | +| https://www.vals.ai/benchmarks/medscribe | `e4250edb7f804703dfb56720f014433112fc4e583a2e4b6032c8b12b3c9d3c66` | +| https://www.vals.ai/benchmarks/mmmu | `403785fcf9a50f899cb9f1d75026bada929f9d37e2b9296dcd1331494631ed87` | diff --git a/tests/data/vals_ai/saved_pages/medcode.html.gz b/tests/data/vals_ai/saved_pages/medcode.html.gz new file mode 100644 index 0000000000000000000000000000000000000000..d63e79d162da1358ace13e617238100eec12339d GIT binary patch literal 111976 zcmV)6K*+xziwFP!00002|LlGFW8*~9=5K24zx&tQm(o04gipS} z-%{u;XNLoSIgqAa>dB{3Jn`i)n9b9}K^m|9fiwxdBsm-`f~7o{lS%CROX)Q9FX=CN z5{1%fv`pp1TPD(pH#wWd(Rw+RPrSq*JdwVb``+})7Yjf2q{-Zi6CWD>`@2`NGsvDU zy@h``I1l`bRTQVtAGGE#q0~h%P3MPG|2zPY=$AbySO#g}g#d>a`iDeOp%s8BSf!Ge zTrDS35>E~X^E6#0`{VJ&#f379qFD(2%og6#oB6Rai5BA-d>{Xmd>Kp+zmlIl`{xhZ zx38alMWC@UYYhO;K71lc7r}BGT_}j4*WQ&MONY%zzx*P-|8M|>PuI&y8bnJ8`xuU- zkMK;Aih{~&oy><`JXc0bu`f?=%X-=%R}Hsn6;I(53Qk{OR*(>Psj7ax!0d@mV50{YF~A zqcD+Rcng0T01=?VG+54n2V&_Y2t)X+NYA6i%8Px;n@rZRH@VuA{Bti{dlWB|C`l8^ zTTZ3ePu3xRp2Sg-NZ&kvArn=8{zjSx-fS5qK?3XrAWQ%mvnUSyM9Cpv#nH--)2qWl zG}}*uH1s*3zXsOx%pma&PvQ<|MYM$&c4f2XjFlcmA1UqD)i+vT2JOO ztCa*l`=HKrO}&JFMJx4!4XC^jM!5tmp&lH>uQgSKU)x+R=ZxAF;ClfzOf|JmH6~(W>Y#r>QK?@iJ zK;Hm9P6K}mEoiD{;fRP1jh=gP;6ar)zL%_HeE)v%G{XAN*j>NPaA zgf}0q_rwYP48}U0#OuY$>mb2aKtV0=b;7_3?pPJX>vGmt8kptl6*oW_|+&4$ZFJArt1!)De(myR4lZGkGmE5!k zmXRW`+V`Z@o)jO@XTEZLeDni21@xOF4KsfM1`H;_O#19Isrm%Uqm=Zgqf(`MtwZQ^ zFMkPL1n}hjhw>wIk8p4WpuwJ$>`7h@EB?1}P%6V<|Ixujs9ON2n?S|f999-%@i zw{;!`Q%RLR1upZXFQp+Z{@(k*I9=M8h8`k38~UKeV{2vQM_|W6k>VqD%2sKods2$9We=rclCDoi zFd)gVwN>YA{*d`YYK|xac!Po=0p0Lq;A1!e!w#n1IWW{)=CgsZAbk@9P%}^p{^r9m%zl6K4Y=euL1hkOS}%*z zn~#+euVM)p`xQ)a==xRY&1zrYdRHiGL9%BLUoKJQqosq-OC=L&y+Ui@r5A@+%$_K! z#8DXH%)!Q+@2T+>Eh}wB%cFIC?q4OZsiep64B~I5*huygKRyNWo%?URbTS7DlKPJ3V zXh_ka&~mMB^ETQlcD}^%AH(Rt51FLL0Yyy)5x&WYoV_T%Qh|AO3^t+Ug@H$5IA%*` z5=g>0%~1WlKZ#;$3MXl_YCfzzIKrwxs*I0cUQT>!c}vJB@1iI?@hGz$=W7s@kgSon z`J?Bt^bV#1)j0*uP6L>1-^J2ZEWH7hyMm!Ek6AYe2tJ6`Y3a$_OOD~GKdlzrJgyc; zp!`epNCJw81ZaPZI$@mvyp5%)f3luQuchzlFyDeR{zKqjd{bcL z9T>*`E8m~u%5UZ28z02W1o&!@nG#JADGm1Ovab37m8)77MN8?6!uqH@|9qqVMy2Po z;d&3Ok>!Vy5Jkf&vokX5X!G{EVqc*NcPJ5NTU?KZ(`Z61j=h@k#&irVmFr}bhr_{Y zJ~=-B`t_Bap1wVunvs$MNW#{0LV6k7HoQWh$S6XoNREj(_>{Xso2r z`Vi}nM!Bk6uEVgSrCE^9*C#)Y!MaJotfY`6KaK&#pbk^$=F2}eWSLf`(jf6c+@#fKv}COuG8!PT@v!-9ozAOIpaaTH;73(oEUuFOhR3UV zSF1QU2Lae@(w95tw zwSfi&v*4Q&E%UBG${v-;03BAzq9hTR&O_wdk#zW^BE_&xFt=Vu7k>QQOZ?#o#KR<9 zPyK{dXfc<|oktTu^pQUS!JJ-wyI!2oU8wPoCqGU<>eup@@JC~RjO!myGKR&D`R#!T zgo5lju2df~lqm}|H-_UMr|*B9O7e&Q`Efd)jWW^3hFA6Hz{Z#cXdU5z8+In^Y6=rN zMV>7R7p~3t#~0%n_%nl1-KM5K0n9z7xTy-rK7MhSwU|4gvPAUk<>#L_joqv*21`~2 z$p4<`dlE6=p9%jg_-DgE2mZMqc#SaLVIX{-xAUY5%7`vC2FtZyVW0dzI-K>QuFu-s zplU_Bb%!ihg#ZSBz^%i|KglX|l#yl5J3a;&By`9C3FE*a!@zsm)`w9`M}WFv19Dnr zB>jahC;(F0I+e06|L0KB4q8fJ53JM^$sJ`Cu{S*3=Ef|=VUIoPX`5UBL^rSzZob<6mJ^yiN# zQm@O(%AfwHQgJ^Mp9Rn*ICIUP{mrEIMuZPJ1wcAv)<#)KYzihTv}?PX+LWMev9Hy} zDDtuzg&D}Oo&8}8b7ga9HQ(?nG67pGEc5Vct~5;e;phO319%R~Us@+X2zZYI$w!ws zx1P&wY`mD^M|AnnPd(n@y764+(F!&Z6IoMr z<&^cqmW!aDVhFW$W!z${n>Sf4kWC?7SIl-t`Gq_*o!B0DO9|YtgfbbRZC#!EaqP!$ zqcE7ziMWj9{9&!#VkmfQOYv4Ke>r_V55nm%Yi-oRLsb^a80GjFaedUF-qG9-!=tO^ zB-fXlMn!Al`gL3}whGX_>6ET9;zkqh1P=!uZvO6{(3NXBx$ss4+^E1ZjY-UCyYw&9 z7g>9_0*yr#aANakO|Coh{Z%npD3nm|Srmo7x2(?pa@z{1o%rg=5B&)+>2UC#9>U8R zR9<@L!HiBdIl(4B4ElTpuR2NiwB2&16{2UeHPtoGE5->j0|zyqeO86EKEod*YiO-&SUjZQ{V&dh&B;pY zjlRJb!vV@3kXCrkAdF&}ZmRqH5+3*&HT89N9_Tfm? z@!%a=l`xWm3(9W}n|zin*T6*fr|%F%sE?{LtSg{Fif@}er&*;^KVM3NkevbCue1!N z%pkc+aEH96y*HId)!vh8?|XU6E054+J)pV3%>o+L)83@T|6 zo%_Q9lCZ8J=^!>V9F+;0>pRY=uLF+N!=IrWlkx>rt_jJE1~?819NnWUJNXHTiP^veU-JE$wBXtZCHBG@HmdP3AK#yrS-U)<5+Rzpe=_%$ zGi3MnUC|~qZ3hW5GMBlta2rH)n-?3)Vsof5Nv}d5j7C2#PF@~U&Kw^n;d*wgJpb3f9QSp2 zs!Wn(@Z^(isy_2n>!szs+NSE#pG9c^w(N2ydx3m`<}L1Sd&;x*`uLyDTQzTuw(*2n z!o@i+c?!l$5Po2f!SIy*OAmKwl0)RO46dTje7O4DeuudObxw`_tv_vqF;HH{AQ z_l?$dX#G!{hG%gwWwlqne6h<%%iiM_MVh^qeO z`mc04NJG5u!yZev<*Y78A8)mD*0iN5=3fprUBRcJWvi>pJ%nX+u<$OkHzs1~ng<Tl!(vmu7uDiDGas_6ZbBqA-}^nWmwfnbPc~yov+7`7i?R z1K@+7aZNFM1y7(&yt$QTZR+}liDA-vIA0C|Xq!5!`|Oi{kR4FjM}#n}OoP~`T5uma zUoVyiGjFwT0Bap&(`#RcY0HP?i+>pXkFZu12{`>Z8oOF83kotDP;7wmzlst1%16{4KZ zHXMbM?lg=ph*)X|HF=6!lZDR0OXZ)gsWWy(>& zI*6nCI3<1u8U5&tsmmN%emhtUiVs?m?Gfd|ir9PE^S>{Hw0P?5p->1`#RK;vBdE6s z!mIuKMu&t~I`-Cqynw%SlzT_7%GW&h-uTNf+ACgHxXS?U!jrgcjRLI8#r}G=^5Y52 z_+9viJG_rw{Epb|{M0wKiENKHiz4bVoU-s05TSza!i%pGP6c!O?XC=egii zB_p4iyY>1f-LXy#t1O%TWH^p%y2Q)6{W_RV{bk;{Nj{zjNpKQ`L3*`6r|*z;*Vx5r zcvY0M3;siyc!0lT;Jy_q37JjFgvlWZwS6XJC2|^P6KH4dg{OHtj@}f5YE=l98K7N>o(JbZml9WFF9nKR^06jM5v?((yf{H^ zmxNKO7KP1^GAQMEU;2}7Td$rw2eOREY*s!zd8$8q@zTnwklad9)uM`PJhz`}rqygUG%j4U z)VSb$g~$C0nB0#M&QX|HigAtiyQxc5y>7OMrfa-~N-wZk0;a(^z2!OelQS^FaSMd)eH4KRn5@A{v~N25WH$RG>QqTs zPnAOdG=)FZv4X!pV^h)`y2+ue2PT>&)zcL2n`jORDzixLWzmR3#rcUdH1m2o@?k zysg^6Op~Fig|PB8N^x2N@=1A01xu7BhW9a0`7pnpov~1Az zUoRwT5nd)^1NTY%thY>a;gBb1D5cSlA3WJykNjf1%3B2<$UX@Xl3k`9gMx)FE|1^9 z-#0M9-{=YfG<(UsHZ)fLk3?Je8qkZjEtn7`@DjwI7j8x1H?)i0i94ZppaZk!1NY>DChm+>%Y9)Qo`q^Df&Z6@g z_%rm{mNWpxBQY!3xI8&+5$b>@EhvS|IjonA>$4S`E}mQ?$q>709$o+UV8S&wrdz7?$V;c z9(dG~>VE*H&9{85coCfhq0d<=21;i%?_C(a(VB4ovMP7U5m(IFa21NrZ>vBwXm1 zPGfK3|DU9ME)8%bv2z!amr{&Dogkrg65m_wNgfSuk}c3Hk}Pw^$9kfBDu7~SOQXuE zwWaI3S(5$@tVJ^tWjiw2BrifPGrmSrX7gVH{#DFuE{1rb<8biZ3ix|-QzU!oU#@@+ zd>St^L`J4haW^%N!lXQB@*5`DO&7TL@0=4H`fl#|+*_Qil|B@UI08N(~pb(_M6@0bek4FErb^gT|yXl5OyAcdT zV?f~Fn?Rx83<@WM!V!RiE{4D#Hj6eI1jXk|I#ck@3=^9934}tfRaH!Nx^QM{ULSV> z=d#*5NJA~D_^MK<@9P!&sZ5&*Dw;(KLbuv53Su=A?{b=dNY*E5=x?LGGHqGt*UpVE z5#x!V@nq)ha`rLHMpJ$)um-<~ZNyMP_$CsXM`$Kdf#6RY7Qt_9ORJYsqB~o^R{}ox zeXCvy?zBtwdm`0?+nDMl*5rg0C$#pX}H;7`lC_n z!EGJ&(rU}?qTds-9>fk&zfV=#DEgyO=s|1*^-^e;eV^YGl^)nuPrr$=e?=nDxqs?@#Qe8C5FIWt}!)4C6sR#U0@lc-+Jeq zh;JAeAgemc+o*&;%A(uQwZS+3a{WA887xJmsm6aQhrd-gYl@|4xQ*a3n{J-5t%BBG zs!eBm4xF%e*<$TClkUc^(xypuEB{&TYg_^v~Ha0dnTqqg@+M|KPjYU|-S zDhx(NPj*SNKH}EcEc7~-k}t|*2%Jp6wA82Jl?5wRIHwLeuKWl@77-x%L!?%dr*>e> zI>~buzYD5wNpC?)=MZy$HEF;XO>1tY~ z?$)aLhc-LZH5Mqr{iQT`9!9{v<=yLjsj;Ko67%r75kD=O%@}-zwytS|;isjsH8tLH z8e$yKX>@@auO&hx3S1Tt@(jb;VlbhX$;897qn~|N8eUxx*FmA$tCg1-hQCx!fvJ+= zs6TRRvnnSRRv1jqfX*94l^X5{X?4bM0yQf*s@OV(4b=>c+XRjA_%_g$;NwsM7P9#d1%I!344;k;}#@ zo_r}LnR_6$%t{3A_9NuLxERI)XV%3Ah9PDYzDSQnv$qH#-~#d(oZ;Wg{S?6ny!VKKKYm#=?*BNZJ0U z>K;qrur9t|#6g;WrtxU@z3b6_@ILx*=+Snkn|8q%-^-E~MV}rHlB>l@6b`G`7<9+>R#z{nrG@m52uHQF=t zM@N2|J+ko8vtS0&0xQ{h3C!sQ-<{$+Fu6l>w&G*A{Fqq9bE1}C=-TE5g`JGm|4Fa+ zqC+G*4DWro=YKd%53)=KAwzZWKKwBBNBb;e==Gk5#ZzSg!-z(E3;K;AE(OFzl!$|0 zeo4V#NA3D$x}OpDG>#T8mnkN$9%hgH>}xur?F}geKn?gCL7yLhPw2Fkt|V~0{N*%} zzJB-Sb*AW9HjQ{q7qmtnlQnS6P*Id3VeRN(I!xz5g88HuE06?B^g{Dz^Q$RKUS5CX z5BBJED0uVaAUGXKF{WCXkLZj^4&!2wwAqwpza1SUpHf_W z_8B7a2p(bXTUuuH*=NJ$`xs5Sk`K-4bTInmmqC`9Z}0(n3V;+ba>!nsyZ58fbp@`; zZNo*=X}1D0qS#*@r190q6+;*FuSrL!7$^JjxMkFCWlnpuT$y` zes_|9`46oZN@WSg&yr0a}w_A3@0~5rdC0@kcr^Lx{!y#ZJ23 z^Xd0drc9SCIa^tum2I+Isbzc#YVFS-`_Tzpu*B(*7kEx{B$Vet86;ri>*?@nZ;(yK zY0+>p*n6L;MxX@Cd7GJ(6oRV#h(xHBVNO-8*xnU%Sk0JIelM-*si5cIT|OURjw|q@ z4+pU|kSc@RL^S-0daUo5!>U;SaGziNYw-d@8%lLHIV^ZYDl_w@EjIs@=l}c0Umrc! zX(M#}hl7vT0|_+s;Q(F!facDjJ|)ff2d_W-?>b5k&?;k}_Sv5bbnMX{`$JvBtSl}i zK>Jli1zmy9>}LsEN;d+(jWEeGJ3pJ??ZwaE;bHO<8v6QhAd~z8A?Pt&nE(dPF9^n9 zSf6_|@y~S1vI5bAAK7bQ*hMhO?-b%uU)KE-yh&9^c3H*~A0?7;uIzfjGC$QoLqit} zB}Ht=>dhs#L0LptWDykyoU2LMj%8jlt5>vBT}!Fm_Qho!giPXQ4v?CO1M8KSDS^Q?7Kgm`EvFrU;il$ z3S3(Z2fA76$wFqLnYJ9dCx%eh}+Ies4G`-jB_0;nS5ctkaz0 z|Gv76?4&fu_0~CFT#CvOtGxL6Z(p4}|MvUapXY2!BB`>;`JU{(f3NOkFJO{^jEk@G zmufbhLC|@iXW5qz*~5axzTL=$qQ)y$LwbsN&ZTorA=S#5jHA(ucXF=DdBN)sd;Nfg z{>%%v0P8iDm%jHC--{=6iK4zAFxty;oYmR}zDa~3uD1a2hv3AQXlgYIn{-Y@Fn12M zKtfFYZD8A;s4p)kekjrFU#P7M#ubFpU!2J2aX7VDhbms)6Gj^+adQvKUH+Z9asop{msue4J_h7yA zV=y4RP-d)VC<2U@=5=Vxvat<-;JIxjK7PRR$a8{-qwQ0%CU1?rWCBWXE9k1&laF`} z2wXq(jGCT{SH<)BDz7nM4Nf0eqg{^71J^Mcoy+Bm@ zPO$9OFwLOXWE+V7{ax&T^xJ4`}AYrG9SSbiG|vfRO)UB@H1E%-_Sf( z(%+8W3WzSKcsyqnkLfDFHek+uFHGkXFW!c~vh2qFur2)QwpRD903A(YFeQaRO3Fp= zNI+!L;f;?eN`qt}02r&WsMWX=W_uf*`7zUw0`?geX4-KZm=<0--_B-?PPdtnE39z* zD;PfCM<8}0q{tCs@W8hGi> z(UaMyqA1GPVtX_t(IoIfb__a5ueRa%^+^~^&_eZ3gEZM{L1RVhB3IJ7@|`Qpqwime z+R{nv|2i{3khFNfcYC_-HYcMsUiRDuu$Wf)mEik_eo7UgR(Vt+p=jaHt(=)Q(7UfZg|iz*cIUSjx>4M{!M`9%HL(}Tz1P9WNohTo7Tt=W(hnf`L$F24qXwe5wZxGrph{iljQ^tQ$6{*kW6_w zPVtP>CUgHhrqPyfwL}-M-w@Zb$BIm^agqkDPmdoXdWT9g}FpvB-~Eu+sx?@Q6Ys zOl^LEKx_}ce9Yj=55wqUz}we9sm%}oYtBNAXN57BuS662!jwy7VznF=fVY@*c!>uP z=x025R>@77$EI~~dTxrV#aCLxz!)sh3p~V7`|6-8usV%7KFH#aHb3^Lpv6S&< z`94Rvvln>CCC1qS4Q(REW--UdVemu{2qq&q(`B47I~U47ZOnN2X0UIxZV7hy9fg~0 z0Ie**#zc9C4XGGVE+9RSU}=zsei7e2ySoYoQ~|g&iSfYklttLjJr@ZCxgV!A7XA`% zNHJF(=T_pU38yPOu$)fjD7>z)$I?L^mV{gS({t4@^;a#Sc`|(0TVlj6XK z(O!L06wd$7NhPFD;>W=$<6-?W9A^=J@R$%jGB=M3j!sdwS9y9P7&7$&-OWiGLHLu3 z&d?&<5bb>#(W14fR-0-3srAX3EURf4t}i zyapF7i&I9=Tx%7-PVKuoWpvK^OSmVEDB&9m24s6TxbCeV_1C$X<6J{ykN@0yn6RcS zJ5>Rp5_Q!!3&PphmdeqtxP1N`W*!R;EhWH56IpWe)&EdjY1{ z$yKxWSrpAe|9IwONO2j%h_6^+H8cl#FnmQzN>CCHhQmKvQW~n-ShGeHC@+;vS5d9M z(6lB!T^h0%&q^6Inq3*vQ!GK79-5BgsDBwL8KG)M7c+LsHWLCTgFrT{+~ySQ=7i~Fjs^~GdQi3E@-vk@3AVNt(>EmvX0}SOBk%)2=6(f|b*>Qi%{-bV z|GF!wRSfF~z*l4GZviT*e0s8YQB=F7^}X!j)Muy2i*1V17VmcRxn)W6IG73na_TLD z@Jgs?Cim-QYqEXKv|YUviUzQqWun&*!Az55bXJ=?Wx+YLRjsh?A}ut3mNdwUHk}tX z6xDa-$GD!Ip@Wj6s_)o0FAPq+6L0+A3xBE0I63G*RU&J}KwAo*hw>@}IS3U@S#^6W zC??B)&{N8yt@aeM5!9YCj;z@yy=A!VEK|&JG6P|&GMHt}Qf-lP-l-f&LzOL%Qq*PiQeDm6snYgc zisOQZw@EYyMULr07GOTe9utQu1*pkBQ{P`DzJE5(Kgj1sCNxxACt~G6X$lvVRfp(8 zrSr@*jly8H`WI1rmW;#DTX+UYOHeSu%vuNNug`+WU<=-l77JcdDiIDpb9tOMXaS*__2H_!g+NO&4NJrfDeXyn-UK#+0yq*+0%@Rn_hx2$NY ztm-f$#7gU(Z_UoPR_EKMmJXH^pZYxGg&$AW@fA+2Mya!45sdMde1`L`b&6KXSzWK2 zV?=*%yjUl`d=a5PNj=*3Qpd@vZ~@!jACqIPS%xJt8yAVpN>}E(3fcXIIKv7+t$3jq zjI&R2i0PVSL!cx}`-bzoQFwVCOapJc^cN8ZaR<=_u6XRprm3FDo@qAS1dJ04ZO>L| zW?+Cel{qdzr5qErf`ymfG7>GUFvD3UMCPdUN=Yp%sX^iHl1Aq0=`zQHua(S!rNv1P zusy9=a=2c4ePxm;>?@M`aopLCMYjceiu3`~3yKBQ=_nHE8I?i4TCIhQGV^dtuA(SR zf~h|ay%~TkFF4b%Ugk1UniD$3^OoSb&aaGC)tbw!g*SmuelmuyAy0<26_k^O#}PWa-<+zcpIzCaI5q9i?D`SEe;CuvI~@frhOHLs6g5Q^&B)uLE~+mwrz;8e|{ z+WB`p=op-rm)$B|W@B!7n?|V@9#1dIT0Qd-NSEBbk- z7`AL#eyf2&Qfv$HF|8`Zv>c--=AF-ugXL9h#d{woGV?TuXUU8uN2JUEP zyPN}+CNOaO!Ek(aUABAZhT@n`>(Jq~W|H2qYnr9skC5xzVdx$_Fp286)182`;2J%n zCki2!vr!cH$?~>1%j=5XK5|zfmeJ4gx})plA#uDTHgb)$OL?M#6eRt2jgE`-55lgo zwjnw(_4c7#iskf=Tr~{sfe5<#c1HK;wJ!Ch>A>`Mzo^rJIox}h>^(~;ePg$EtxL?C zhSg7bV(1UUwAs><*A(J*C^=C7y1?i)M>CuU!S}W8?e4)7plt`=YnEaN347w|_hQ#^ zmwo$&thYNbMAH@36_9hPsT&V~+bfGc-Qz~@qn*;WVhAWU6;y=d-YcP-y8xZTHeAK+ z5N^Z);n#1%f}3u;5674ho7^yT+wEGRu@uMd7j2g5X!kR@dsh2<2XD0VI{|mz5^!Er z;yC8LYxKtEN#D>d-DuZnj-pwPfF);^oWEaH!ZkN0`^IZ_SgslY%T^ayj8sj-y%#=j zZh!WTSMOMP)h)$!1$4HnJLW^s*_*A{-cfJN(8j1OAfPQ^sJQB8UG^b(rL70|`j7z< ztP2^Az-lBII*#$6R;8?Mk)U;pcul~R(+q=X_pjdsHF}5XYPv#>8nIN{bnkh*m7V17 zF*{{mZjPFmAQ!ZLUUv*zv+jG;aI4cGk0d&|sT7?Obnx_!N=!8o8fs zCefQq-8eA8W$Rd{P*jm4G^*v?k6qV(V6J=AAjYjOd1nc2F{fuABER(>7;l`vzUwF6G$oj$+#aGhDUx2eMw#e+#V-A*5qb(l(URae)+k^==Ma%Rj?UHt)&ymSV? zchscAjt9@42y8qUmU;in@21#zHMf0P%Ea9Wft6XqF|2#pP;I>=&^v0QV-enU#k88G zsph?NH~%_M-}qId<5;8vww;LeW}5e6yfrQzbPpC++d8JS1tjRO_adNUR2jTzSzl5b$~tK@#x*gJzI^bNghDb8}>wZKXoST^eY==QcSZoQ+n+K;vCHt@Hm z35a#|p0-uQV`uf@!R%UY*Kh|xfcl!D>JQ_%cSjheUQFmt7ba{~WJOxH?&pe2ca*Tc zp&Mq$uBroy(-N@ntp{;=w=E=A9}2n-jcS_W=(-T~E%Jcp!L}&ryzoXuBm-dNeira} zF_rqpOFAz5=qlJzn#jsD(e7strZd`9=g@Um(MkJpIvYlA3Z1OdZ1etY09()o&>h7# zN!J>!EpnL3(DeJc!7Lcdst*U2M!I%mEV~Z}^^x7P%o7{6(G|2n14&?t>H7QLgDM68 z>l?Cax0^SPi@vJB1p$NHlcCoP^w&Gy4g(*zo;88}Cr#5-?IEpLmBVRuBfwT1w?lus z;NOeHBq5e{KYLJpF(dm%uXi0Vu@zJ36p=;@{r)vtITCvJuq_~Z$C)up(KHc7Zd&%e z412pU!+oeQJI)zsM9~Z)w1DS056i)C39Q|X4MTA|?l~e6e`#GV~ALF=p_-4n_hM_pN zz=SvT`w4W=kvyh**qY*YM0l`taES~NH?MTD=sxza#P0dSXzi}^37QVNP{iV~Z1R92 z^>k#h=|hLr;qU^_myTt2IDo2pKi9?DgBSM=+i;Ao$g76oh%v!^9o1>ixYRfD(()rG zE}v!CB1VmMPa${v^B(jKzbobevBbiZ=@xMw1iy<6nSi(3>Ezq0A>`!W&jPMk7PUSU zSRHHFE_i(+4i1Rd``^6hCY|dWz20#+38PmLkytOpu*sxJtke30UrVXenu27_2&Ap$!zV;MV-NPp3vn_4D$ z%=Dh8ny)r80ie=GR(3D;)2LQ3TCBWD`nB&(%UPyqcC71xlqB)tU?5Gxz+a|^gD~=@ z11Vai$>AWQ*wS0%<6?h~CSIEQ@oVqOkN**St5x1^o{7r~*Le->272)>ydae)^B|nY z{t`NT^2IbbhhDt|I$B04E(RvPbQ=1X^p`w|Lg}xQXqkG!(vPJRZ*m64#u^yx#7q2Y z{c5e5w~{r0(RSkKqV(+o z2FbpHhFc^uULOaT=#!t;NgAA9$tQk#;rmPW5J4zVQugv zW8p8xtSxABV94yarjL_j$aF0XO z2UTD~f96B?SWa3@`7cj{**Zq{NzP}|Ks;Ly3WO z5lqwh;eZ$eY3>KJdCESciSsPFJRGQ!DrpA%KafsgkjTqRA7&Nc>dZeJfUE<4PoGC& z6z5OnyiLU{9wF@}-s*5bF<5#0Qxq(V=lNjP6VNjf`y(%+b6-A3qyw&DuEl{T&x7d{ zCmPN_812eS=hF0W@P=rTcCM-LXpGM`Zx)V39Lc~R$~n!~7QVsvo41IgCmA81K7aoA z@1H*ZS5DUwMPnbNdT{Pb!4gL#BTb=^lXaR#ODTZn_ha*Wvf3X=>D3C*#@^&Z@sd@v zS_5*zAW2W6%j~&-3H?lcs7*8GWlvFL#Gr@sHoqqO1=?Va_>)vh7Se1EYJ^P!?J}vc z3M(d4F~)xtdMAD;En=Dc4moKa#lg=gczbwh0sP`HevvpF^G1`1Hs`XMB{__f+IXod zW8(D^G!AtA9Eetkjb_1&IFq@54wCBkMVV|+iM(FP)97N^LmFEmrbK^alS2Soa`6qm zZy~^#O~G6|mw%TgcE47>pm`fMdaX-wGXbBZVAzzDxO~F?TvoMtsq6FLJeanr`Cbj* zs=#$L5%Q)QYP+Jb2^LjBUAGO7DzFR}RJCu!A4lSW51k|^*eF?f>MdD-s6>PFaPV}Q zN?!pYR(%Su`7XKe(#c#xR=mQdN;sV$H}C(6ShoG@)!<2W))ifD*yQ#G!?w)tVlXeB z+O}zN2E)3%^})RG{ne50pWVe+j2G6kS1;>h$$Px@p?pP`eC}c(>MQq!>C^|p`n&aE ze3S1^-N{@b<}34AeI!Mfw?3FRe(K!`MI6h0N$TUk-fn*g?CjTFi~>mUOjR4BU>)B2 zK)ze~%ctMm2{q_{^>MKNZhaU>D{tce^KJ(6)P1HsdtM(%-s7zg<)8HQ#a#^K`HN@v z(}u)i{qgOll1UK7G?!HngZoi+bg7-eUZZ`MU3KWip?!pdldVya122Ei?W(?o$|+l$Agyb`4ZpMxh~(dQC8m)^z^ z{G-pMp{k8FYgC8XbQRSq!>l39L@UFrJBq1sVb*tp*^s?>hDK%vt06tb?+w@-#Zk+! z85Eyt88+KhOq~mxu`{u=tk?y$U(xB(Ah&5LYK7b&SpYt>0-sA1!{EYa?nv)yoQDm) zrC0b|8WNzh*`zb3t1knK-Z((3mx0BY)Fu~LYggvX_P$GOX)-`L06KaFn+!m=GJ7z& zJQqOwW>edc?ds$<6uSYTp*U90^rk7`xd1vhWk*~SsWPNO8Xy9CNgqq<3!M+5yDO3H z29fO+B4Z+T4v}r8fq_l_T|q(K4@v^$VUNI^N{U1p5*`OGo&b1C(AChTo2#R&>ci_ zy*8n-P;I`6t?ndJ%9)mPk)m+`vui@xW@I(0T&K6dMs|U%bdElQj02Y?BI+DPYXMB$ zF^QwI<2+9>fCHa+b`(ud+z@D1%|zj|m?o?gFf7(_mP2^(k&? zXkq$bJ}}0PQVTQumL-=&xcsD>Ji4KUS%!}$`sKrC-qc#+B;&2yPo^|{m%SQs#z6Wk zW=ze?S6!NQQ}C@;sI4BT^*%R#q4PDFX740o%v`SU<}Xnj(CAfNu`pWJ6r3HP$x_i$ zi0}&-cuhS4uDQ`;*|AKte7M|Q;9|*QGq@yNaP&BE8E&5g#FEAG7c_3{U|1Yh`!-sp zt=xvWU3<9AP}CN1cXl?MhO^=Doei3UwcpvGxmfuy-e6&?d2YV4zQtBNUc>8rnX3Uk zCQq#zrkVDug735$`nrB&SE=DBaey@YCX=>kS((v|EvHf?9qHgnq;tU0+y*#KWd_-8 zuBCosox{l!TAxy+OL}9%&CvI=F7Qn6TR>P-qO z$CZ?ph|)Ka^aFh5xp$b((yn6F&`stHJ9c$Sn|!;?Ps_#$Tx9U^WeV$vJUhTKFW23mI1(xSV9-@)e98gVsP1$sI%Pc&GqrUa zl|L0svg0h>=n5g(P zwS3*ikcOA@U^eH0Sd-bzf37Vuwjx(pht%UoA7}BS2T#6XA2^T^x523fNXeQ6YJtqY z38&`HeT7YPm!1cy&9M<`Qyf#1@eM80+1cXHpxoH}Z4g$E(O_x`X3uIB&zYtXwdJ&+yAwoZeCD7v zd|=F6q;Xa)D+d{A{v!T4L3KAlY*z%a%@f2lyAa3HtL>D~EJJ+QgmoE91a#b<2x#!+ z+w7gBOMeeJJ7qKvQ2Xf^(yGdA?S`|cWA1l}SDL~{X!;YrnqiE+aY<|-c97N9598;gKxOk> zH_ZaWhfP>_6p31nyA!nxp3vxCfY>)#0S?G!ZN)*$FzQsyX_~Qx54L-AyS0&2rD48y zC~eHH(tB_SkE-#{f~!04=xKPes{D-unPCXbEw%NaI=QKp)S0Y=BmLS#B>I;mitaDY>teyn50kru4$x5v>OM($Lsm?Wfc4@=QoSb6K?2aRkV zgC~)}U9qP*JgWMP@Ycvl+wmC3+}gHyGy}ETS=Qz<8)jT+0ae?T-X&T$AR-OZ zkH6+j(<1QU(szH0?0|k_z$EAE~b3oPW zZWQN)ddqnPS6Js1sQ`{ksQ~c4H~k{KOaVWXXmaGVZ7$xfdwRY%x+BUEa2o}pnX2n&J6_SX|SxzOX*e6xDQPu z;n#3$F(|A15Ev7muf|<>*L?`hJ}K=(X!f%PbPhpoCkT52B0nkS1(`8zEXZlXk;pk^XR9O`9PY)QIbS)FvDP; zU^^_Y77;FR0s>0miKI2gwE1`H&X4{Db)v;8TKdaWUdQ34?`L`6KS0;7BaBv~JX^`( zKW2Z^pyP5q{}Q7q9}cGeskaW(0gMT+zxr{tMHb*&dH%m|{Poduoi+l#)@l>y`=s3G z-{Y_zTznh6R6~%b-fR;*J%3m<^X(?ScwgLfI>L8ea@J@qnMW7zFp|$RuO8%lQzA}Z z+(zj2)%qk1ChubJCx4Oxz5V9k($dSXfvYwJnjzO|)S{k{>l}V)4Y|(#fac9pOi&!EGW+UCQ1%jfyzu z=(b$Q?=3@JU4Fojt(LSpku^gCdz@HOIK$wrDxEqQ);jcypk+lDQ&15>m`F5mJO*n{ zq;kYr)Y5;)0g4cVe!|OQM|x7h|=2}ieM?dM5=le&@3WWhI4L+Sqx~qi8Y%? zwcSM1H3n@pH?eoK;Ocqh>?=m%y^wWFnj<%y%;fj+>||)D$F_ZJs>kDFP^zO3(iEt& zUkaBOEYD#e7~ps|Ps?fmV=*WD}2nW#W$6tBr zf5^gVjg&9OD_|6keHG09@BC{tas1QZ5;)9hwY9QzCSQTroGZ^y!>=cAU(fE`$a?F? zrzrH76Hdmkdg5rtLC>OUUk~B71)sht8FPTb2aRpLr64P=vS%yILK_M?)$E7il zO=hgs@rs{N2R9#E^6uyu?lv#SNhnutrg(kqpPokLbU5Ww=NRcR3DPV6Yb5eI^&3qT zkk*X9I64n1nRM8vB8+&o_*SXZYBEKD-c)dCUCFUcP46PPKo&90u!-^TNuJ>wghrJD zJCeU#rj6z-O8E^4o&|~&AvDH{Y)=w~VInoVjnJCXpFWG|HnHxK%9^E>QuO`K$=s0O znc5biH0tX1{MSS=Ht7(Xgx1u4A#S=vwaKFpzat@DwV^~Pjs{bEDpW%$Wi{qZZ6cBn z-6D?tD8uhUh8HHR7^NMpJ@G3I|Jp@qfgI@eUCZ%55~cZuxFL%qb;oeJW_wyo8cCOQ z3G)6_rEpV} zoy}`&n|V!4y`9<^bi9k$o8`CmscjhAqm6cFQ+pG_U4-0Pm+z(NAjoafWs3>UZ*sGrSD=DHiLuWY$Q4pehH z{4m@|>Yz9{)B;Q7j%GNI0{xBzdXcC>gxZ*?p+lf+mSS{T=9a*Yh)!JHzT*l02b1Y| z;B&l*ATLH~y?sN3Xn0Vu!%)*Udl4PWrlxNEF&4^2<}#*$y@BA)Zg+{07WXLH=P+gt zF>km`3O2suK2mmfDP2t1a_$uY+R_Xx?E;!OV4!u-*6w$&aKGBVQ`!=psJ3=X>bN0T zAJPnNIf){qiZNr1+jSVlQXIR(^3nt>FUxc^-G20~?#8z&Ck7HExX~`RiD?17y0$zl zfn`zRIHvX+a;`2Px_4Ekc!=v~H`EpYTmH4S$+A&zm|1Srg<;BR3*AFaHJ|IaOHjPd5iS!ysb z66+m@MrnR9qidsDKx(_XW8Nu+Z}oK9uWFJ-(K4Awsk9}LYYsD}Toa`AW_`0kYnp4V zV=_Aen;W{TZjPKj^tG^GcQ)UW`OVE+Cq{g*!Mcd=bR8{JMRu)S(=n_+&^~69fA085 zN+b23AmO!c!(8iHD^neTvq*+PH2ZM}`%WA&L3g`jIn;D}HnyC8W1Lv3ZMu)5{q`NN zEG4A~m7TJXZqk|Nq||P9^2jl4&H8O@Z~Xf9+6@WKa(s#r+HkCnQ%9I!r_1ru`yDUC z1mDehr1Y&(x>Z|?5xle~)w~9vsAkt z8;mze;YA2-DVozR#c6){4mzs>=Lm^LcD&sqxc&9pI!DN;bIUs+B9t~2Vs{)^P}Htq zDUQGrDfrlq_PGCXdxD$Aiyv?Dm5QApH86iomt|&k-`P`Lft^m(H8tlhXZ6E5dnPii z|4B);^Jx@IjI1{4I6SI>Ip%b^R;Iw|epA);-+-{b2XlL;(2!s;GW&@OQ_m#!Z)oY{fzR)jut#(uT#VvImzxD%lG`zxMK0c| zj_T@$Zkv{-zFvU;dU(g_~kT*4*~fMkZbi z=r~8Ji`>vN9K#}yX83KH;YcEXnU3dI#x}c_I$o!1U*C4!ZilO6z_nA_(o}Odu}wtw zs*T@}1@?|nY4-~}rQD;Uw|I2CWz&Ac!zs&T&Ps_Pa`MD9t;apATNBqmBe}x`=w+ex zZ9FN~p^+_DQ9DE22v}YYan;|j70zW|~g0nu(^ZHumxX&7Cq{ZJ7B?W}j5G!Y4u;6D1$ zcObci_N$0usU0g|wqn{{^H?)PqWcrqdfb}4b#cwsMpRoDBy=gH32$GcDH5fh^0U5u_ZF@Z%;@UHB~1@7Bh;5M>!+qx`DV$JOecVjDB z=QR#PfW|~4>f?IYEvo5!mulfn;FFKJ%+Er>T`QuHj)Mo54Qjc|BooNNq}l3@r-{g~ zOlU9>++U5(?`lN5u@29V`U_IsZV!{8gT1G91(`HNBHw5|nYV6Xk~_v9Nr^WO7i~dq zwIgJPtAk$eS~Jr{QnqNN`+bk6i*`6&RMXo_Zy6QJo7jS6?l|A?z`WIOc-Tk<3(t78 z%*C&?@a zUTZfCU!@wvRoy!cZe&`&(HZ=P5o}wEdtq?mop|H_UieGASzD~?_0rmEkMynEAh5L# zVwYrnhw-gCj}G<5p=N}-^>cz^)Hb`0cx!kVzw3CQz`B{C8LIwB>|4AiuNB9}vO@^6 z>Qbr;Qe1bsC~m7-2gQY~Zr!@uTphpnxX^8Tmtyoa%#O>H4!CGm*Gjl1KwwL~^V}T2 z*+btuInizNdz+KqYG(vWO;a4bD~q)$HnFWoJN(w-V6@8dZRy^bYKwGgaFLxaAiMdf z!EZqJQZQLf?2Z-~$sHM3bk$T0jdZPr3b|aGyYph`@A6LVj$vdgaL=b_5fuz zklEC3ct4)ln*B)TTOf1Qp|%AbYTZ$6lXM*owY#Dgl@6A4xmddOXmH=epI$;sfj3_I ziztnwrL4>AFpWLgG}RN?GtCn*N?RJ~y0u~1UDu`6KAByar+@TQM&DA3-G=ogMp~mQ z=RBSc7^R1NpxqB9oN5pkJEOi zdpG5)Rk|QqI~K9=-mGSHh3XcGDW;W@HawDbr|VQLCxql=;R)qI({06ZJDgvafNkX7 zaQ+g@GOfpxLbrNSs4S{ay(E=6Bk!WR-WAxwR!q{Rj18eUpu~6-+BcfrRd>mfSK&Gl zp{@mUx+A`nrD&b|k3{G`hGjo4Xx+qLIt^m~B8tzFaTt0F&j4o_APgpFaJft2$kNmtq=Dmj$jru5|v& zpeO8w<=$f3N(1-U z?uyE;+8uYJZIM*9Aj0*>%nCPIOUbX>h$RyTn%Z6W-8CJ2uFmxm5yRTD$pc^Ve#nv+Si0o1z!QF!Rs+y|)L2@D8oh4Sg3Tp};*#`0&PRBBd z0~5N_&bGxSwEO5$?S2myru3NZO@hwM7DzNp6%^Q|19;8*k&0#5UDOpxm_aP#F*QnQ z^6q@PKvQK^SKaOQBel+y@WkrL8Z8oGO1FsfIDc=nA6b{qB9tYH<96;e+iKT263rCa zX|^8Ek=#Twmz^W|Jfs`V*)oVLYq}Vnt&TIgZuxc$C#R86^j0wY9__0eU3sn088DH> zp&-IE);prh5C_j%8#l~cMGV`1T+h4-m4oHPU!_5`954JBXm}+Hi)vl#xJjY7otNK4 z&gUAYZayZ2=Fs_@oMyX++|J|5h=<(P)$%$=bLrYu?9OEnL&&bSwa1W;cakE39s65G z%k4~UY6^EL zHlx#xhR9?yFlw@L5-q3lSq|xGypT@zYdHr>} zw3Fwv;|X+FjgqKnR_Q;dc`jV6no_1X z*(UkZRy{c&@Q7h@~%L|4ofR=W*X zg2&sAs#vyG{OGrFhrHPjOln)vOU$kBy7JL%+j8ADf^WBO#uF@xQ*N2uEIAsd165Oq zV;4WpO%``KbHUOTjn7ZJ+(2t~U=tG~l;=P8c5`hzSw+C6a`<$v=%+1JlcD4B-r~pX zCvm2wh)P5{&`nKo%aYHy&3LoC0l8pvl<;KU)(RA7T7&3NT6ETyd&!)EV1xm){If%- zIKG)xh};ADJX;A0TYBl|CdUnVoM$RV3D0VqiLGTOwgy(VSi_o9fm`1pN-d8u9K|jp z+1d6AIhQ3A$IwbY?c7!oT0(LIZ8VpwH#%=y@i2<^3zxmNJCMj@T@8Ko(oesj5Hj;L zP-J?G2iMy0b_}8GhN26J3UCv*6cz}+#V1~~?WTSa>O!*=wXBt`tuI^hoN*{#)+g3Z z=}&o#Vk&ldQrkOUUf`ObWfmmX?z!K1D6tgH=G&iQF=cq`6Sm@NC6~h3Hs%-)P7Ooh zZ+nhO3> zNzJG^CdWc^)KFbTD+6k7nYxlUcvIZ($(9Sn7{anG#T3{&(cAN1b6T}=xrIArThzSp zk~J}`>TU>s&05sBBbYdS^P}~fW3uW9ZVIB@67=@$ZE)*?&@OS8vE}h~{?w|hd)hqx zBs6FT6Grgu+cw=Ui^&#?MZe}Mz8W^2EwE_Y_$%G!8}^nvOF&Z9brs3Q(i*r)qaBF&em-V z8Pc-FDSEYs=h%Sp+3O{4W1`q1D=y-O85pwVO+?b_Bv_(;hk?l^%0gc6Xj7Qq4GT04 zf8TA(gGC&7&UNsT-R4JNgIkcXEC8GE{kp1jGGs5F<@ZVZRtt&9S~}=MVJ?7qrMGP0 zY>`HtKl{s4)YN;AtOz33R4eBN%Hppu&*Y0}IrU@tbS-E~f{R-}yRngfK^ppv=yv|} zc{KH`lobN4ojzH+0_+jIE)57~X zT1xMV2Bf2E1LYH7-twwZ{~t%ya!I(J)eBWYJWkqrNYaEJs9OYK2#SXtduxp2|Fid| z%Z($+n&|&|3XIOZ*Rt#sNJL<->1yj#F5P)5Rh3;*XV*Pr$7m1Krm*+iF!tPib7|Ic^589I!9%qx=dN$xR%mqSc!}%YbQK3TH%$(HbJ#_G z4XKHH-lT-)jXZJVto(Qa3W);@;D-CJ)g_#pef6!2mtw6O;)cndmU}!_n3}?b%vRHQ zvI>1ftUXgE;H9qv9AEwU%aydYfj00^Swwgl8>Q4tN~r0`Ldqu4Y?p(`<|{k{bz{_| z*<|KL(_pozqq5?ee+|dG#3)qIDPvyH&W%RqO@9Nqg2;-!i}V$|eB=0$i(}^eC8E`G z30x9QFwVQ{haDhxvVW(!TWGxmcxdTl^c#F6eeLtiA$30V7_ezPQMkyxbH7)R%kT&2E z6>c2Sg^<@Jy3U8P-Vpsu+Rb)9uuas9yx*GF$D(J?hw#W zN@ZtoFsiY^5YTWuXi%wXSiNF{5E&Y=syH9|#KU9+5y;6$KIqB-rsZ(s=!h zH0i3$6{s&?BdteIBcfDshN=iZqhfx#LV`cRfzR2|XwRQO9kH||=kixC0=`+N1(UDv zT%X=2LPt{CiS!0KJdC#if_ibU@N$7^t3ZnYrsH}hF4u~WT)OVFOsuc_( z6bykQuM|iA`rs~-A1n`uNzYpW2Wu}u2U6ZRRYIgU&>;>#N-HbXLI@jk(nPqx3~_NO zbfOZSNEw;`br3}98&D2}C+EW`GitRiqY5|?a3bKOA_PCg;l{i4ap_kSNR$<$D8q=9 zmdOq+-7OnBzgN`JB9$>#$j$fNVwGPCPpAX)M0sk?ddB$oK{=y9QAhTmHLgO6U=G0? zb(dyIid~H*Y$-{gRef~^^5v%8cY|G_?Nis0SE@NhxX_&R#T`o_pmB+ninzHu=*``( zw{D(9P*Dh{zpgHQWG$sP(Db`OeRlU~>@2A&rhXENtMJ<`gBeA$bkpe5bC(cdKX1T= zF!<;A;m(WTHM8_(_efFW?`bRP^H!^#2(E$Yor}-_0VM)T4i`0;Uol$iWKDb*psd~S7ogT&DyaiH3zXsj*}Y&8I3VmN zuiSp(c$5HSMB{%r^J^!$Ura6*aJcdxtWk3P`FYc*$$9q^CGhxlgs3Kx5jQC%t}Com zai4W+mmO3$`iK#JRSrnn@O&~~-$vls1bu_ijZ5Yw!}OoHLlcZ57-g4H_UnHorCDXF zVUytH>2fbwJL!!=$4sE?IX&hkD`pj$LvoIBNUxgKg6pu%s?r8t-R5S&twRK2RC~bD z_8xHbP#$npc)$q{IBpL(=A13eO}x0!E|0CAxBLhghEEb-Qk{1LUzQb&l;Zp`CD%6)v(vxybI-wpL|5t8dK^iYOF$@V-l$vwF1B zUs!WsxY&(ioXIIC0%3B>ADuMN+ufn0isdv;*IBFL944LhXLd@z!edxzP~W*vVe^$; z{q$_0E`t!5z=a#P$B@gU{UJf5l;W-~uBQ+|D{$mcIHCz0>D7Lp%F@7(oaOEkS)b*r zJ&gT+m&~EAhlTAP zv~=C+-FUU|?t%8?fL+|K95`clro9ei-mP)4@?U&C?ceOfGumqD}YP6P7k9$|uA{o@sA70h}p zX3fm+0m+QpaDtgQP7TNqmi7`ty86LZO3nrsYnmfrV?XGfIoW!mS0j5IuAUW69L_)6KPSzlHRKR@6L$~Gp37h{Syh-w_aIv-{5IK_Lx%M ze;H0#xjH7M?2s?OMSx51;3DoRn{(c$2{yjL;?igNi?kr}qBP(hm18%1Rq*z`kHP+61ufjWqZcOJxHu#n+B-B8x-Yx$nY8g(sAH;}Fwn_MnG zXT3?6thW|s;Ge&KdjIJyM^=~o?I!Scll1K-d~eqk9V(GQc$c4YtcglZ!xC9uP7ZQ7 zjl|X1b9K7F)2}62Ado;H;We_Kh&C}Y!MrY_Y3N*pFvjnxwEGAv8 zyW{pLX_v1S`w~a+KNDUi&&10l6-bWcX0pwbo1M(XdsjQn24S_ha*&sAa}zA}j998P zql2udRO3$TkQ(JQ4!c~(91jIDMWP3h=s}q94;0(2Z-N$7neql!=m!a`D5Y^g!Im$t zn=&HZVy*;G3?I79LVks-Ro}3&4BdOredqukU(y~903Ri^3J4JpA|T}ODW(#0xF*k? zdpGo^-()W@v~cZ2xl=aIzOP9aStmVKhS=yQ7(%-{sZmi8;#4uDIfVpYu_zJx&2d|} zS64=pFJS}3GBM9(VtiN^XyHSQq>qCXlBuV+d(w!SQZl+QbQ~Wz1acg!ZUoX7G08}e zOJUn46}@^|0b!;tsznHo;Go+dysSlUuDh zxH;2338sGa>0cXo`qfln^U0Kw!X&!vH^DfB_4^k4Tn+Rg;Ov;p{Ua zgSq&WkaXzy?5d2gLM!j}O2BNm65w>bW#LGBiG)S@_kjh99W*Iqb>@bY!Lm))64zOF z&?-#=BZKqaQ2>kp7y&S~0JCrwL2H2d85$1>Fr~!3lzh|zVg$szXov~a4euiJ@dWA+ za8g2uP=Q2EjK1MRl*Qo02W>}#z=DJ}TpgVXl1`|O4mAa62+$CTKzD&X!nR6?oybFM6~^9 zT@#sT`!5X?ELV}NrFC_5>_ux<4d@qWD`^KVFjh)%S3_uJOu5OWDv-*X6Jm)9K!ce= zRmx=v1!X}Mc8LK&q+M;X98Io{1tt9&AOxleOc4f%){v6b`)etgWHky8Et^yjEvgfq zD30Vi{4R8(yTG4S0mKgBMy1xrZCW-emc$(C-3>oSXasFqx)eb0iDU2Fngm zBt_)+@mrkG+gGaC6I!jUknpd8d4HQ`k#^v)>4Y|k0fS%xyD4_O@lW2|lYgR|sQ_6k z=_e{xCFCo=?9E1%Fi5dU3b>|qQci1E({3fF2!D}N_7}mW{}Z)}`*&Gw>6`k-924`q z!@Ir<4^-iS%ISgn>U>iUyA5#?px zb<|ax z=48k?C-I`tZ;DLF17bxXT{$OU3|bzlg0Svc1od7Lfr-*BCyyL2*hAqokfTssi5EGB ztou|!SagCLsLW?+JJ75>+tm{G)?QtBOxJ)VSfj$q>s>Quvqpz zy1J_%)quboBSK$bGGa`U2@z+0v^ew34S4DYtJ!#Z=Y`9Oie%WNrSy)ql-{wHlFH$s zoHj)JDJKL@wg~LVx!5D`RI;@cdoN?7;`iOSXphuNF6z3~w`ktp>$_ilYkG+~#;s-U zJ{W4JCnv44krnlT)ygaGaFzAAa31q2LH*YD$jF;ikk{RyU&U9($)!LtDABa;fi$n{ zbQ-L$=CF#?$a=^}5M7r203D7{hafeTzz6FgKFFZ5j;WU@|T1Uih*KuM*)^a%iuTJ;OP#!F*stxiFA-$}}a4$o9w`_rARJS5f8~ zY0FgQVwSI;r7zHYNQSARa7dqvXUIW3)tChb4VI@ZzXVNoD(u)!dE4wU)xSGZ`nK{H z5$W7SIyaHd?GzG1?(%2XRNGsKb9HpM={pIxz#V}*A}LHa*y%^K$(+%SC*2Gq*oZ9u z0$_w-(=Wnwm1%aN=IWR@Bg%>hz!4J7snuRjGtnqEMLHYq6kP|XIcz%KO2S3Sby0Ht z`ITHJN>4S{MZE2x;Mgv`a1kZ1jtPwr4Nf+-`lSt zU71dZ_0Y_%P{zf?wwmxr<}b4@noFp$q9U=|Z}v(yv=xH1O-Dhh}!&5Im_OSZM1) zTR%~4y_jrZloqR%36Tjlu`SCvfJ{r@Cf`5_Nh0llNIM|X4xC0I9AHo01SKOfaEg*` zngm)1v=C@{y}dJ!yF(Z+d!a5{1Q0%Sp@M*z=2I??OHo${jW zl)orDZ6VZjEzwR-u0%U1SeCfn>a#?9vx=lyLP2U{v|R)X_mG&vDZuQ2%hh3#rEiy0 z0Wkt%#I?`>l|-|`oW2v@x-zI6NYN#$0xErG?T}^Y+f)v%AWzf{J$4C&Z8vuzI3y;0 zLqC9<9KJ};ma81PR!U^eG zUP4lFq9el6#EHItqSL{+l7)K7Yc7c9j^BzwFImIk*tDL?Aq$PDf@cNKo&?Ws$>21L zB%?ARJVH1fPW0X3==6$lM53}Wifl}|ORflpN;_!PsZLxdIj-nDb>jQ{!IE>z}n z!Yts1tPq(ZGtT`LT7tF=JjIuS^=&#bBB&>Qa8gZTcL$j&4mM6LmE%EEqQ00=|KXAq zkMwrwibwjed5TB+!7>%A{bVzL#*%2Gho_K4k9fj`UK9th?~d2M;W+i}_Tvql`WGjh z!+K$YRP%l(PSps9%t}0f-FL6`(G9=*?W-OVS4iLguYbM#bEz(vrCdQ4s@~2GV;m8n>{EtFIwf%CnV14!9n9o@BfNHe z=02kL?>wR(s067xI$W`X=j%YC62+v#Y<~>1eTff=&h0~@5AH*v3m=jLuqv?)L$PXE zb?XVisz>u78R_!U2eFs$G)uuEY_1ge%E zI86dovV%G@g-Jk|1YW7-al@^0_6ZC5K7DN2r&i#Mz!`xvCxSCO!RjN@z8LF>B;{DY zq-~`)3LR5p(4h4)YDHmIQJ7T}W__`RS&tT+ziVLPh|XUdmhj>Etu0Fj3ee}ZD80O! z8-%PQtS-Bv&VXsTRQ|MF>dG#To`;CvmdfHq6)x{V#{l4kkAONCDvMc^K0G`2auObXX^fj zfk;N-q@K*7JP?fM;DFH9kGq{00DU=L0JQl&apw_Xpv#ISw3lHaWZ^7=))4YDG#(N{N{Q?4K2Q@pu0I2BL4hCwL5?Vc zl%$$4u}Z~mivd$j^o158)kL>c6P7Rp94phrG`Mp^cbxuH%$W+ODLYk>A(=_@Lbg{F z>!z)mIX5M4(m5l1P@KyiRbAP&l2^GnwMxkp=BVD`L8Yc) ziD38B33fMIy1)r7_B&&nkCfyK1Bz^B7vHK}3|>QR|Ar z=+1b)irh&A#C$`Vpy7Gxa`SwX0UE|YSfAmk$u1>mPA{&-K$)ywWn%7rcN<_*u&~C- zKh7eSk;a~1lV;MdJIxxlUD~R7bsF3yVh<&0`ngLK zm1smdNRbY*|0s<6z?*Je9g>m45r*-VH7^#kTsI#e6r-9}acGM}`$P}zZ{FM+vmf#| zIzoC4`~;6nKT%`?=Zhd%fX*MIu1**wlO1w8+g0ppzhcgeWv}s`q|ac1XLsp|kQS;c zI~=L#j3EQfe2K)?*3t{-kt=^#YoqBNiJj9Ds?{}CIFs}ewX(*_=~+*=_u`?mNYmq2 zooU7|VXYn7TKniZ!JNe-&I9?Lqd46~AbrJFru!`!qZw&ZS|&TRbdNdIs+}z;C%`Dm zrp0SYM9k&j|K6Rt^J_PhSf+y)da-(JkE&h1Wn~OFd6(5%7?giEG<7dcr`C8(Stc(N ztA=1=!Nez4a?+5gdCe*HQ}cXSX@5;pXBIZ9-ItA1vw{i9v{$YvH?UEqXLg#o?lN-S zZ{z$I`Ch}Hi|?V)B{Y)0;?ip5&o}c_PR^v1uI}nHt>!G-;4R#$y~dMKwX|33P>h(5 zPU*3Sx@aopc8;1l1%Br!tJ9I|`z)@bRcEy-U;>iwiQs#|_r3D{VKQZK{pNTlnrB$@Dz>Fym<@Te9J zGTS}vaM6u3jf3#%WcX-{9>Gb1lTL|~G9f;Wm(dU7akf=Qz)id7-7*OL2&Q!GJGZNa zBhRr3H$ErNOBcwT8(*?aIpw3Ae%h|(bm_%4N=FD&$Ge?ML102k>``cC41 zY=dR7Fo^)xKqGOUO7}Q`)PB z*KZtdfuxQAM0>k^FIF+x;W>3g=rb*xCo+g1KPS!931c_%oG}nABi**%Wo@KSi*FLn zb1QFxkZ-dmCk2AUy^+WZd7^IbGxy$GjH3rvCbA*lI1#cs&*k4g$Yf+lAJ;7!gJEGH zOe8EkHrb~~u#mWVeSRs?tu`sqtwW_mw?s;GkrMqCr$ld=^!3O|(<`w?&i-a0m#2+8 z`;>~~T^#SHd%UwtPFSY#1^j#AJjxd;HvVG!`Np`5gMMBS{y8piOuF(;13iXq# z8(nR4)=6Dzhf%fFnK$J$edZ3@+Y=@1S;x!&Xz~}yGzTw|Y4%wpvnF8`cb>Y+GMUe+ zNM@f%R0+@I_^IA|^Dq&1_4aVHfNAiY@xq-4aTqLQRR+33N7i-aT6T0DnSA!=ti7a4 ze`D=Y@bmN6PwzjyZAPDz#z_;6r&PuB6(>kKz!ABt8Q(b9zANio%J4_lM$!k@g|}$y z>M2s0eoR;TNIyoJ3xEh$Nnw8LwIbm$><2>_?wW22G91ABbsq<(`t$654p=2_3j|!f zjN5{(x{oHu-9TRa(UjhmSm}GN0BQDj70_oHNL!SF6lt!vvRob62>3h8Q0dQQ9h;%- z&z~)~Dd-&NrBxfXIQf;n&Pq)xQA{d~VO-^9j#Y@_#tYqtAp91k zY`o-SW9iMuy#06mvGkgz$reo~#MnnEIPt3Ge1R#)yXcD|cYf_Z!BM|jc=te}-B$P`y?OVl;L4k zb_0L1esy?0@_HItc^sF8FnvVEJUsYcxu8Zk`sz#(pPrfnqqX?h&Ojp zmn`$o*Y)r<`wBO4=;L6y-E-pSurs}d8?n@v{?^OXG3uw{!wnpIm5n4_WJ5Kx>n?E< zuYA0IGh%Bag_mK?m*?9rGDmY>$?ePskK-S~dJ-!m+aA19EYe>os}zfKRBBf#c3|Hm z&eR|3^GQNfoA_Sq$6{Yd@N|L+i1geR11z{YvmIJ_8Z&nroWdh0sl)ZQ; zE5unNlS8Hf*2v6|EhDS@$qT7gvaJ5hF{}q8iDv1hQD4=5>k+}~+cD_RE`+nHFsHw@ zrFklIN}tzF2QJU)mkiYM5?$@BuC}6z3Y`bWq*t6j@nj168m-EJ8wV)nYaFHwICl*( zQOZ{U#VY`c@&XThA*o48hv5TKii(*znjWVK~Mo8raI({_gh!g2y_KcGT!eX(L9)l{w z`LFuky(`1QETU=XUAt%>R_QY|l&*Ff_V3GDN}os+BND~bh}}1iAGycM6jSANB}a3| z_qklJq{~f{!#Q9T2_4S(#~DEb2dME9&(9`5EQ5;6zv}sTUVd0J*Ih=g`)!>6BHwG7 z)T)>&(tY|pw7rD3(pN3J6WFwEEUcV1#ne_#37phX+P!d7O(d7fEnaJ{(XU#yG=UG* z^HGZr$1~e)6=<{Mwc?Jf&t^$`wIBN}&sHGmM0}d4-a1XwGtSrinrlEErg+(Y6ccR` z#YBgQV%nuhL@^!TK}T4#TAroD#W)pASu^M98VLC&U;8VhA?tnCL^@wL>EDD^Lf24N zV5$hnK2AV(b+IZG3i=zQ_RWO#z8@*;5}tI9??8WlLRFS?yDmyFBYi zmpaQEB2y30LY zL7)m7l#xO8m+JQXirKSul=i;Q^U?2?Hbu0Nnm}587bpC47SZlY$`?hng;U0>DWY9q zU@oaXWn&JJo|l<>l;oS@lyniNB-r|8v32Y&rtTsh-#sxSn0xQJ4>-pD%M-KE6&f@} zRipzUVky!GCwmliW^PN>YVxOSZXU6AIrw3Z$Vj3r0mw(z3dCJp3{jUHAUpJRRb?wZ z?sTQ6-~u9wK#N2mA`wW{O_|l|rp!9PO_?Qb%EV2XxG6iFuUL|It)i^qhzeAp)s-3#6izOJa&ZDpVkqCIYF3&VT6K z$l(+z(`;uzlsWWZTj;F2+bwGQIsk@AU3C43boUXzrnPTf)UZf2qimVa$DIcntkJnU9 z_NeN7DAVdQ+2r;2d5K4IlTanQ zyY?}5ZBwEScB8n?9*}t76!Lt-VX1JW6WH|-lGe=!$S+V$tMC>Q-a;qpEi`kdUc_YN zf4KKD84(rBU;p9WOU37#TxNmdOXtSL9JVP%lU<6s+pM%|0r}K;sM&th(^;1$#eWuu z-Eb*tOpY^U{q!IyYWgWsBU03e6g476&1s~l8SXaz(Q|^iH=cX*K)$DZ^*}GYi$MBH zxyph5mWYdUX(JM3-0Tpf_OrkH1=Qh5vkB zKR-nDk6+SzTN7ioaY7g2KSv4w>Cw0G^+o{cA_Ayq*Tu|tqCFl;@3TIlUXgL_LvcWC z?Ae220HCO&up7VHlo<8wD!es%lsuxq=V7$Vz#mTTi1s`0*JXoK?asNJIK)ZJi2 zYTBV9*|$UgL>h_%rJ=~~a`LP!?1LG$8by{a=6#_}e;H%empNsmbq|VtJ~6Rd=~=`j zOaGTFYbJ=@dm?Msvik@x?9JUp>>zcAD`+Wc=hBD0mQ4ZT!L1wLxuL{$zgbP_tBMv^ zscBe(GY-HRGo(-B4Cl`H+56|poAK@W84=JgHh9k)eD1OA*}lVq5R>0zACwVS*05oS zNh9eqYb0s7yq9SkMC!|&wn-^%os(axMs@N_-dwhu@Z=X`=G6&puy0V^5Juj0_Ji0_F zeXmKg5Q)|fraMo4_eKup3lA%2xHr-->&F|m|2sce&2B(3;7KvKLs56hLKhmf*!H_; zKEpUVzvYDX33>sZ zC!&myC?hl!>+*2zfD9{m_4v;1f>%$BS9f$Sbg__B=fV~TX{&S5{wf(5Lgx~)W%ljw zEXKp1%f5Sy_0szD=L_lc=JAI8{yS!`|IQ*iwyEg8&CQxT!Ib;1P8Vr{#Ua(p-TdI^ z*VK2S$h&#M8cp8qJl_;rIdy#Zeekg0vXxg2@d^xw&3qU=OWCJ<5y=2-5cMXCiTb9 zU1tub2+|nAo|XvG7V+zPlxEi6P&C=52+i5W1wQHyKM~ir!*BT7nq>CSZfr>*FXlWz z`5f7j8e`0zjSlb1wo7AonT-gMO^9qYo}HB2tCJqYqmv46PQd{EGl08{T=&~J|3$vn zMkGUBq7W0y4DRw0T1#J{b#^{%S~oC8-WbCcDUp+UYCE$?bybKFr|tbb+T}9@Q|mSoIq{wwGJE&wfok;y?ykepWHppWsC;G?v)$mzPtZAlcFg6@$5{R z`kuRpe?0R8XLcq9%Q*V+ESsnW%wX2l`dTZ7DgATkEVIp=4RG)HtNc;69?-q>;CQh- zz4QE8=q_NSKmK9n-AlW14rZubN1IFIBgcEVWt$0~~1z1p=KKe+Bfx^3Rs*&i?5**n+)5-b>eJ$DvU_YdQ=-~AVk?n2-P zSxMjg0TxEuqLjU>x;;^$>pQV`?@HdnhiQ~qOL*dX701Cs@@7At{SaEeE`Ks`hE_9d(^Vz3t`NF`D zvSI#0emKi1i(F6{sO+f zi{s@Fxd+dO_3_(7O|D(8KU^z}z<%BK#@RkG?^AO9op7VFQm z7xGt3+mzCOgb<{f;f#SYj}fKUng7wnlI`-@~4M zSqqfCR3xvKG91Z^h7~d;qD#qtvR<+drew2k*!LCdFix2hegOt=!R5rRW%K{HFE%H( z;B#I+zh}bJw28|v2$kr4DH~f7u~IIUwbHMR6dO3X{OuIxz?nV?7C#1y7k};w0}n}% z$wmmm3P}3`(c`vURFKN2#M0YOL%hseRsZmQ4}lAV_P;bgfBp3S)7v3l#x&0D{gp|_ z>wRI;+9!iBeDc5D*GR|fe>uJcpbUflFvzh;)DpQXAF=l(xm-F^_qTx<#XhI!?{;4$ zAF=;sxpEiGQaa53qUUel@4iYl-~exuU5rQ<&J=!iqd|6>sq2Ql_fES*_y2akc}_6* zhIplS@6XM*d!kr6Uhk~)`!HNXY*p3wxaRjF!|YoR{<|#yU$VTVP3QL?#CoV4B{r}RUIS!aG=^T57K z{*t}OcNVih{MsgA}R_sdb zDrV~V(lnT{J?ur2`v@{d5W=6Z!v2%QZuNZkaU@+ajR}_Bm11|1^b+3%s|aXG{+M~r z?IMVv2k6Uj62^^WB*gu6#FwC#SI!v*9B5pDTa_`1N~f4iwyYZU`-* zOKIsmxuF~_-6?!Jl|p#Z3tg6jKl($oTK@45j&v8gH$R^J+tg#dfiQ9QM*?%iHUNB| z&>OkgkoU}9p#R1!Y-W;f0cz_WJF+Q^whpeNO%Wlu0BUgeRmoNi<^{mk1to;2BRsv+Mob2c+ zV#ct9oNJeT;>?0&>?S05b2c6DMF6YlWTwxAlpEL{faNd?ak4F7*?lhtHfPNifds9> zi1C61WV+0XoluideHT2iRxlCy;LG&3Pg*6(XhPTV<%b~jXF%j)<|VID{d#KQ_5Ltk zX4@UM|LvU=tU@WjQoM$pcE`LFzLy@{oTg+dGBW4kGs1#>&%REBvSgQ1F*0)pkr1a!(ok>6dc$%ADE@VLu~4;VCfo87d2i#hw3ZH4Qnw7296J6++6 zWCz0X{6Efo`9HEhJ+?2xdvD6-2gd&H`|QPFq~!{hdkV1P+yn5k_e{4Q!}8F*1Neqb z{JpeV%5fmGVYhejnvH$|_Q^M9Hcu(m*X*`rM&s0_X0u16Xf=oZ%%(qcX-QudT7U?# zWNZ$P1KQj&i2@KLc2mq_!vd6MpJaP4SjEXC%~s)hv#`H2N`R50^;S;yaI$oBCP^@0 zyJHX>x~@F)Zehay^V64vomejkGZni@H~`R~`XETi43g?Hg(C&FTjIbF7g+|d3&0X^ zQ;L(#2TSJr!2^5sRm4UKuG!2<-`uCI_6wIC+zbE`cFhm(Y!9)eWiyhjE(mT>GL_k) z0FK-PXpf{1?9C#F9p=&l+bMvxpzg6{WE*7x6TIGk4{-Xh3%GQ7!{EkESU=si$=UdD z7f6idZWqZD7_4G0Ewh|i_8BEPu%7@FVV?u^&fzpk4Kt&PcU8E)Wo3WrTov9 z&kurct7Sq3FsV07=I(n5gut{{c*AtQH~p@ecm!Z`lPwOAn%&SOZ=26everpQutfv^ z-klZE{hpot&@1p3$g&$$$<9j_2qyh=<*>tL&0fipB;zM5lAbKdzOLo3L_C6p&QIyo z4RDW@YStt%$~P%>H7h$I zoLPFZ!&wi)1fvk|Vw_!^GOZxXwB%%*L*aTCfhI|BKL6$_cMf=z6n zL_e%2g9#<8sBAtGIVL6hCQPJK#uWkMv@kn3*+c0(Nq}j~cr7FPFruH}VzLbZVbf@d z{H@jGeKQ*5`$%4oFE4?IWX8+z?q6~uBk6KY;1_s6`jtHZ^Pnd4@U!cLjAJ*;lS?YF zZJFDX!?@l};Hj_?cG(ex&_!V;k`Mof^ve?FM^9dBEo_a`Va9u<69RacRZdtde#wI} z+v`4){m%ZFRdYyt9qCL`OEm~~6fWQ`+{esRo`_38y&J`eMEpHlEWkVWAr+t*5`#LK z$RYr7#RK0-=N&kxYMm2o0sQ|*5HjJ8;e6U>vcLdl?joPdRNaZ)L?(y1cXlQ&6W*6g zSOo^6$*Y8`>(HBK>s{2+q!nBG|NM*lc&(kWRk`sVe>@BAvt*oL0ozPog+9#jBxyFz zHoy+8@y}H-4rmReK%nKT<*yv5G#a*?a|R zHcy&NjP~=iN72UVw(iw7<%R0#Mop!KS{GeacSqc#O4mx}DcM)7NC2BGLS&-$I9X&9 zV_7p>Iwh!gG*pV1)w0fNHfZH+X=>GM$)JD-!*HRcqh>;-7Ghp4&J9+%j3z4kn6<*@voulO!-_qrtTf^)gW1 z{6)cH;l?n7$^7X=8)E)!%=B4GGkdn)o@IMo!|XmnaMPC_yNXTiK2Hu6d2&$H$cqwg5G16QB3xIXYPzYv!Afd zE~mjNmWg=|bN501#4co3)40c3v@s{Ht*BfQM(q?)+C1C3&U;YYc%l0cgx{iZGJ&x1 zoO>74|8KIFO&W_s%&Tf1l3P_1b$pt7v#_G|bx2rPS*6dNreOG0v-{Do4CH3CDfe`R z!M8E{k%%ETP87@Qi0VGiP|S4M8+hxc9Z-s!E){cz+I$jRpJ#%Bw7H=UP#cAMTJzaq zaSI+Ln`{v*`q*ggR3M6CUQ20K0dUqgYM{!!-& zI~4|zKQOm1I&z2TCT#|FqZOAoUo{kXQ!y30S35Z7WHj~Wn{v(UPYG4`&yhwShHZ9K zS<0#$_78tMkMs=DZG;B`?My zTetBngjiws#(1>)6a*HO)A}r=-xJWp0hLNnK7P1!65)VM`RA)bK^)qfpj5=NQ4)TC z@>BCwr=_7L>xf4Dg2-7rYesKA-p!|~X4hc+h#XQwEW>z|3zcS&-Y{Or_^8#+*uF(| z;3$DoTfT;}<(th?m)e^naC~d`%4KjU+w%Dt1Pf$X;fzba=k64O-su|D8# zeM)siy^GOdpIW4m7v~Iui$>8~T=_Q}XYz{2lcc)vz=vMx?E~ez%jhUc_2*4u_A0L% zNSlm)V4Dmgri4@)MvZ}QamrW@Wo9rDc#D%AC^yS($pi-%Gj*(nqg%Zh*A2}5;=k%k z8uonEg^aV%G|ouDYQvA#n@CQ#<)tLSEo`Xlx2#kUsLFkM7-7{t10`RgbUCW@6X>@s z4-{1x_0Z`BOwH~|5TY#w&r&~!Iv0ikOb2M=l%}E%J5+K3wWkXRf)Nd@W6YLo?7WdxFbH)pI9NhO$psXL|1VmlIqg=^QCmUEEo2GfM)+oET>>8GiaQ|13?*AC!+$^`%(LPT++}a}*LWCiz zY=Zib42r6Yegv<)Qfp?3GTbaXX;zOqPL*lJkfXpqcmg@3H;iVQeVOPGn7wUY*_ox- z4MXSP-u#AzwC5T9O_YL(AE=%vACPFD9Xle(B7Of42 z3{i-)Ue|ilW^_!-ls?ZwWZq(2DnOm|74m0v41{MMQGH)qp;FlC8Zx*>v3t8-H#<6^ z_mhYXm07HGYoV^#&E8`D5K@+2>Gf(1Er*r}OU8jy(rbv+J(`Yj*pL;A977}2486aZ z)%wv0b)}96&HmvE7rmz;{m!&QI-Q$!(gHQF`?Q@S1!savv7@5Q2hDeasDK-z+;*-W zAy!mFWyv*c(vNAljm%kbt0$9zW423x0O{+gX}wj=urc1VHSlEh4^JpZR%}W-y(jnJ z7VuOs5*tbJS!du&^%7MMndn8c`;sImZ*sXZ2_cW=J9wm(f_47XddjxMA2{SKvLavO zxf@Pb;gj66iwpMHL*+nIwVr~u72abnPRpZ@@1C>)qCv4%PZaJOy48ga%_S-9!R? zlb!H3SR7jV)HEEf;T))NUeqea%vEvvfmVEk^AB%np$E*ls5gW<`mm#1xz~CdoCbt$ z@+;2Vg!Il;ZA2m0o{@?oFFR4U$2vJ+)cUqYvv%B&9|BejT)tWP{=qL}EM)#&Lp!7l zn#OxwV%pPJ&guT*Z?( z_~tI6V)lMzV}~tCid5`o{q@AXfys=^pSpL$>mc&tr}D2^LHkwg?sc)IH2r0>z`?x- zQ@8uz+kDNoAHQ?Lg&RdXL$^$1x4+)HJyJG~T6+twTBCxdRkq|t9TvPwmMd9kYh-P#6nhJ8+OW`O)k4z|Y36Lz+`#=1t4s|_ zHa7t;e3{X~k0mQ@jFf5!x5-E=WX#=LXsR@WxNZ|=qf-kI?-|4!GESGQxIVJ0F@_lH<;Lao=Vf0=kFS4V2~YTK}!b!&|{vQS+dw0^IJ7*YZ?_Y+F-0*bD3DO+_X;jqACEn zN$Ny$yP3BE9c*!E_s85@`6NhIVvWcpAu^@wZK||RD%T5n`%2im*^;IZO>Ho*6sPi) zariH=cyB*-vafKO$YIs&8a)hDaZ=Q?R6N+F}!>MYt#X_ ztCFymEG@gEsODjFq}A$M7&9rz_?xJNSHH3?P#WjzvA?#OJ+f;AFk*p#RamTfVw{ z!L@4KCb|LZRd?lKAk3n~vUK{4l&i;6*RF~yp4E1@WW~g-cOjv{zGh{?idr?A)kZ6; zF_r0sOEjTNch{wBBN$89Sg$dMQlgHCS|^N|2-hP;FGRy|fw(f-$oMrYT)KkKfuApz z&eZ*lbB$~Bh?Xv~TH`dT8Y8>z-jXT^)V5ViA9RI#^)5T8NEQT(=q`vQ&PB%L$SYs# zjeT=(tqowtI_cX&TyChgvT->*bz9hBxw$9tTgwh=LKIS6CcJdv1FC4B1rGAGtuFIr`P+)3?+0gM#A&X}U(f&jz;J(P=?g?2H6l`hy! z_E#D>+pdFfgxRccPcpK+SGIG?^Hb)hKSL1*1vHIx7f2LPAN)=pN+t*Yolt zOjoDpn-I)O>my>;I4q2mx@0L9!bA!>wnfGLM}NSLO4eDj(ja3Sb&{D;AG4>}h?`Eu z)~PilSD%jW8HTNYOO~p&H;REZ{kGuix`292xUFXB&&nzM@i}>g>$pXxcD1*W z&H@8!{76+ygq~(;1`%R-Ut&0>!c+M|Yc;;%wq37q6$H5Il{ToNw}WkhxvSQmkpy;? zu2TB|o;1>yD~d;DQ8v{P9##>PODrWC#`8qi4RMFn8(h~FDJ@xG<|$TlA{+KdsZDW% zki0BnE6>Jea!5J*KkvNdiZ`}{uMTwSir0WPv0%UrwJxef-Q9D0gxMYUqwcSQey!#N4i(n-)m@0Ci3n@*??ltOCk|CJbHW~sqUa2fPIDKl&p&h%2chSzdy>J%6`tT+u za+z!V`z_#GwWG6<0ur7kB=Z3jY*acFWn`Mw_7hug5Y8~kVl`b6DMUNz*Z6|9r7K+L zNoH8^HSLzZO3{d|*n=iF(v4@z!58U*@vgYzd)LmjGydm;yI_&EYpY)4S{v1wzEu+d zo7OCL0qbjoZ^aS{^+usi7piwZLELKVbuQkNIpf#42O{id8r2k4EOw7^@^tMsmN_BF zik+-Fa>Z4vZpBT7)~q;6>Q;?mYjwQzywFv5m$>!S^qOEw3sf|t&Js=$3v4KZ*XDS( zPT#GR=qmo^@$^<|Gf+}GveddN);f-{4Izj3mItE}$9dPiw$v7>J;5iWkX%!NMOE~XMoFD56s2-eyR|X&MaI;&Oe4Fsyj;wy+9Q=zCnz&vnRVJR zKAxD=6wBPhGH)x?CQ_kRt&ypdI@eIME^AT2v7}DLQVk(+chRR8(39tk7w$ZW!(bt+ z^2(1xN7i-aT6T2(8n@C0C3R67hFKRbtu$fw7N5R$FQcE673)L#!Y!>yVGv2p9FnAO%0%Vt-Ib!q!Ch3+)c(*E@l*|t^OO)<1eMJ{WN z5u3%$QmrmkHzubTEfj4K%i1cOs=`A^j^+-k4w`C?EW1YewF$^Zc1QjthM^n6Q>a%@ zp(0e@I8iK@ym{TKt96Mj%#lv&4r2|K2b5?6v^#P4wz_18!O+gwb+;**-VN}`Ok z8tyh#jN>OEmauzvDL}o?*95n;idCm~jdabblfYGB={#_%*S(V(v&?F7srGNDM%1ob zWt4HuekGOe29K-@v|`>8WbS=fVY3FKLH1URT4zs!xqmbiU3lmBn4f#slY8#Yyw#j# zlS3}KUgKJzjV!a~T*kywD!QtgBJ0#puG}2gRut4@vo0&UV%Ee)o0w9y0O6{rtkA`l zl84*iiiyLTnsxE+R0W-@)?R`UtPPW#xXJsJgKhStuRF*4w%TOcSNLiU!;%{{x0C?c zg+zkwlK3iAQC>+cq~Rp7olsaE>9P&Wt64Q>5(^l*R?aqY3~dXIs(l~Kc0dGG(a%5;`-RfcrW)MRYrBMpZ+uFN;Dzc&~Hs5_Dtt|;pjE1Vwm<&^@ zK`g<)o$e!d(HXO{WMtX3H_fI}=SMExM`UT>pfq-OcrHUs-l;&OR6z8jL3t#QN0&Mby=3Hc@Cfvpt z)w$!_s638M$$7;RcX0E=EU`HvHQCj5khYCF6AA8A)`U0T0NF*#JUMYou9$Vj2B}(i zUSmq$z;P{n2Rf{H9NfC`og0qlZtTcOD8~3R`w2LHIfY4)iFqy)bRNIk&`XM6*$VE(~RxSh{Q(dHoEQLEuNw;n;U>R|`j;qr7)veG55(_Dy9)?v7#H zxwDwbPI_ylBo0Ct&s}1VOj47_QBjL&XG$b86>({$>xMn3I{mEbO1?{$47bYv+E9JV zd_JGPmevY=h$eC9ETU=XUAtp=&UNNxDwpqEyZ)X=KX-l)7Sh$a1L<U%o65C(cf&b| zsniTo`7$Q&yO?#avUcFk+<|3~(e9O!v%7SR+=m!F#$z&u^~gEOV>$|Jw>QqVra?wZ zT3Jz{$W4{Blc1gSB@Co>yOI~#jFc<{HLqnaMvsy%2CbW^odN6?{oDK97Z^_CWj9T_ z+BCqhf;s|rBj@s0FYvLrY^7%s>^ffk>w~*UelR`P zr01>1WbGyBK*}4ZQxCjF(L?t=^8Ua(vOxG;x3jZ4Vw4F-$wLNKeF~@*&FbiATcNMX7ROV=sb5 zmGvx&g-tL~DDs`&CC!-%A^n9lhjBM*X2W<{%yR1iFO&8aBUTl}fw4FR@nj~Mz*N5X zO1Ph}3a2f&KWVqm{l9}sK7mDbr+4Gk!UGjJN?vDiyK>--=E2PMkt!wYO1ezClKz%= z1#g&kC0%T~!byc{XQWqANWXjak-MF^k4rAzgO(YCf5=;w-I9X5O$nB(p@Br}`nper zX5KC{Ik{rL1ti;Rw5(2BX}1B{?Q4_fX;X;;DGlXy5TvgvqSugc_WCQqZh<_|j_rlS zTbQ)m5qZYDvaOL%x>^jvC$fqhD^N?%$EsK2m3cSsP%iWC4a1JDWV&csh2Ey2*x^+& zM{KcgLuWo-1owaqGN-xgeb!w1IcYAvSzEilL!2gN;WW{i#|Y(#mjJ9ncvSd14QZ1ps+xW7IQ*BQp66_emJ`&zyS&5XL1rW=I!X#6?rO}b>g z)$h$-2;|(n)s$B5;Sw!*VEMY1VME!oH^( z_wkzUNv(Wa?)!wSu^o7f7u)qJY%d>X?Q_z4s4r;`)w9Q$ezInlYSKm430}ZOds)hYC z&Z@-Mb?Irh(|;lKg}?uwp>ee=T9XT^LJY0yt2wz;d50QGuH7qxrJ;3}0$!vNZiX0= zHYR5Mfrz4l^T2B$r}EU>09qQ}NLBq>b|}5(xJ`hA0GjRV#+3=9SRyz+Ip`v3m?dEb>I121lLUgg5% zQvl3z<)X26NA)40RUg+`p(oh+Ke6guKT*yDnzfRCqSAIN{(>?%sCZ)nMts)}vpS+w zKJ*oS+_Ky@2#F5+LZxo$xR?11P3vcIlW_|xm43dEw0$p_D&$E;5yis7>lWlNY;6N~ zn?bt(9vdBTN-^Ojh$5L#2)i>xF)u^vR}Qn$AUn_*=>_l z*c=XB(|4IfSgLF;3Ad)XFDq|wdm6x2Or(9n0p9PRmGsexDxh^T^@G)HJiYV6@K~f)Wesez6DWNt#!vv6O${AUFN3eY1t*Ig<6yz!ir7p zR;me*(o!|r4BSmQVia68%ajI>Olef!lHK^LNBP!WxD3eM1?MH%M`$hmIc+U{Sls$v zu9p;Z?SN3=&&JumaulNv?@wtH>GJAIB8@AtpZ=6<{!_;h{tdvBbrm^&Qg5fjZU*h7!|EsMLK5Me?Wwq&+!Kb|LJJ*Z2M@ZEoT01*5gG?5b!81Kgy>de) z#X;tlyvPZfnfZ{=GT7YCxXNQ2z3?sq=_^gUua>k-9!bmG-yDeIM-}AlwrH8AvdOwo zB@z^+jgj6?-Y79=hQKaC40~h!8`5ob`|&;g8{Y?M5L&9E7+8nsuW7SAw+)IKr{Fe{ zOG`bBol8y9rmN<2&1iFq8STzcqtnr8AXFS~WfJ&LM{u9c&3$?;_pw_FW*Fh4Hf29{ zTTx;asa7(Lb}FU8F@l_Z!VZF-M%y_H;AC$tK*RA~@YqRXF-k&JP^E0@7pC-7*g7m# z{=-s59hNXb`=%%Ec`8_%D(uHwZ=>eJ#UZ>@{FLcgd`HYx+BVa*s)W3)$aGCs z32$h+CaL6H`gB!~!3IbCZwA3e>DkDvgSp>bKSf`4KVnmf1_zvw7eB% zF8eRt~aSv5-slLlHreMABt6I4~ck;Y+v43iya(UJD`P=1)qWwD6QCqi8 zy-xU2xrD7f3!)#~dQ;%6MeO>%ck3>u?%46&@@g2z4H8-j7}|-Y>;!- zMp7E=XCsr>U1DUGnU|+d7`u_@j6u?pv4P}S8|l-cBz|`W$J=5YJ;1RJiusKbAtkfR zzkgs+bEQ5{Rg#I-Ml!L`y|*?iRE@N!D5#ujg4(nxxFuGzRASA7-_{{-s=V?65M^Cg zu4PBpk*EGTwoR4(#@aW|{Ec(%yE311=ttH@(y@KxL8QL_EfLe?*dw`|X(o1+YIThA zsAIIobCmVAWkQcYf_Z0im0m3M?xHT4Z~h_3-x7CY6G$R!E!`RzJOs zoli&iE`xe;+sc-xOK;viUBjXIE9;@&BxJyNK3XJW4PX?%q086}Lx-=^t4)*2Zherp z_IgSrGgf53uT~#s+e1@_OLt~YM!>D zhP`|M)>hHkpP^xmy|9WWQll^zXQcLEyw)x&rb=3>?$V}hb-f;5k=iQt*x17&+5(H} zd>&@b({<=NpkMeasCCS|h1THpgc^e$c`n{=tmDwFzDZlRAQPRskLgD!&$ zwhXNONLDPO@^6m&S`JuFU-<% zy8wiDPa2;w^)F9aWo*fs^AZ#DgaDF2ZNC@&5(eBgFQfGe(nhNok)~~Cwc2J@`K<3_ zliC8C>U`h~*@ZZ|b_|E6ZYEm-8_|U;g@~#!J|pgEoyt*AK;qD*%d5lP%X{ z*W0#zm(u&JkEoVfE(<}aa$IfXXlF#(-xg8qE^k(gG5a?$1aX|_m(WJi5$-+)03F|C z0Kk<@$MylM$xe71EVzzWO?ta&Q?e&=#AhF1t%5n|zb~{Y={j~<&arEjr^*ezIY>|r zaz?J8rKFv+9926lONN;t*=#XcW>lij!hQ*4jz|&neTfxZRic%M@B4i9^S*_(&q89J?u@xAPR0` zIsZsT`+(;y-|w_65x(SyV8Q13W`(sK{5@$PeO&o|Q?EQ7Hq0zq@JVO3M9|5VDBr!s zvMf`eK5rf`-A7Ioc{fiS%E#98)pD4|C536#OOZ&|GQ)9h(fXHlqoVWEjqgqq!TQ{F z=6K&B_H>!6&>qXD%EmnzQ9h@h~6~C6f5GPz zhKQJ{?lvDKA0Q!^Nbs(UDa?j+14PLY$PG7A>^{P;^RbqGn0nU0@1dK$ST_2Zlq}|t zXAR4D_Q!??{xD8y+?tNi^_`d*s=b8|lO;2P;cc#0aU3ipZ}#Ka525|5{R2?(v;-p3 zFJRe1k7ekB?fK(blCwSg=+23jpvx>_*MJ0a9Xw{AgPi5S@c{kZIDX`2pRzRy13$_} z`3w2sEGH~-89YhRT)MprqBxx{`WUgPhOV|Blar0L^Ho0S@bQ>^zW%4A{q)Yghq+-r z+}}!%bAJKf-No_phw=E~;bHWkj)L%ZOcg~L!xLxHgEx!semo=EnRMqe+idzf6TQ!a z#~;rWNs*`q|38y%Jl|(Kc>!fzmhvok$XD&oo6Q)fF$RPeUOMre1nV+4C8A5of3nWz zjD4~7wava^-*3KUld+lJbaTKju)*@5yR4UR#6##T_nGez7>xZ3r)n)5(w&6{Nx%hu zVf{(-S^1YY-t8)M%fE{roOpUCvDtlMj8*`bDO;c){!OziTeo*N(z^f_aT=FxtKms9 zjdRm7cS!=Vf6iJSho*D--aLO-LQ~`|Zr2~~Zi`(k&zITQ2JSogf2xMEeEcuzPxILi z@Rvr}U+~b>S=>931V92*dGp%>OX|v+RaHe{l%)@E@6q(V7=~Z z`XoiG+gmq^Bk4NyVmK=k#7v|>BvKYz00 zzkWWGfQ}mv&W4b{u1vg07iES@U=2dp>NEGoS^4pq1m_AYDG0|~{id?q=-pp0+|}hfHR%KdtK~+do5H>b<`-@p zdeeQ^H?G8VY!6Rz7MUa)cd`T~p1@ArH{*OTtrB&cJ?|C^_R>xE%`IDvW)*7CPPU5B zDkH_P6pg4BF*VgTO=50DWY~Hg232bZHFVXqRH~|$W~#bg7?i@cY94n}cXwx8YSfKu zD>~7MYExU&H3KGQJ?;m?7}?FksujZ$yJlDd?+arpI<;Z0;n&TuKT(1iwrZA1-O`es z|IxzR64bC-)v$W);%b^=5Mo&-v2|4`EN+4djl(J}M%|2&0Ahz3)oS;Ts#AC=YU>uY zsAd#iO~cHn-DXrmEyYl%ViAqnredmvbyQT$%T~@Pt=+XIdrKIQnzp4{iT2Ew;d~I7 zQ`^CuVJnovb_A@m6wBg-IlUFk>9z`}s;V|Dnq?UVH_TDm3g%3@TEsZD|AdYTYmdXusdMCKSL7LhINQnYdY3n*HC2Sr| z)44%TX$Lp3TMd;`m{hiJp9#j1HW;UvwrS})@CZ>AY7x5>V{*r0InihI?A)`Fl(4T}KwnhK$&Zt#O3X$OL!w;QHTHCA~*HC4)sfp$9x z1Qpt{ESP6QC8nYALSQRtk0`os63|Bs*oUCa8@vE$wE`jv?@6g@G9wA-w(2RWv}P;N zQ2}xkm`oc^aCk*tsOx!99MajX)LE);LtCd@#e+Z{X$f^8IGdVIVOEO*0ZQ5U;9~MG`7wW8br~^VXFttXR z_)Lm2`w3&+5Vkw+3RZ(?w^}gtnyqLcUjtq!)X?;kguz4D@s6Y1 z8hwm<8lcyuI`1G*XS9Plsy`Da71GX0#RO=By<-@__tfO}Qn8K#Zdxl5&o*tCUIylt zVOhHNOg!;~)b}i|LuJMhJZdGL!3+Ro4iqcJqH2vv0rnD)8c;F^6nv2fe@ ztmE8k9L3Cc2+6N+&p90KX~%F@8R!ZZQ|S2R*z{hPyk{II^x{Z&8OQO{d$}n0Y_Bfz zKntxHRCcgv3rbsxZZ-jthA{s-6DE#Rcx!2Csl!E zHFeXb!u{^`x!;+sWH$|zRV!gp3l1sM+(x-LxOG_!_L!yfk$vBpJLAvnC&+S_(_j_L z#5@Pk`yhW>un6aAd=U5}EpI6_cwJ(%Knzne$usfE)7zn+VF6EqBukk8bMa+QS*$w_ z`n_D$wHK3}#tVOak9tVmb(3~9K|2udoO?I)rr%^QLpYY~!<&lS0lnuAcuhceEa&6n zTv!~(8Sgk6?F+iGt#ea(x%hXqD6`|>-;tb=k0j}}x;llIWky3L>8XaToM!5jRxz~% zwvVADhY6vYrRvWkt3W@ZUZG5YOp-#Ca_BlyusGBOENOQE3tLcC*cE|l6A-jGt$39p zKm)u1!`tkru&Anm6u=E{=23j@^mfu6QQ$lVumOP64E{6%$BCisTgo~eY}1YrRna$~ z9?y!`?0NTs6Q|g%;7)^+N!Jy{Gz=58T4Fqx*s|leoE+;q_N6!%_pLO(x|C2QvfLtA z99o{!j}SaTTB@r$RbkJVs!phCJeyPjS{ryvOw0d2dvDt0Hj-qEet*A$uilqxyBB2a z%UJi=)Y@IurS6(iP0if8c1?!Ngp?d{$tIav+U)-K=MLrqW)e$yk_eE1tXoQ{2n2zM za6j&G{J66anOpG5PS{ZMW}@hP8rq?(0XGTuT(Nv|?P-vUqaeIEsX9YLcuGpEjRl(v zZ(7MvYVjGlaBupjXvp{1E9jb0YlXqice;-Q zj2rHE2u_{uauKPvwiW#pMvJjwu^RB3faDoKOnxE)$Y+%AD?8G-6A1;6-;Uk$v?C28 zr2*q(0D+;~3Nw3(#zdBaBU2UPY^mTS*;6rlaZWVir2HZUyuA_>h$I8wnp$4VQE#u1 zUo6F_NZl~b8Ts`o%GGgz-|h`d)-Gl8KZ(Z6(BgfC*sPa7Q@dpaDMb zn1-ceHDD14JjH2LWJFR;Io}fw-J=tZW3cv$l$J<@mN@6Rv8i(y_Z&Dxq;iqr@RZ z4ft@5d)x@O{ve|d0`QUAV8lAe?NuZ1DB>!S(tAT=+XL2dFw5GJ2^A(rn9GT3dO~37 zujF6}y(m#`oE+cr!(5r%td{r38fRRROH1&+!?ELRURN(!AE9E*J>Ygdui=e+E+#wEg17lr5i*>EU zlSZr^<5)XJX!%GJVDVBi*q)3rE5MJr6KQrP;?Va7dH8cU&28sE1F+s1*QLcO&4`2X z;uyI|5HF6AJ9Jdt5aUr!RC~inPG{4L^dg=7<72jv$%k2TGrL{i)t>o8$TJoR^>I}6 zBB8iAB$Zew?ri1Ydz7s^dWr2=Yz-Y332&*`Oh@Q?5sQ}6uR@DRy}9EE4D>2ncRuZF z-%sb$3I3M6$MCgXRtk@xlpIY6K2oMGju3pmY+oE+N{>3&9>w%w&TgK5n!JKPU!|Xt zS7OYE>e|Cs*Qy?Aibmoh_lb=c2l1V7daNm=TQI4S?c1mA3)sBt^ed4eV(Wtnz>dGdoT zrGga&VreGF>2VeZ;l+wh{YX$8X!@^`40OwwC zWZkI~Ij`kIQ^%vL#1E{yK_$MSO8i}Qt!XEirHMsp2_dDD8a$Nup~V&DV^)iUItPGG zC~k}eaA)=D#Ihu8r@0W;L9hjH7!2Of5vY4Y0=}FIp7^6_%(K zV3@E<3km2v)XLEmqKs3Fdwh+;E6ez)(C{L)H;Z&;5eMV7ab|I7Db7x}xHx*1nYufk zl?l`5TWTR`rxq3iNU*H1bmWs7-cG9@$LVkpM-49y`_9CvF^hxnPMLB1r~ow<49pz! zZHqwh)`1--7M8(a;2Lpd3TX(qjvb=GMKG@OQK%?!4rqr47e^Kohy(MEAqe#ZaSyMe5*q zzL~s81+?-Iqv}r3k9UqMOfHXi#zkeZj9i}@HAHoZWqetZqvLB?@Bem%wKvN^!3mGRi}Yst5CSWb-!9HtA%3yUyOMZ_?HKbJlsH zsQ|#_c&EMd65@MFX~YwWMi+@Q+$W`{NOco&&RzZaq~pq}$L!e_%1T$){cwjh&I?(~h#SIo||bG?pC=?|BDL%Z$|!$$P<0gm@B(PaMVI^{D6U zp;^uG4kdPoC;EKxUB7C?B1zmuI*o{vz>8HO8wcP;vUtY;cvo3=v(B>jllvc%530TX zi!u6>#ZokmL^$bJm)SN-WaAuG^vcv7=(u9K-mKCYAiBGsl6iJDy`3krC2$voMI0@Z z?N$Fp7Kc$6NAjr^MqL!z@ChHr7VE=Wk|$PFd1B+NlK@^UctH`!Hqmh^!XkAVi-Oi7 zCFahTI5!xxm$lbvRlh{i4jicpp5c&c@(|}%l@r1l-4vwa} zZmsZwAyi@t0vUNC3MR46Slb9g)gi3s$sDOhQffM$_^>-;+ZL%ab}ZZ0Ck?tt0-T42 ziUWD>7`E`06t^R46F6Vg6 zFkdDg1Vpc`ouEpZ=>vBNHb62ionyf|C9w*?Jk9bMkA^^CITrjyoW==59F?ZwfZ;rJ zNsd5{durp(xLTi(lBSqIQ6F8i^p24|m zD?MS~rWdm;X{i!|Mg|6uCpAYYQKaQcX-=dTB2ss-IF(S^DmFktUzkV=>4LA6#yjWciSktiTFL@La+gZ-Ngn zSQOJ2skKC$^NThx-l2~T8d|vn&|Xv*Fy2!eH*ZO*E{=(dl*!X^h$wL^96toMv!_z% zIy96xy+q!lEN44#E{-f`BwH7Wj}oU*Sgg9%D5n;_Po2WfCW%+93u~z_A|skRSf3m= zzc}N}Md7>ysAy3CjXJF?6%r zMLN)q4<6C>sjrUfQ$X9RmJ?CX-C@jJL1q1{D`xwpyxFhDVLeh=j}*SfIiWZP&u()) zUA&(z-p$iX_$gaY;IA|I`+)7v2t5U;e43iVE5hg;LCOtEJ<{ zn4Q3`iKSCC`C;iCU_5}8Fr<*$5_D1YA$Y~^RCG~dRa8V0fI4)Zu+yRJ!G~>40cvkM ziCoGv0lVK)Q*4?{vC)QU@J>uI?UnSWp}HPR4Js1VI8sxcI7Y8mO%u;Qx=q&ebk+_z zMOb371;BKxw1C%Iga(~%I}t}QAQJsJ5;x_1tI6Uh2E<|C9W0`q^NE|u}F7ua(Avr{lem;?%J?xakQUK*{;pAO*)z1uCq7mn{@U54S&;8u^7#!84(K5 zXyyI@uCe@9DQAp!+?@)IuH2%89KY&QiBNkaYuBp+v10_N>1=wDUZj(Me9RVd?2>u< z>5NMzVky~0GJC97gFjmATr9FpOQ)}Kjv-=!yh!a5Wv^5K;-ukv zrDhkW-}O;+0G_l|G%dDNR5Sq;&AbxeIr4lR3+32FYpEznw$BJK^?)*A-N3lTZP{(4 z7pbLUoFiBh$M*HB$>K3$+>(Z}$!xmEsGW3LFIG`SztVcW>b4jstrrLI`Lj{ED+)iE z^G)cLi|bVzJf|4<=re>@mhn?Ljx|Gsd!}s{;~deR&oRbcuJO%knSfbtsf@NyI-jus zU!3mRkN zC&FZjBcal>p0tC4%K0j&v_WNYE#~T?n2i1+2*?@WEt}o zVP3B^kz+~bMS*!kzl#Cob)CpJU0h~2o9S{fnP;oZ+tuepwA8saj@^r;C)J~aTzd$e zJaZyu?|k*7bd=(Z!>A8zGB9h4&E)zsvXZk8)9mBWqw&x^8e?I(NL6T&sxqReti(C6 z?NvAKqnuV0s?h8i=kGM_h}KK6uMz?c7}*(fDq;h1*m#jzndukdscw^w2W5d5laBXy z^l3{~s#gcHarh{4j9n}`N}QNA)?vj^ba4Rt_f-EqeVeQ<+hHTeQGlnZ>RD~2U>bG^ z%pZ4zj2-Q;7UFkf!3t^Su(^gfjkDTbv5_N5tVF4}J>?|c4kcxFyGYNtt#DV%F3(xs{stF*c|h)sEHEG-#*5LvW9_&#Gd5=^zo> z_s&1PDyWEK_9BVGA0vDB%<#2XOBREe(l^@|Niix?FzP zWBww^ro=LTt&oxr98Y(flYB-JQN;t?F>mh(w^wB3NXjX_D!quLoYF6CyaTfUBigTDIRF#KP5AOd0C>)n5Gx0(l(NMz(8R+`gukh`Mfw~wdW&|@+_fuolf81o_W<- ztPHDJqpmvHtA%3yUvo2%f(f;O5WbiT8eQk4dmgrGXl;yw(1ldaw^SMzjn`P_z;h!H&h+~wu%D!u$% zIChp}{b1T=DXLX{E@Gf@aartqWtadjA4xS}a^V}k*<|a@&dbyWJm#Yq)*L>RZf;lE zUWUlco14dfUx00YF#PGKH_O}2&F$vRX8Ar_6m2AfU-zF|W$Ajkm@eM!zs-SNC(a`|;~^waC`% zCqtKZ0pNM7<1%5f=ku*Sma&vhxCCD)WYv_L+9_v3*+Sy5vr~TezNhc5cV~b&{l(9e z&H116*?Yl+mz`hknP@k5jF(Qd^)S)4Y@)@O*Y9j;3qivXW?4Yo{m39W^Jy+?aMWa` z4LFa|iS~S~q%WUn!Rn2;_MBNh81W++@jLm903)u)zTAc@HRh^f#`z--DYxOwaAyY_ zeheE9h^wUBV((15M`Fcue5S{ua;EF|c3!z}hM7K+8P9>ZN^Fz4R%54K>0HBq7MfQX za$P&wLO_VD219-*LmtqUaL;^$Az;3IZ*x6XURCUFQ#aupEP!#3aQ_$+j)}q^$6vO& zDVJkeKHDCqvudszxNPa9s7RY?srr#9Z?czP!5qE7aEr0tE6BBHYVnMs zd=J~Ps*Jaqq)7qeJ3q`U-AXs(Zatq{<;T7PfN=)^tzvF%eRzfE)%K`!%Oz)49LOC% zGpt?O!Oh!Z?44=rtH>y3>}&<{5)F`-(N0J^#8$_Nt;AhyB_O8un_^i(8kJ`3tIjRKWYlkCRN*=53?GzT)+!`hE*Ih21a}BsiN^xO_6YQe|VhzLXigc*qpK+&#FE6U474Ny2%1)D9VlvD?j&gV_gNn8)VTK z!(KB33FXMF!e&#g%MCe~YV2iILe17Q6NGiv`@xtWINDmuyAu$VaMNt2tKw?d@>K_% z+%(wL!aF`3fE*v9ZF!{3F&&gyfdx?E%B36&BP;B>X+UQ%UT8H?RSuO+F-;?FJ1&qs zo@YzN*mCtUW;k=&2|L7J2ix2T(cmjiUcsNQ(oe~&M(1BD)UgA^&m*iH=RABQvCjas_ynQ6e-rV5#v z>Y6>~8=Q19uZ272uI>)^tjV_iuY9V=Y!uUMS}m_2j|zZet}3+B4cPsfNqGXsIs)7# zu~BxQ*X(wYHk;={mF1q~w;Nkq;gn?q6&%K_tRX54;Oh`6lY1x4yzoJ>l^XL3dZAV0 zLV^4~ZL?QRP|TaLc2zQ0x_)wD>uGLJD0R~D{aqq^lqbTQ z&A6%xvmi?H6}!1f;VpB_%OL}Pm<;%nZM}gBNgGZxugHEo_|H`Z^-aocv5}e^Y5dDh#M!@JPj7BRYdfy8>}H*1?Px-h=X^QRZNp6 zLJ59|p^sp>8=Y72v0)V%Fgw<^Y99;LMBSfzGaOAG++1&SBh1Q%6J3xr*>|o{B}vv` z6MIurxY1((MzYuSG@@qHE-NS|Tw*OoRZgry5)`5KSF@r zNDhUKtu4==cwGa|nux_J%?1h47sc1}Db`HSQnb|hPsuD@y~`5SY=*0tWX8dN7B7>p zVv2j_k1B4Y&57>dSKA2-o$+)wy+|+8$v-}33#_bt$m-S3wJBHnmKF@QsF>`6t*-#z z%nbqcV?e!~sDtwao6WYaqIe4=<5$rG8|2NDP)rV!eZV?-@jx59BOxTr=d<2$#-*#8 zamz%-j5k8v(j2!}$GPi5mz`ZUn_s0XiYXn0Xr?L?E*ea*VMp)Hak8XuQV?CXzeAYb ztWtqeh_RDZU95Gpjco=TelQ)3Qac_x@0G2#c%=t#a`FY!t_KhPf@v>^$%LD+nzeXgeLT0q4%2*qY2FQ} zZNq?Cy0O)~iW+LGRErA4lBy6(ngMXXB%WTtou=tzkc%Gdd?%qKv$qFA)7_bZrH>x0N0NN zI0>`NH=9>x%d;WQ;}K$&Y=4tRcZw79U+1`ONp&~H(kj_-S}T&+ST`$Fl~i{VO)*hu z+AvHjshp~WA(X7=X`?=9(vF?42!1^WY~)|Kf4MQ*3{0VeGKGq!`fa-2BzvB`YSv|y zu!S9KUgeBcBOOq#2BZC=-Mt$wndD}6yKXYO2KID?zLXh@%JWCws2;=EA))n>c^U-mfRKSvpS@1jFzRr0-TE-Ck~zi*$W|%2hYvx}w-!@Uc*p5+$k;56lhN z`hLQ{zn{F(!PIH;GiR8_pyZbVQ|T+}qQ?b!-hD@r4U%6FQu))UvFr5t^D|9I(Wvnq z%y5Mozhq;ftInETld}n{gc=<63sfP?&ZLUk?39Bkn0#JQSIOB}RqSpnn-D)acSGLu zB@L*b?`z&{YMHBwyT)3%Dg-WvOlQkdug^SnlWCU4sd|1oH5R^VmK!CT^G3a?i7?>FpfVFSlvdOdRIaR@J); z2{Bh?zN87THkNlh<=y3Cn`h|jI-S11eN@_KVLgXo%1y;7C7A4jkRVOOmkGvt(hKQ$ zNbE@|tZqozhUrzVB2D68Ls#N#+iXMoA)=z!!NQiFZ6iIxa5Vtx91ixKUJ0ywn=%EaN|OO%f4C*S#|K_o2UqB#?{G4=(nrjNdaR; zd(CMU8)jXqa+04KD_>QfE}C(*{lL{eRDsR?LzPPq9l`QffCqshF>vE-G+-2ypx4{JpEMh1OI zls(qc-A!XC&){acoUP%8lUe%ic9ACYM(kaf--0I4^Hdot>iRvOU*ntDa^Z&xtB3jh z`EH(GG`XMFjpTT-UaI0OLqbSimFBjMx_*P%9y&h1Xm&4`u-~%@j5|*kS4p}(wPHCz zs0nkI`?2K}I*v>hWoPyvGTGqJN^9ef8cv^7U&(LsDQh;%|M&4*zeCRF=im1B3jMf# zvstB!_2p`Mkxk%pSvubra?R3r@&D@ZAZcDo{h5`Y49=Ic2;Df z;iAf~8^Lv>pI6J#!*zQebs9K&|wUnjp!*VEP9gd*7OUUdom$KP5snMH2U5=rvvkJ6cTeH3b+CK{p~i8(I3NM<4gJlkC$?(B6Y9?>6{% z_gC&aB?Y`Kc+*`C7mP&Jy*_L~bLcJaaB|6dTk^ZLCieAd+k4w3e|oP+nWx826b;eP zeIJC;Z?eH%S3K~pzwo9@;J(lf*Fw$~=EN0ouXwj_Y23ebSuDcmiEKF5QttJSWsek! zc^{u)KjwWl47#oSb%}i)dh3Ql=e6e@;bgyCrSr*R`2lQ05}5A#_U`i0-}1{xUp{2J z{?Ndi*n&6F$QTLqCidQ&eEGNK;(key1}?AsZ2Ap;EggOrs%;$_ADb|&W*XtZJBSl} z?|tk1Su&i-{fE?*LFreG;iYDh0xuEnf5{}ed0ifBpx<3@mW#<^3QK*fD?WR0hYt$l z>sv%{%3mZOATkq~Q`+!Q*Z8-)yNzDs-Ky8-LB3$%;w}HCrE7Rkl>C??f3z9NDqG(y z7warIa4sgyD{RQ__jvpYLF4`Uv&XwbIh zjY8^+Bo*Q!;4}do0bKSxHZB#6LLs;?9P}!`@D2#+{T}BT243FWuB(LSs$lQm;aaee zVW9iTfHc5;Kk~1Qhpm^`s~YbP+=o7EfC6u`(L;j^8x{cA{lP_T?~ZaOp~~I$U85&h z`G3XkTz?gT4jQi-{Z)*1uL^G{)67(SX~9PPfo8K_Xr=Cl{|uU_ORL}FqvBRmRQl7ak$TR8DYHCVD8gJ*^SWbl(6&;{-Q%DyF?cw7122W(Ww~A_ zT(FDgh~@9EUytC5uj}s{eA_*D-6=4xs4XtNudcjf0T2b=whk56mqS`&E-|fdE@#WztI6f{banHlX&jVaIr;{#9DRdV zK97o?Yf0P`1I>nT>s9)xEI=(_uJqkA)YZ_vy}G~j9(VDv0NmyGQKepzsY-{9AJOld+er71lndpo=Rl)TFp85;6z5qL=UJzPEd^Y-e|cZ+wk>3v*M z)Y<`1FuR)E{h6#cjh*)o+e=2TfBkjDYiHuQ{Atkp-x5c{7eI*WYQynMr@z<(I&hTx zDD&5ys~&Hi&9V>KA`iW{X9;gUh&SN;W}E4}VT|-$ev6;r7NghOlaCJq)2fAA?Dv58 zSyj%1xSYB&tBQ9G^{fz=?Deg?=1&%n9fB;K+)}QP#8ZuQ_zS=XJVYJ*2$o!EFx(EV zY8p=7Z5kf9SMQ=oO^ab42Ypcts~g3z#aKv$$3mj|5lhAie6A-0mOfP}n@w{;VYA65 z+qXHC-PywX5bfjlzsf#rAE5bH|Mu>Z=Pawo+oJhQ9`1b~7X^>74U0tO+Yxsr6N@rJ zUQgX!gH5JrW-@p1VxZByOCNHv&us20u7>1BUrk>vN56^f?aPst4MAG=`Qm8M{HWyX zJ-vb`Q(2c?D1!t=v@zB+ys;;pxq|Jo+`!s+_^Banbob-W_TS{ga$AJ9jZsWfLiBIj z%bw9TM5A#Ew2^zXbbzt%QPb%0R`q_(P`sF-`B3vB*J&DrIKjar>_6RL_vzr>rz>|K zo>E|9B<$O6w;!HX?8%Ch2bqQzQksAflDmIFM{uCgp7;Xr>|_89?!Ca++16t0DOG9I z%BFf^_IiaKXDP#-rEKIZ;RQWUJq5p~LZ+!Cf4uWFD&#p-^{wu_k;E#yNBy~mw;p|o zm;9me2!S{MwD;~|^88idkYH7jS1DW6(Ac7N6LEijl)o!(!`@Zh2@>4@{tSt9|9z)O zjhevLS!`R%{l!0uK6}dnZfaKZ%PaVn55B7@#><`79(cF9%(4V-wpqbLZ9d<{Sz#{b z*X^};u~r$#+o?=fiR7WB>1rhT*P|bPt+v4?p#JyH1e?9S5!sQG`Ry8OkE&j*`uw6>Hc?gE&*#oRKYI1t zYc$gGs=DWYy&j9kyiPf$(mHh&_>w_j>sbbIwtP3egq^k6WV6}yUADN)Ch07@xkkOE ztf8pIS9rnCg$oKm;&!!OuCR#cX1UH*Nn`8s&D|BFJqG3}{HHMdN<)^Cneqh1~Ji^T_ma24ZZ$DZJQkZ2VP(M=HI3lvn&ZChklRO zjC9j`H#Z{H_uqk-Heik1<4l|9Rm#IL5fYAx3g#%@?I!GQqc6X|zy5imYQ4_p7qids zMISX4P?e%FWc%vw4qyH7D5a3d62w`O_0yC*pKg5j1j37Vq+FsNefiDj3vimh;2o+{ zLI%d0HHKt(0gN)Mx#cEXt1}6#w!-fE)58^2k00RF%U{8h2>T6x!$YHN z2qT}l-23FkJiUaUvNd|iS>t5J55*7fM~*rOIaY$u0K-;xyMjrRA+}65#c#p23v!2j zST~FzVNkOlBJM{8gt~=&MyFB7-2Gt>>TJk+*@}woH3P#+s?PoiF09BGmhFjcQaEU5 zZ1cg*qxOVZ$%&!+=<(X}em$X(+U;d*JhMfDV2i3a4_E2ui&d6Fd@;LiII-W}T{8Om z;gWq=;Q{bL9D2Q3E*_81*b$6V_vt+9BG-k{crubZIT4rRKF9YP~-+Y}hHU zftwtCKV39<6>l2b|J~QI&kHnb*Si1a+Y9z(-S(Nf`;+%zOLu?X9&l14UUpI=o}biQ zWtX{F+vLBq4+%hU*>r>-vk#-<=l8o78{}GS%`u80Q1#02<+ED{pU{{_@$we7l7MT| zx?W9+m+!}bc=@-VD_FwW9HQx_*7dvMhf54pSR}hYj(u3R^jnU;y}QS{^4kVw2=)fY z3ThV`#B)^*!g@p4pTuWOqEYXj;<55H(lcj0oq8GXOX{&5RVYxKi6-@?P?XJ_lt&#;_Fzo3ly=;HIP2=^DG z^?LQEFTNOE&ZcnPpT3wa)2lB=%bU&mPhZ@9&qexBC;B+l{Yd!;U#F{YuG9O5Sof{8 z(#O2pW_6pvSn0=fx=Ajtr!%k@3mEoa|9Cb1Fv|DmpT1ZuHwf=xVZ-A6l>e4oE@z{Q z^zuCfDz}R(RAtOY+10!4Z}4e&*8m6p?Q#i@db-F~yLa5Z&%5*{5!(wdcCWwrl&I0o z=R}VFO@5mF3(R4HYh(Mj{QA}MW8rTfSLx06uhUI7UniKOf#sL~%YWalH)uO9vdzaV zTWtRkhyHYbd;AO7mhE3)Q~_Uljk&;g58oReCs_jkc{|!GU0Dqiphy2$+PT+;@ytSwA`~K~i-Isfv{BLR4&8Pn}`nUPjzrf#=*!~S4 z3;*iyuW-#}y7-W;N2}#5`_mWG`MWPh*BK@}|MUgd?2FOI>DA`?PhUvJ;IotMhws1D zv+tJAC{hcr6QM?rld&@o+qW%7_b(dhK5tP(@^g~jZk8j6vzN2kMY>9^!E9`o!e;-9 zz`41&-E5YN(dP3FeE;oVU@5PrDMI-5I=h0emCn}L7x!NhoL2@f55LoBJ;&L-Nf%H4 zQazSKTzh3VyfhU?joU1Kfomu=1P{VsJk?5ryqb_dF5Q$lG?2_?e^Z# zsbp8Bd*kk?x7)jc^DE`5%##}1Zcl=;%3V}%hcWN=hS=*Po*uECpqqF?A3R;Z@W84# zUC;2C?L0;2u$7fKVQ)}WojF~P@Q`RcL6`8Dh&f$na9753g07&D<~%_+@Z^Z^1iiqW zP|T^?{!_I4cW3Y?XzlN4uqSQgk#l&0hQ3_i?BwykIhBVg2?e%Aq^D{7@5uwFY5(s@ zbtmWrDwUp2&^0_|WS*dpsMO~ zdVz=TU8m>=wsndp=mhqat)8&)@4PE6vHn{@%oDWt`>W(nJ}Tg+pEcT5{yjmDa3}F_ zg6?1sAN~Yg!;|XMlRZD!DlVU_R}d$eAKIzGJ3)6)lxI0z@=#fSztcTJXx7N_w0*|u zRwq<+JwIJ0YJ*0dC+J)rw8~G=y*zYTIzh+sRM7Nv52T-O_EUE;dG7BN(@%G@;3w!f zc69ts(MRm>e13usVW zcD*`5x3H^sdWt!yeVr&z&>=i5q>~LIM!)@Zs!_zA2H_{@6AJpjr+ReV{X@H|B+^|% z0HA4dBDScBQ$P{y6Lbtkf%}t9F(Huv*VN9EI>kfnou@e!PhBaX^fX6sI%)7K{fv^# z$BEOT?z4oiFPFEQ6Lc?+eNa!)n>;pLJ;lV+L%Yt$FJDw*{C|_YamDgqaF7K4HUC>O zUC-dEfBk>{cr8a$c$gdbF3Ae@CXnu=vq5RjT^Y)6w*UOc$+d(lZ~hgq=z4UM zZs6Y*qwLcLhFFg_%hBdK8{OSvG|N6@v+-8J5N z+35Sxm#=aa@c%jbI=fCkOqVPC8DG-hvfCAm2j6IUvsDToT}{`^)fHU)hx`-2kG}jq z?biH>_Q*Zd~n}j5E2pvL^E<&gRQk77Y-h>bWNazrX zNE1*ngepa;p(DMQ&^w0Si!^B>BE2XUu%KTLzcY94y>n*H-}}$K-+7`QCk;b#fMg<5w5JJMqOAuYdipT0K2} z=p3HLo!3iQv^KeZ?yCGO!yC0Qx&K!^!RS2IcIkmk5zAhxy4~<6_I_^L@rS-5cP@wC z7yP9SSCmMs8H&Vy_PdAQJRBtgtR6O5A_=)k*(gmVHzmc@zzwl)Vt+0TPP5k16UWPH z;TF`R4v1alKl7SuFMg^1RgQ*PI?)IL==>Ub{WlHj`6&PPg`1)-3TwZfic6@sPhd(~ zSeEa+_+@#T)#No+J-VZL=f#&-txjjDvW81AjBi;dYKE$gPMYrL+cjXcG}oK1BTF1) ze_j<`L1g2YdVD@RrvJXQ%2sw>#kJ>~jACTn2O7rT-hV6GoK3BN70MJlOwo??wgasA zV*eC{>ptjH5U0#Yy?MPu#+R}#f#tW;->9QhS%n7pyxh;Si~tOt&G+l0v48y1Yt`fF z7@TC8+nRxc%ZJ|oFscu#QkpHhqdaH%FVd%QoF*K1|5W`~Mog-#-mJ>JYK3R#GfZNs z#%}#*d#6~O%Y1dhgO#sI|O$;;)(;|z%fKVX|hKhX^y^xwzihC_+w&}^mp zC{R1?lTGgXGBVGKx{;{!#5l=P1@=EIoM+qN|LTIYFH)^9?RRv{TH7JF;1`*$e0*`g zwz>cO5K^E07q)!napSX0t$v3xL(Gyk-DFua-6n&S8*dbrm#sgPkzE&1=*(_wKt))x z03Tyntm%z&R0YVdp#ZexbHK7SV@C!TEhEhfOf)~*NRyxFEP@rDHk;LbfshKiC?w*%)UcUq)yqk16CY}5lgk|j|-cTuzzj@iO zPaDxzc4J$`eKH$BThwDl@Vq~?@Or^y^hmCaRL#XHQq!u~y}I!EaITbLec+?Vcr^3Y ziVTNhG6R@?TCM@oqfKSd0iE;F+^SX3mhVo|1t6V1ty7v^J)Q&wFEbNwXt+O2;K>L@ zc=9q$_>nGVFg{z)V6hP7>U@u{(F=Bh2k90~0oz5)XZci1Vc|)@R)|O5SEZ)LcjY|3 z_YCIc81EmlW!<7R6^ZjB4@jC&`LX52*eciFb$KK~HE5CP*F=QcvT0ku^tl)e{9?^l z#?x#01+0Q)7-!vB6Y09KV`F`Qj`D=*b6m*H-aOkqwyALil+1M+EJ6@H!)fx#Q}_kQ zIa&Z{z1s`X04*;+IV&7)KC(Y}-*2!a6Qo@rLU=~-oT!QZHWCkML*D?9XzXIJ+0qU|k z$MXBB4qLT#9Pfr<=yRy=<5jm&a{tscuVk zWe~HD6{NVc7udGie%7-q=T^3ww9s# z7kV}QCI-iy^7xH9>XLc$<`xuOt?TQ9N3?V1kMj*|eixqGV#dFLV?WI3a-q2Q;+!#& z<1&(8ehwT9n5-XUwdOq9%IaSbBQx+7+i+=@XT*xpr@fP+Z`Pgx2GhGXew~)6`f19o z2v`Ho?cpz6*H;A`Zkv4Ii^WW?NpYmb$KXWvipjQk-n%rKDG}$Y`CV3&p0W!StUvG5 z7Ffvef<4(XtS1$;X>I*v{@BL*uy`__xWfx=jdeP&c0}8uh-Q>H-H!S5R+K;00D@id z6m|r6_M&InmLp?fj|cjk>)pr0gZrJ9wQMz(G5&7ABbqxJ8)LBczkagD+8rcUx)saF zr&0Anwt_Y6)cZ7PTW5uB@2PF{2HKT8_Pu$`3sSm&&Y29m@>Xm3HtWrr z)^{N5x7j!Zfw%`&GU;x<6GcmuilmgO~;gRpO z9>xGqg6w5JaIM(wtNQ{!?P0C_19zHk7Atb%N>!3Hp~jIm^xM1o;?h>0IZAqg+0oZ<_)4vPLq=MSM#4Hlk~RL3ek7v4LOenbFBT)vntCayP=aKCJKxl? zLLAT=l$9>;;i{g3pI++Q(*DP=J#=%x&WgjtB0m+1K5CfJqVm#)psUWO-*dntY?ULGB7h zE9{M+Td-^Hi^U|0!X$ck5i@=|Q4{RvYhIdd_j})JvHLK7ZF!1szT{4>a=iYoBv61w zyA!+v0Q{l2Mjr!Y58A&$MbJm$-C0rXL39~TJ+$J8v_(``yt304wK7iPbW!76LE6)u zY^jgmgE1BIS0^KTM-i>*Q~omF0n-84|komzZd?%$2L9smd86PTi zHcJI7&@!a}IedbSN{_$PiS*ku|-I=I0NEc#0S-!pBW?@Pz#T_C9D5Yor8zoEs z-Bc(Dnm?v5;znTg(P>B46-8j8d$&F)OR{XPY@69L=BBmIs&l5c<)Z+q4FB*8^UWl6 z<K}VrP?bdbus3gTQuw5dh}+&KNs%;`1lr)}Noo7glX zqa9&{8-{t{M@(v+cc(xj>C7TkwN(AB@|HIApI8cAj7!f|`6`h0MEydY92scVh|_Ib zXc^AK5KL!|NPNB%G^73*_N*J4Z=5E1~8}5vVQJz ziaQRM5VY0g50*cYnWv$-^|!{w4c<-;zj3B>@3h(l)cKVl$l_=hDJqh=XCLQ0CwYVr z+cLF2v)}?z_$8jm4nHp98QrcF_yq?R61o9GP$FNRBK!(*_uF zT{LpRjtWiA1j7ixq9A+c3nd<4@dgo8NwYjE;{%*LB;D?KL1h7y6X!IwW?X(3+XyZd z5VUkDYi0-Camj8&1@tiaTwA0rZW%7yMkO8qUQm^hMiL>1EWVGa!a&LgECG0hxjrls zysW`k>H;ytu+y#rC~2N+?hbd9zMN<}721eUm>U*8aqLtI5=1evG-5CLyOsg$E#Do4 z(MH4VhATZ>QWGnm#;*T0j-_cupr>Ao*FNDeK9)qGLBA^%^rtfO)glY~NIS*I^awv5_w2wof)adyUsXlEEt+8P>^ zYnH!@h;j_(FJT%h83<^&*<5!^PfVdAa1r>@I-o47ZQ_@)h%4ppioi`(^0HHawvw^y z^4){qY)C~#V}Bc?y7!Lw2a;MO#12FKW&Ech@uC{}DQeGB&}Zt0=s@l+=G<^SN<5pO z)YUxoNu}X+nfu%lH9bG`O^vOHblQk|Vh(W^TSyMZoJwvLg{M7Or>xI`+Vb9X-||&m z;SF4XDx>dC^$XGtx$jFBBv%&E4s=@i(EHg?WW8I5d>DJ@<#KrnD-QTMZ!r?Fk7aFa zhb_=?_XYLkdS8;;uH{93jqFsZM|41-0$g73xEBLrltf+SF{@ha1_hofvMtX>d>4O{ z>d%&XQB?UihiNZ1Gu!15bnCLdnvgVua@^!aNp_v^3SU3#=mEPIkXznbTg}`&rX(bnCm)uYo+74Q@!5>h=C?L$8RhVSX;yQa9xlS8AiQ)KwQ`a&$d4VOj zh}SmVbgV$?Ut?*ZXuc>rBSsIdqx!wF-A|^)FJ-20i%vcCZy9yDCL?%>H*n*M0(R6# zuB{N;X)2Gu?#gdVd$qY}r%+s|@*2Jc+C)~@iwV$@z?Mw}zm`R*Hufqu8dByw;8xgY z%g$|I`mrwCkx-`(Or0?E=OkvC9Qn+TXMvKlHy22{r_=}BEhdVv{qk$;x7#&ZW+dHT z>3Kd^hw&`GkvrYu_(;)?-D0Pcgz;hc)uOK$K8pN7wddl`9?=P2_vvNSpC=Y5EjwCE zVhuIYH>UuLds5GA9If*p_v3{L$x{qW5;gt1CXdV57 zpJY+Q6q0LRNX)b`;+_Xk5MRxdJf%>pWVM?Z!!cLuEXkfQc>|)cd>$ID6xH5{H%mkf zjubEalqDWN&qZQfKAMr3Ls@iMP*@+c7OJjHz|Z`dIG>tSXfC(uZ)q0Vf1{&&)@U_< zm3mz-vigYLYYZf`=R7*G;Z847ctPV_XE_lZ@$M|E-GLhOMkQm}JiEnEVkOI5mqIKW za`8jZkDjzge})i0xM7)CMwwL4F5jgld1vy?d?8Pb+0>)AB1!e%j&)Vt1L)y)Z<-oM znfCmcRvMXd-i!z^nX)u2%e)lPhmJnh+HNfZ+~qCP7$A{+qSIhAlp+t*TRG|`{d+fx zSev#SGVD)af0=_hUul?2yzU874kYMFCD=5lEuv+~OABe@oCjd7qPNM9)e|%ugccl~ zES?U-cAJKTYjF+eYckOUrxDE#fJ0U4yX750;!}^kmL-*4S%p#gD})l8 zjWa#O-i)SCILQIaPeh%Sf=Ukg|5`5vG0Nygb+TUwHUqAfe$Bsx{{VPiJ75S|TTyBs zlP3H9Z9ep-p3^EDK{09)TGo`ts(3pR>s5x>ssdsr5gg)kh^Fm<6z2d$KuWlid_#dtU!Orrv(Yn*tM$Rac#ArdX3 zcH3YcPT=z7;(!H$x*+ZnPz&x(tt@rzSeZ92I#Pk*>MUUTl~4b{@u$g!^b!D7pD&q< z+O`Szw2eWZKsPJHOv_RCZ?+Pt@gg&R4R)rnHiG4B*b6#+3b{`tnlRmkAbulmr0eo3 z6|c`bfvgs!AoK-|hcZT=UW_GN!woi6Dx!&zR>)*0!o;t0-tuMG7R9aKYq?hsmS8Ka zq9D!FL|5-fJxloO%Gtt*r4;w3H0bw1O@8zErQ@9jNE;9#ONh$LeJA)e?Yyw3yTOL> zo4Nu4(oG;kC0*EW&hW7`<>}SLD7H=Km@Ja5KFo2_;7MyXpH<%vw^_K?ii#XZ--8yn z8v7PGqiJP~eSqa8{reObt4v+jIGfGpYWVg1`ZZt;o`@qB<4S%bDI}{LmaoFR{{pW& z^w=iNfVqqwdvmXNa`Vw?9%s61>>8Ud ziHZjk632k;`_{QiC5#F3F=APxY?epV2>8T~@m>^NOtG=mZYTILaWyfszaIA|jxC(= zYSv%Cve7r49b*l>M11Wwof867LLN8keMfsSNpMi#c~#yT6OS&#j(AFE-1WBN&`wcU z3_xTB(i#92P=K!;g-;ROUmNeG$i%%ll8BE%Mn%yQUQHXd< z^a0-gxbX#q%y^UHMw|7LW!c9W6R-Y7nZqvPBzPF5bdZCQ7BbDF2_jl;LdqOkD^h7$f50_n*+J9`r~gsKE|ERR zk}^Z3wZO?Z0&oqfT|Btc&dM}QaM{sx0u2IH9w8S&VN@z!=qF%3pWhHymd)sWI6?gw zdR{j+{2(UEv(fyetjr0zc{Jg3OTsA=#$-ZVu$Am5v&Yk_#ff^xaAXtt8}-_+>7RDj z$rUfDQ9aiJ!8Qycn!2y{Tfwc>D*omhdsiC_6P^$OTD4AF#ay&PET(Mj0A_MB)|K~O zu4m}1sMxOQvWvah?7h%fkL#j@oF$^Z?V_8#0#1kKv{zBI*2X(grJl1F#!BDrIL?x) z&=D|<(xVVigk}~7Fu#n!tj~_%92J-QQ~RMwq-zdR-C!G|%K{Agdy5LHGbh zI!z{AJ`f}*LA(*?7M(nU4moVkw#kbkn_rb?fR17GqP@xrV=GZs25fKoD06> zDP-Z2IZ`6?r+%H;L>w=$5t;|AZX(LZaFvO$z+`HGZ6j<#ab!WIHaJkWeFCC5CM%{) za8vC?32{<2^lDfAzbfm@{CW+-`hQDFSs!6g^l?6YqHqnwN%=7rT^Tvv7?MP{lP`S_V0baJ+c-?%aV2 z1g3#$PRsvCRGBH)^z6S5o4XtK_XVJVXe-b)LCsGTR+0UKHgQ2N5Gd_SSiTVPpmo@6~InC16!@ zf9FV3irhLym2L-2;6RP)I1k~95L@hSJ$`DHvk|ueLMZUkBfVnn+6DMKO1Y0lU#Jkb zn=*u*C*_yRLo9SyNY_+0e{Y=#J9Zb8guQTTCxDv|MODS~EOVf4W*Zxh%~EQ%UmNul?o~6a0XeaFRDH(NYk7`!|TP zC$J1nj@k*Le6gP`hUkm&y&3lqtBYN}a`0rWg|VjBDGR)ut5z;l=i2J^`DYoY+g?@h z?zH{hj0mc2Ke$i=LGrR*MZ+FgqbFx_dDV$X3B`I11_BcFZj`om=r%-iU53v0BS zWa6Q~Cs|?$H!+f}uKm`Q;a#y`^K}*2;YaJ-oN#H>lhPdnD!w;Uoj#AVykFn4TJKqz zf2u|AI&7-1LZ!H6mBRJ!+oLz8*O`v_LF-zm6Wp(G!fO$l3wuIomvcth@$R;}kt%AX{^v@1>6)iNipf76@5h)XW} zJHHQs+%@G;j(NSn6kun}t|&zZskD)8398y3W)ME#TE2-hMmeTr-dnpy?jsj8zo;B^F)d-qjnrH9=BmFi=vj+x z(`0UL;j{7dq?F;CdU>MQa<1*}Df{IVOcS5At$jhFJ}rt?i;?&Kks`SqI=x_$W_7k- z^Zou;eV!T*tHcuo+pHs(0Q!nP2flf2`B=kDuaSYz6#r^O=qNv@=JR2Am!kTl0$vJ2 ziV{#fRLt|fW94a+xgAe%H9BUc*dKRi_kHd_rPm={Q1hSevnjYtP4dI2{+0ma>OyTv zRjI*3#w`U9ZP3ALVEid;?K+njbLbNK70ukh4X;Y)4S+o(YQZHnclGqQ(wSxZCKWP7 zJT!1Xzm^+V;+({&KT4bz_spHaNtg{R`_|ey;dDnYSj<-A$w02 zMrO|Wk`nvxc?)4x?Tu5oREGjh0@+-m8!O|ZtfU*=d!S`nBY5j81kflJRFuiV4n6R%@w&b2p zapdB~uZB}AKWf3dt*};PdYn9Z(6+s-InDz$>2IV=f1IjU@>?>0>Gr=sK-C*P(5h(u z)Wf%{x#vb*q0e}j(2gb)kb)_$7hblbk6+Ni`7Q6=t-gBIrYm^MBmQvwkHrEgb6X9$ zkH~HrL6~Tlzhl_2ii|Aedfds-foq5!DrLaxa<2rpedp|E=)ze@2WbMs73!~wD$1!M zgte@W0j(Gb86A7Cp!LEk;Ps&6RD4E%ioPgWF`%m=qAcZ)8sHj*CuMEHBSqZ|jRg{7 zNL@ExlOYKr!3}mj6=1G^=cD!{2puH03)k-y3@TPNp0bidqA*sEm;18Coa}#f-69t8n8U0okwo>uf<-%M>#2cvDaU2bQ+> z`%DNs6lb~4wf1#HTw$6X`%p~Sm9zgOxzL+yslcwHenG4CbH%qq@H2zWghpjzV@!9S6*5G6%3&D(1>w@Nhx9m~4Re|r=5VR&yyhwzJI$~k`Ies~cdI00- z&4J~oC2BZQhqbdgT+W^F69kc$m8d+fO!9f*!U+K96uI^*px3&rIY6cJ@^S(s7M6*#bb>jT4W1vLkLW{eCs%7@ND$u&&rPbokBqJ9SCjpM&aLkWz2ab2hyg6U7?aBw@`XzGOIbfh`mx zhM<-Vdvkgt8GK?*0^~yI0A{iSPw%1THW3FLyvr`ipGs{i$8M+|G#2@DJ_tTo-CQ!} z_a;>CV(#geBCWSK0uIFAtox1jHy**9L0svX8H+uLLjFmh;>TnMyRO=DVyJcWwB83Z zhmcE>XmiZ9C}D{wFP)4oYthEu612(IHgTWm5L|WIZy(k2(|U!_`;4FmzuNw1dbo6l zbDc6w@^*lR5;CF=I1+?s@6MX}|8A)xb{smL?-9M4>eOAF?yHxE}WXlZD z8&7I358nPy{V$-0fs9*e*s|HW(dm+xT{I;T&JDhMy5$1wezHpzcQo&f>+Z7N7vb!j zp#`Z+@vcx@MSBD|IwEX+-L=>IKA|q>A*aVa`Su14x07k1@9GXhR!tTHlt%>{e#!mf zANd_T09j2#SA03i!?T54k;ee<(T_BI(y1s7Rw)3X;y9$I%`R4xwr{Ayb0{R+~z*oD%e6`5> z1ab-Hb1P$Uzn`GRu5RZNib}ZL|Lty!9bt^XhPac<%JA}-_1br;m}Tf+X!R(tDiLbl znKmy_KAvQsrHyF)g}8-X76@;B5W;FkPpDe#k?_Y;fIFPkV)h({BK)`-;aTEIQ5*?u zUJnRPhaEJ?%XbWh$4Omzf)rLsJW54CZq7l~7q-0D1+9E~peG-{!sE6Z9Vxc-)0gEE z9cm~RsgS1!QrGPmPhWSV-wb;nd#zkVQ^(1Jr1-Buo1n;PK1%O8q)m;zY0&>2&pG^P z?V1ekl7d>7>H(Y7Y2k#FQFPnDlM|Wx4mZ*YVW+jUT%S#h^i_PeH=f1(Fm5XOl zZlu{P=?}JXxRFeAv@u+j-wk-^Ah#}F^%w_yT~-pQ0zxu;156B5H%>rY93&NJX=g`U z7R=*#HNFa|PpB2>pQQMcPTN)D3Uvw@1ge(2`!qr#$+(?(W!E!;rvCHzy9;XT$eh|DGv{eT#vby{y{FPM_N? z4M@5hAh`#Y%8$s7uRZRkn|hfJxL~f+(?z|FWO?w!tY4>;2~DEt6qY9ktluzOp<=U* z-K32aKcTkpeTT?wKL0MBJ2n5fJwg6g3tvLXmbLN1Jg6l)Zjxeo!6(*svCu5@%I+7@ zka{V8v?ZI+yZWr+;d)mPRG&LK8zJ8eYN2?Mp`llUcH>T z^J@>e=vm4o{h`Y9U0QQ3YM}a=nx;RTyt~Ytjtl(85~@;)t+8V@IS~sg8Vlm?zh|$6 z!CSf4f9tMV3|F04e&#_f-5jZ1LMAVA`niGXqS$08N2>DMtmvs~3v{tM%)(jMSRLKu zNNqz?b3OfA$hzP>5pBI4a|U<4bqay_lpQGFk3G)hCI;L ztsrW)!xbecKYA+D;EPl;=!+?3jEIAd0Dt+%D8KaDW2K_NgEj>jTfKO+Hoqr74;clC zr?pNj|1B1$a6tFH$MfV-Tz!0HR2jKdc@Tm|k3QCHf@)joh;$LcbY@-HE`HKX*25jI zKuy|E>{|y~YmR04Wmp8lZ#sQ&lkulS?IemZ3&@l+5^5CPG~y)i%Gdn|g#vV~VG^{V zEz+g((4huP%Xn=2HlkLkeFCAk-AcG~Q+!mA=fvpE%%ALVgjKAK0I8X%y2vgw71(WX zvU;Plt>K1@{K}Dl&y~tbJJrAG#jLJ49;&Q*?F?SjN;I=Li3f{Z??smXeG%$K@m=4W z$&&7P|beYGX(8uv+Ns*!bQta#r#(JjC=e% z`<>NHf93@%n~9ey3`X#A-^TRMp5uR3-)7cvbU)CB&wa(&@GXCqoQnc8zaNE)Soo^uJI2@l|P}VR4x> zBjZ8Oyj7~W0qsYd)SVKxrj()O>{A3P9E0a`jCE=WAi2N9LF93x|8;}zP`o2sg2s)l zMPmq-&Vk_rQwD&CdqfPT%`*vb_wW$dbEy%Ae-c|v^-JJzen)y+V|hAr=z3726-=J6t-1$C)%5y#2Zt z6jm#0M}*?C+h=?e*2-i)%T?AyD4gD8=3I2>7|^(SqbJ^V|-z=*Y+n5BgEv1X&KS$6>N`#gN< z&X;6kh%5uE>c2kd={|1#;ew9Al4dZqE8denrZ8*h#4%0EG1EFazpS4@NzI)nG@x~( z#9*O?j$EqCYVOLg71OTuvYvx~tp?F;?Y4S3{+Dr_GNUKI?D!a5JM}1(Pn$_`%fgwK z|2|G6d(P+j`tp*OH(1L1T2t%Lf6Z&bWpor95cbJq89NG{s5aOiB>8jeqtIJiOGeC~ z3Q)LxD_1&9)%^g^`#x7uG0ebrZMIl#^78P{|HyksdRYUcU>BF`{}Un)UWDOdJ4O zK%QXqAZ=8UrZTLB&nbT}gS|x9R>T3qBYwkmmR=Q#&otUB)%xJh61<%EVjma<<2Bz^ z2)HvhK}y2|J{hLVETb<1SR`oe9lV+l539>z5zx6mpK`Hv$=QjPlUBooX(b8{}SW%ux^_FF|1Q+ zD*0<8?B?J$aHBBgo=v*JY3W7;T!qk^byvdOH|R$uCuWtwHuC7{$^Xg3R_a7Mnq{>t zyPu%TI$%@aa|=_8zfuhUF{f}S$vIHu5;7D8YYavNk!bQqOucZ_vV&3oBC>iQi%Ym? zJI$?%<^jXtlPL~OEV!oaBWt9Xy0>y;4Iz3OC`z>{?p96p8Y;3T#!9iEGK@7X=H-NH zIYppa)@dWf?854kXnqOU@X9PQ5@IA~6mpFf`Sz@nlq>_){vL!>z7l4M}W^4Ox3U zx!Y~67?L?%k>DXX>S|D+&hSHnDWn;+Y4jlGGyN@xVQii)9n^=8hlZ}241yvto>e;( zE4s_FqQG>jT`gE$dE0A|m+#EbZEx=jLD$7*9WjXPwHnIgn zJ?Xm(0QnEAEw+E}nfnsgysPihnvFS@?{Sdug|OY6zcX(VFdM79PWe~LaZ-l01Ty(@ zH+zKD;A!`m`WdB1YvFHt9HsRdaA~5kPl7tv=m+5KeWSNTtv;0pb>LegJZQ?#@h%2h z-O<0*ZJ&EZ^UXIKT@EalyD0$lNq^?6dF?*ZxE;_j-L$0(d$tvJX!zK?ePuXdN27fcZ@@@7foAG^14 z_RFe6)yUpNuho&o=ZYIR;QGB>GIv?H5QJcm?z!@(c}W zhG=~1EZ%E>cplREKyz~a9%UPXc}4u}Ect-$kau{}hw%bj1qpjbs(3l}sf_(VipACG zxoeX6v5gWXzOa%ZetM)S5LnI%?k`{?+OUCfk_o5527n-k0hSzZ?jiXa#|0X?4Av8x z3Mk%QA^eNlJPlv(_-R_{*a&lggD?sYa!tiINDuozgp4>jAU*x4{&E(U!WA$=rDh*iMaMMBH0ajN)vA9-F>zJm=z=<}6~4i4-*@#BmtUo&RK7 z>vI@r7CLbJU7Y&|6dGSF%?@(NeL~eCfEcmY5FRnDp;JKCd0PvAK)Bx#|M0J0#2LN9 z%t~}=l!vlv;nE|l8%1Im`bqN8L1ynp*HJlRWry-L-eWGowd)6fawD&1u;>>eldy+u zFA_TJ5@P!%vnifT+>K+FrxyP`-x$oECk0$HH%JzHgWR!A@nv7wOSX$5^cM+Wi2=D7 z-y?0Z+bWkpYO{tMFc#Q@NbvdXk4uDc`9hlUG08x7%7*xhFs9#hE17H$|61MV*+?9z zno^nIj`ND+CnMZpq~WKWTtj*qj>^=Pb0dDmRdR(EgYIn>h-vodt0F<(U{nw0vqXD= zpc($&*I|rL(vBfc!X-mtOvr?vJsFOqGw-HN@cMhPx=*d9)8#2F82xs!$Qtj!p93X_V5s*TEw1%bS`Z9F1rsGb#;1Sc*8GGM`5&txssI8jW%{(Jm1)sZ{dMR(L)7=Y<6aZ~>d*M597kB)-wQ#JT@u!m)%oXsW3 z_h=t7KPV&fl?f*Pr=Xai@%ax2s$687B=Ixdf_tVDK+B?=v*h&lHvEYq0`tS>gF~LfKt0p@~%9`*z<#g5-jMJ+o#NNakG$cPG#e|?*Dmc^>fB=d>qt{GYABZ=k zX4BU4qw;^Un?|1*dTxh5v}UqCivG(}ha5=7$=bXh-!fC-`koG%t!}UV1Rbl^!pkI=V_kmA$s%%um_gTNky(g<>Wn6!c>Stl_z4 zg~@ejrj=$M*0XOWX-T4zB>GunLms`31K41qb;+NC&)eZ1QfsM>lfH=IWhJa3NEVSWOd8|D zMn#XZE=^^A-u^7PPG}SgyfkW90XnmUsyMaGryLb;M^=n2V=RH z%`ZfphpWeeY;7eB&9(T4g3%~1L{{rq0I-XRwQ(b9ICzJB?B4Zv^NsHq0Z*PcO)~?X zJ$~MtO*mEc<9(kZ@xsaAhEWUo)Z?e|jV*t}$iLg60oeskHKPsr*O^azy^=m%x2G8< zCPvrMeV1rEe1nIOd1*Zt*|H7DiR8`LEg$p2EINaDmqxwD7y2NFGYw9F#P2tz22=n%b9{OFszXM;N94a1&*E#bf+-1cQ$njE+ zPR$M<($QL+wRwT{ca67!kQu~ADb1CTl23~fh?%>cL8wot# z{oa}AfdJ)z`v_MPkChGN4n|E*h$!R<>u8;V zq3xnwhURreL5T?JApo8zxv3osu!65XCeoVn^0Z*byKY44ccNK+Mc|T|BOO#n*x$7g z5=kGQv#kWP&(+L<<%JnR^@55*D6zTmEtpK{?3|n!YfTBZW5X({$irRD^OeYKCf$;EQxI^-wa9(q&|R6cr)6VjOQu5G8P>!R{d04 zfsG0y6xb@}s3*W8WnHu~c2&f}G=`hpZs>gY=Xu0{Jpl~TFBOd5ok5r`sI$FbnwDFDzyBT+7~N_W_uzV+vuS)ia~k~bdy{}s~K?K zWFea=oROH**#pV{#nm~5XBuo#I=1a}Y}-l4wr$&1$F^cU@at4l)}1 zsUKh3L7C~@101^!Lg5i(E0$;RX88T$*38mZAj}(3-J|{>ph0VD8vOi6y{O)NJYT*$@`XrN%kY?oj(niN9W!N{CWW0vZO&2ctnO3!$ND%ZbSAWe6{& z6<#soXm^wn`(u|ie1vV6-v5o+V>dUzah(SbG|Uy(11*(c1Y|*rnY}M+%&LuTv2l82u3jn^k3_DnRyV7G9t%rM zgE~VK!*-ytXCCi9Ytpza_CE->P}8>yrp9b>0y7Q)Kl3@8)wIDd2vVjqm0$~3nys@~ zKaPX*H7uuLWYb~18u$n>%*QQ^t1?N~-H79tVDftP)>25)GbUK17VrVXi6FOy1!_LN z;3E(SFg)FHwGeksNrBV%5QAhhHVuZ~C?BKa7O6sAN7s+-i>^A$vlx6p6F2*2tY9i0 zFQpgGCsJA`lmY=of@=(2KFqM3Y8uTZxK=?n$wQcNL~WusU7Xou%hMwgoeO=qdASSJ zL2n7%(N#wep%QG{q2zaOIvhBk#zxstq7@P2BU*_ z_j|}EzrXGwN^!4bpkzX#38s}5bz`VOScql5c8Kf+GV@8Qi-fGl#0ODvpSfw8TPHjk zCTxhN)}sl50)}@^FU-AQ5ZIB;;bTQ=-6ksxZ=WTDsjPTomi?t1|Kcu<^vvrQz0wtl zbwY-1$5mkUUf!A3g{Z)>u2cTl=4|?oE%{rKz z5GZ9+$h$5S9*G{q>i7ItWwv>j<~sY}M!K1%ce;%oXIVG#jamY(fF#R$^cWTJi`Wqp zB(CufB$dB*eyv@N=p@rF3lCpP+jOGsPd9uo8=)~@8AvUhmr(H}4d;SrvU`otMaDF^%_f}VpU^tSMk!sHCS^_NJ{B2_XrFd{#4baOVQ6Z;E zv*>2U@|}DGD4f5B4C*GOi99$gW~{t^lN1Hs?8)9BFm7YXNKnW;M#L+5HFRP)C&xB>ln|YzHF43RqBMbimO($4_ zRe(m+=uJvD*}R9mHeFC?GS2Q*1oAdn@{B*@lvL){C@18rGr1GRvHO?~oN9{2-Fiv0 z^8y?FK-%zAVj4la#h#Nk|?Dc z*sjHxRtc&whZk+rymqY6=xR?rOJ{bK-Q6^~nviHs)JY?v&SV=Qu;TJczG4MQ=6*f6 z7I71TZFxf>lhYs!G%Hsbv`UE;*&V46rLlVtX z(7w)o08b@(@fo%=?1>s!yJI9r{ODc-P@>*ge$-g8#MImEo$OiAu5CT9xMh<|jx-$YM0#c83#N*sbEYm>6a`qmxSz5BY#qBNWYq?DyX_aq&f(9PdU)t}AN$+bH=25FiPT(^L=qI1PTG0_vx zs7eclLyr+CYJTO)QpC^bBo+;Iq5)$N9rXb!D9F9!9q9QCWD^9uT4_L8ADQK%uyYRE zIn@yg5nEa9if(IkFqVDcLc7w@4_W~I0rF!AgP=v?k`l`%5Va_2+{Xu#;L8=V#m4Pp=FACN& zYBE?Gx&u|~*b1Og?1+BJhSLjvKs5ULLqZ2$}hG3}f(hcf^y1=uj z9p*(YGaS1*@qJ3pc(2}RXVvdBj^6C=H!$rv`& zt-zta^>qF+?^%#_P5*gFQ(Tlj9}rdg^KyGkWFX4@gYE%H?kLK+EW9F&|09=!kQ!kg z)YYvn+38o~LlD456e}V?OzCV|!f@4c$jf3t>b#_TK{l$3A7}2xp#vI^MWD?dlVLoM z4Do8$nG%B(?j6fqm%v%Wzk|TnvtyCVf}UUv*K8!YIH)k|zbg*{mJiF!l94`_ zf?uLvE|fBgm>k}(gxnl`SwKyBhhCDWe<(>el{+y+Ap1)AdP4kT1*Eig1Vv zM-S78VA}S|7x4&@9O&!sH^fwSR-~#q$~SNa{a|}0|3p=8gega>X0rb?ilHvE8bgvg zIoPa1#97H`KsZ@Z>Jxh!dyq~^tei#Twk%apRn7(q_1$sYzUblb;UN`8wQmU zt!s|s-s-$gl)4dnb|yTd=3GGVqe@{q`6PL)998H*w7@@L&_y&K9zDAp^kKv~I?xGV zf!lHLcqPJ@JC0iWJRm^Rns2ctNz@Bcm`bqq@l! ziWcf;0Ir8UciCdY&Fh4w zOFxBwM6QmPbyrfgdrFU@LgIH_R8>lZ9z9#$q_|5`S{UVvvlR68sIADeh;V$>%|IIo z>GLE!^D(Jh0hPWYG1Eg|+PhL2sg|D5>t}lZ&AsKSr^$h?x{V@#?Hgb3c52bLEy9(= zS26opMBF$#BS0kZ>_`6G)`qhkmmoW%d^n$=^^3U(-)*fdHBn;5b(VWcmD;_p1s(KR zqf=uo!fjE^I!~=7f53A`b1z+EqGNNUE`xJSrSp6X8!~Sao8b<4S!%(Y;NEX$m>u%; zf}PN>ZHM_PsXlk`m(x*O30R6Q5lS@49a(fd<~(c#4=F$$hK5fWAk;O$IY0`5(2Qzs z-3w~#9Y@`RfD!vIzbgwYs~skuQIr?mjSTpvo}XFycW3d%0%I`sT&q|G-uCGwh?vqq z1bkdT#$8=`fy+r!I4ofv(f^nwI_H(=v z#G?B`PjrX|ssleKwOBk|?V8w%Gt0x_A5>4>sJeJc*Yz0o_E2P4kk@27&#Gy8zmBz| z$q6-e^YuZ-sDvkHJMA^b;F$>{z1gq9*^1W6<0(bS1Woh1EP21$c%3Ks0_($%11+<= z*Z2t*cxk|^_!M`oj%`9-lT@Ec~d`HFjMh{BAf1^c1lmZkdWDaqZglvutTOE^QcKTg zaaE0QgzXQ}%I5oaWL&M})*>1VA={7>fEE`TrqE;TP1&*{E*Va&BaY`JUVLxUn;;*W zY{9SsW~HM?p5(el{*LUmo&hV89ep(?ZmG7sV?CEH4Tht($=%(=&VkMKstbKFBKm*zfB-p~f&QSGnMy}H&&z-mf$gy~0<>_KKYzLMPgR_d4o z8Zf9NOFzGH(I?94NIU<+$o+Hozffd{-mG%jMPpnCt!UV4v95Lw?_POPsWs#Wuf4Zy zhl%CUFke~7$JTUHBb{0P?dMtgys7HR- z7(Anal>0-#k!^-mM&O50K&p{Vy#uVE_NXT<^)Sfk_a74?%Ck@bj;X4^ZG|O_VDUw zDn$o^PlP{wxu{<)%~cj?v@orS)K#Tq|9THZ3g$JvLip*Kq!p7q-eQ}sr? zP#c8Gf5>KscE3iYEBhlJutQul87dVT%|QGN&M!h{7~_**|Ko$GE?&H&FFyGI+F!WVDs~ zg>nKRzoLEVR1w(`s5LyO)m3RyxqG~QgVVoyBUw3m^0ei_^@Hc8#j&)v&+bKx zEL0gCo+<6CjBhwq9@<}2>_Vq0li355UkiTwHJ6^~W&Q*q8|Szw`E^TaZMr zG2T<$MS)FUQogNy9e)v?e`nCikhl+J-qqS!SL<`vVeBm=d)KlZwLCSPCH(%0OvXNE zy6I=O08Fe5pL72M(1nat9pe|&+nc?5FlO)v*hzp6>h9EpQV66CCe?Aw)!Oq`N7QPd zMqrZ!E{oLFjIRjDQ(VxYw>UT-mECZ7th9a5t7zk}W>zp(mv%Yq!t8_1>bA}z2JQV> z(O2ThnYZuWDE`sua@)zs&hjNd(v@)Hw}~k8Swi$-OMw6;*fue*VW{O>sX(lK;JzKK*y2k zcWKgcjy-*v`X&K}Hy;g{>X@uVV|T4-*~scudPCv=o@H#qj^b|1$1$*2(78HPdPId` z(DD8YA?@)_i|*SX(nb@L5DvU#7kv6i2Li7*xnkMur5ysGwKvhxE9NrB1a5^kfHGJu zu-}xx3K4NIb8hG6eegO3Bb>zN12~fUl zU)BzM|ND~|y8Ecjx7hyI5Fwz}onFh^ofQ{)&Zbs9CfDHP8@$u`GXxyR0x#twgsih@ z0C(c0BC?L{G~9mQWjR61s{0?$x9dflEiY2e4d_y7$vfYaJmcagMr*WoPx-$xjtI`F zAj^4PY0BTxbzWPFT;-!^~CC(!-zhJTf@aTXv;U5O9@Sm2MV|@50R>X#THG6$ILz+~JJP6906-Me; zqCAfB<>$1bv(Kg7dpFKb@0$zA6q2MBPG-*-62$I8Vfu*9%Azyq*6qW*OuR*y;|Y5N zg|sqH?q>1W)EgK*N9;Q%dMu6UtNPs|1j-h@j=RV8G#;~xV^g;o zBD-^WqOzMuw*ceTkZb1WKLp7fwXgo~RIjE+Zbg z!__$>#@o{o`jcPkkq%L*V(A)YMLMW2)Csk7ZhHsWBp(nr>r5pu{mGSgsh3_Z-&EaH zyw8HL**wMCW55Jyf@<|^ps@GaH5X^ZR2x{sq(D+ofFu?)(! zj1i+TE~m)5pag~ReqhS<9<;5&RX$teO9cgi=B`Jg6P$!@ou`WY)&K4dt9p=NpF`5$ zk#U!QU_0I5xOtUKS+YtPNz)^~v2_B;TU-OK;coArb&+95pfCQ9H)_HNP1S&f!Z#TX z=ep0rUG!8tWal1yfKrA=GV;CGk8n*bV@lridUsf`q#cG(Nv(|C9a~-4sK`FlzI5E) zhRc^_ex67|n$Qj*?pPdN-`-z@RMu46r2ra5WloIz;8skQ(-~sav?HLq;dPwlKcpSV z6Z>06U5o-cU$mnvq|XhfRiSl=2d%P1_C_&j3Jvd=%)~2|D>vC;4JLGR;U&dVWzROw z{u$D2fiuT9liRbTnT4=cG%PIWcfjO9SSvc4VuBfk@^E`HwLyr>BZmc)T=RM^9nd5J ze13nII`mwO)7^749`en7_RiS=?`ZIKQuRJN11GX5lLX%69@SoZ-8%U z0>`nsapQj0Q!d9RM!4rrkI3O(zNIN_?xAy&NK7YJ-55|FIf-G5I`sBc?xU^})?Ed_ zB~ADD4MSt|-=a%YM_px)slj#jziX|W);k7zuUv9!BiV5>?(|{Ovuw87-MVa~R7$sD z3J%lq(Rwoc! zZIO7r3_e*7n5D39&8uTJ?-ODR{-Mo9_9Tq!Pa2Nbc@M5TCdCFycW7$!$%Lix;AWbU z3hI7_4p#L!vvl>tD;S>Yw_HVzX)DUy+4o@D+79eFK_c$us*jRAC%Z~gz1IbuMfIuo z{j2Ke;|~p_zWcTJ?b{moYmdzlsJ2q(3b-aHy2C{tJ$8PbylaL5b7AeIsPvzqYSrus zy+B?US2K>Dj$SR-_Z_?Owr37+l?GcB9pu0QZJi*Ow{|{DfMG11~tbR4PH`w>WgNn@B$gLE~Z}3lzEg9{}g0Ev(nD* z`=#{5QAyb!woZEMxWO0l6gTJb%IhX-M)&9Yvj&@b*2Cf-85uZcrtK#2F!W3{ZU;1v z{H(qWc2P#7rIv74)eiub=fT-06q*q=S-XTY6<^I)DIkWzfAFrx{_8(-C`%2eQ$$Yk zgdXR28sz5>9wcSR%LKNWMN^%UHGE^DYb4Yd#){$XS&jx~Jwi}oQjm>f%JBdylp}}I7et&M*XKqhffgXEX0wlRR zeiznB+O0B^I1Sf5g4u`0rvsl;ve4KZ^lwl!?FAR~lo4SU?C1YNb$lq^>us^oLxfcM zM5@KXkzHY3rADiae>`cyytqEH+IyUOBns7vOKSrCg5q8EXEL&k^( zF)mw4SF6YWW$G-N+2EZLV-!T`a3QL{og>eWt#R>7Y+Zr@cc*wq+YxMm zEp(}eM5l2)$IxEKM>da$(aoFSlmi>H#I)&eWwV7102Al?38aU2g)4P}zR%mnR(&Dk zuE=PBEemSPw9Kcrz@jYgZ;p1SU6ssPOrjTMvYnX=UEx_dPXaUMS3}M_5A>Vv$M5Tl z@+J_!E3FF(eT4F;j-`r@$7%2SQth2q$j!z213RnLJ=tkhN6o|*_%BV~Vxa!^8yJ6j zg2eNa3)v_HX1b0e98uBYbGE(7;EG^-i{!N4u0J)3eLs#Ob+OmU3${M}uPaRPR0k#q z`*rnBKmAXK;2@X>lZ(&QX+TUK^Q#@(=;o_!FfmxmjH7cr34ORz8{Ki#{`H$5bKlOS zph5tjKddpzS;BRZ;9N*IoPq;eo$*RvBx~N54296KD(Q~h=;CEW%gAW&Z469p%9ncz z6Pq#v%pqQ&AK~HcT#YWu9)!yEb4R@X%!~COu$u`&SMZ(TiOf|6n+2|SA&ckE3{pIe>QNUr? z`&_y0ay+#5b~VSXX3rWUVx0gaCm~Os{fHqkR7ixG zV)7}sC?5e_&36a3imYD;0eTu)y`So#@%JXthl_6I-~I%BZd^oGW5Y1NGs_P()Q*Y; zYFdq=dL;8-B|{ud4I?BHfFj+rG{fA{nKoUCt{50NcPSdRxq+3P{sYoF`-e{Ou1I|Y zSc+a^S5K!q&oqzsDii*^&#$Vw#K}1%xXqhh9NCAT;%v+4>%C5G$2QU4N?6_8h>Jlk zs;Otf{FdVaL*wFD^Ob!bkiLakuk^)4NIyThu?NLE1cvcBcvBmSDCtS;|Gi|=At;Eb zjSaXcyHES~bI+!~pG4fv+?2144{MN!tjaoK_~GVs0s(zJIKh;9a#ERfvRuzNC+(Ej zaNQcYe{7Yhkh|RIr3bAIDqbK^b>A#Qj zo*%%iqN0rM8YsypdxuH{YRX0h0@yXbfYnTIOHQypJJ)>dJ+fU8;t#kr(SE2ocY>rx z`t(UoyC<3rUSw6~FXxC?#NS&6?NfU^nL9I#8G)o5qZ^)$_e=dNU2XcbrAd!}m-3Fm zyS`|A;_{9qUrQoss#)kz;$LjiNdP&sQ8XGfgwwD->zzLKiOri|`k#AObX}+R;zdW8 zxq}>`abJ7x!AE7kdjq^H&TInf%p$LD&9Jn1G;D!xEI+M}Qm*XB9-kCDJ8#%O1#}WR z<-a*zXYZE$`|`lFzo3s|{aIhcv79sH37kHTi}s)O;YT_2$y&ij0{p+xQx2;R4{og- z-{0#viEh{W?+0UcYS1Xmw6{xT+J;`CX9m0|j35@-?ydaf9^QvpGeB{$W9aWSyBB#Y-D$F!F99AEj&4KHj8*_!xGN_S~h85BR8RzV5s{oCRpkziq#m^mVoeo-!Hu z4|uAez!2VCo(R7QiNvUrN41KN?0NM5^_P0d?da?3BYSs$e{L-?&AtV~6oDbbBQ`rB zq?!&t+VeoW7$bnBTWq*pMKkAf%7zmI_HumZ=SG1 z7KZejqeMls?}Cf#;h);0EUc*B*eNI0nCubcC)U^i;9X}{1nJTk7661QaqdG4RMzSR zcIF>jn{%I-{>sJ+I9VaGd+$&)gPY|CxSGC5o)aC{Z!W7ZRjmPRt9MniTTL2InL%D* z(H)srx2G5VoXJ7fHjv4V^@qjJ3!lh`#qSH=L6&9^x%(;>R`_?xN^=X6>}1gsYvyHy zFYNUQDgoy_gVz4B_RMe=Hl&TsKTWNO_m@|La()k7IAN+e^IRk0>}%sE7u1XFWSsKm zhPu~2vCH|j0nakJVX_e)uc7gW*z18(B|BVNfwIC)&ds2c^V2M>Ag(E0=G7m0-Bfme zzocXTvMfG74gs6rp#Ob{!?{W%$}14nXwb_p;3}s=<`iD2SC97wJVQ#OoHw<=T+$v0 z^O#1MTOd?Mvz=t|E-v2R`qUwPcxmUhPIK;@gNc*8JWPu`p6Ysqj+hGX_=O&uE_j^7 z!B6s!UcBl_5E48&b1<1+-rx<&)0F;WVa4NF3_Y9Cr#8+bdJl15|n@ z37|FY9fdu_2-PUQujQA8TA$h0l`{12P%5X+z`ibNM_Db*J@f+j%LHv}z%wFG{4Z4P znGiaAvJ2K$Z6Q2kIhH3&hfehN+~hKzA6=1_#0<aVv4NmK;@2`>DTaY=|%6x6$AVTDtj`EG(|G9(7R zuE?QtjZItUk$0v~U6;&nCM4m^V{D5OC?dLdJhoK4pxoP-Jp0^-o1O%)U2c@i3e z)MeiGdkoV(m%H=`Z=IR+|J>S59prWQfALP}!_6Qc92aKNhx?(jsY5rK$4Vbh9oDAn z%9|e?9?)U+R2llQWXyy;fURT>R?#JbKx<7Z5{5->9Ga;ej={IT5?}}H5VzyXVvdV4 z1x1`N@=QhP*M6K!_5?)K)kSGCX(ppm6zu~q&mg&3g72YH!h9@# zd=XND5E9oQ8 z^iTS~`@)w%;nW{VMG-t9f5`Chj3r z|LcQWQ_74C{Ex=3i)rD*qO)R+oJ(i`(hlZ~e~sbk@U%v3KOw{`jh@+UM>ir0bC~SPA@bcV8DvV;o#Q`gc=Z z@AMoWk-GmXq06{C`t)JK@1?@{SsaEtD^^Xz&6jR{P)Srwma?twG3h%86&`VA`@1Yv zx_+D(-a%^1t&28TIb}TfFqAAd8*KaMzD!k9V$8oYnzNP7s3@eU4J&IR)7~pwU1c>c z&o@z|z{|2xWhGo6Cn{_{L6q42fM-J9VWCLSFsna^hawa4ynAs+lIy^zlqx&WiyIp1 z60J(xNyV_LqL-+};-$o4l{Ceg6{HkQqmLbfRAhc?DoH4c0jEjYy_O&yfKZ9FrBdc{ za}`esE0rY%#SX#R`Uz1Gjb-${3UY0EEK=DVshg`uoEFq@HK`iIj@chhwz9i@lOHck z>up|gsb=OGX`T$B^eoQ24{wQ+08*FaFjp9h(Oy{k0Tupelc}rF?$23Ju@|JIkfwNl z2=L@#Su!&tU750=rrJ}xf@v<8jt06bG@&A5)#ZuVg3;i}V%|2ApyQ;?)Ue)#XTdg> zKxb*Ff`wp*W35Y!Mm8o!r1g1N1%(|=QIhV5v#@4cL%p>_qy$(hd{F*wDZQ&mPo)!E zqV_1j(W5E_VQCXBB++? z1OYo`0{?4?7HXT~5t@PF7R}|9i7`iXu>79j^>RgGpfmyxiyeqcrIwqTVx=+O2SC(M zh9HYLLY(F#>kWP{v(3?q!il@9N+%+y8x5D*4^G;J(BR9j+IEQ0wZcrQQxbR-!2^riGqxbqn1*SM3)l`YvJ9c?Ec zCD2CBO($?^Qqlz5rDu|K)?qP(&>~^Y2PwqttxjN2JC@CmiSey+>Llb`=;|F zSg!pT|GKQhfaij%QfQCf~mwX{`5HY6Y^zsXQ%OXt7Teos_paRNJTh(fB{SEP~1Y*TQuv zvJk~s*k|z$Ysjkg5B3(^S|nR44YXk(rob``R1Id4;o4aB&uEE|K%NrfPb|~@11Xqh z0lgR}ItE&cf=iw#Jt zj&Y0hs@HbRI2D_p;SQ}<0qNQt{_k+y_uz)mGK09o$Q zDSMB!Hg1VUWEJtM0LUsOO#3(Hb|(T;V?jn~y&T8Wu*()5Y6`bjkAE>)wnvDeLC&Lt zU}V<_}l_N=`CXwf6~+E7*-` zEb?~7)H#FQe;O4iN+WvuKmq$AAb^&2D^cUAD#xKLEwvLoU)o!uNg8?ljDKLb4^SpH z=*k9xyA)>lZ{i^WDts5G%VIU)bRk3lZdb?M?{m5LO~Q_u`1P8H@9OrW1kuU5*$mNaj~|H}Vd zU6HJ*mkQ>w4N>AT4f{ffBApumn~eiWMuUi@Y;Ay4aWUZ*fEyhI{z~){6n*Ek>}hH)KvU#AvBOK3dg#sGxwm(Ewaj9Y!CerQ>IzjKhC? zkb`NrvwL}$B4<239*!mXJoXg*>L)rujf3&cri!KxjGRucJ_P|LX`bEO>%wy%;Z>MM zljlx>$0L{c)BXj+crnTeXR#y@F&!ZQB1z*P3*Bz+pCsgzE(4d#a}PY)rXULkNgEqr zG+PL0fG8nwA7c>GDvw6Kl@23@3+(_w3d%a$pQSF%T)G4$p%4hZ@;%;tRaKcGK_!Cj z4v;*kS>RN;#22Ic7aecO`uPaP)#*^GSsTHn1~0aTSOU)S0B+LO{5{&3&J_ScWk~7) zG(GTKBrcFHp2*u~*A%%Wzi)kx=6qrZF&eqG<7!Rxi01UMpu)zv1Y1A$ws}CLLBk?< zt%3K2X~!Ix1E?4q)9E$XZ*1Q#1v=OI#l~OAkxis)A?m6!cr$|4Y!HHiIVPN(_a>gZ05|awy_@+*a#{zk`B^>>zzjYihSt; z_BR;(PaJtzXBhN!3%`I3-frnIcBv)=>#!9uTYYh`4NkS=t zfu@b?pNP0#XWdjRt0K5xBFDVB)>cjT(etqt)0qq0=@t=fhGgA9gn1ybkZ7pR5|;nY zOBiV5*JPN53MXIU>T|MjU4~oYLnRjo+O9O>r^9pZFtC32$>#@HqX&m2I6n4rySY|p zQsDf<3rA+vbJ<-M;~7W~>xWAf&>(rXS9TMMr?%(eF_W<@HLAA_@zrpWj$au7LF1+a zv(naqq65#?T6_eP6D`JjM0VYTGVE6#n3wPVdBEG#T6+YCIhf;E%opwNE|!plr8kd$ z$$aQsw=Yj&$5J81`XQ8;{T?zV;0*|9;<6#>i1UmWFlOw@!@N0hf4vl5L6?Zlre9ZD zktQogF=M(%9hEfIQ5|l+qMt(vZsx9!k+?)wIfn9|cZetE0D@y}$jxZ(xR@w9UC$%C zi9mvg$DPj`3SSUe;bV$Y34yRy3QwH92C<=-&EKHxEb@t}{0T30IA%Ok50E!MwQP{x z*vCQu3jU>7WHo7g!4pA}R6~aoJ)9##6>o@5Vg1C@K`tFGPb=sE5gz0i_?V*@F}q%Q zIt0=_g#)?(muqcv-@#j2txWZ2%tjIyKr`@vi!EJpwJjxCrYs6%!Q_RBvDxPHG|@%P z_v__tt6;_I|KcQL-@j*^i1l*ov5$n1w7*f|r_nhnZ{MA{hylA;qvcn}iLQIm70px2 zUhbPB^`!!{z<-Or$I_-sg#?Zk+|Iz;M5mgf>j)Rex?i%go5xPF!vGE_yt*ll5g5lIMMb1AUHw+1F~-NGaSfi07rtfw_viF zzC+<4H;xjmMJNN-A*2WTYHGxmBsD-6UzNHfC&4~Am6EzT@lTIUh$9g%tigqvX9a#4 zagh=#(!h)%DQbwR#uJP?997CWzQtH!b()*qM6d3cP8BgB9q3NU7%p1RZLkS;<|=h%wIp7cp$X~nhVV@i3y`Y z-n%tIye9B^)2QX-jpeorS3f4|`B+f?Mf4Uc=uQywGPr{i?hfYl#Fu)Sz-LNu!|2;p zEj~IGj%*wEg)2E%iIRL3~O9ZewyL*$IAvddtt#jpeMn|7$_#SL|IVG zuq{wCB};@axFXhjnRziWxsBg4v(1SP1b1_L>f@VAcqazjFdZfjyXyX5Ns7}dgSz4x`0(uOSW8q) zcs>02aGdl^`^t0HB%7oAt}&`P)LMQ@gmaMy*bXHP8ke>@pSr5td9mErd3zETwrr4M z2TcKxCAP^2t2!2o08T7@D)>eg>;A%b=yMMb7)Y!^gG8+m+9f%npkHR5qei3+NBaSj zXm4`6yf|H$(cCfI9UAyue)ZxI6?(LB@MGg7;mx$Ra}9{0l#~y+-Js1J z_i$Q2xTb<9i9EzAC*hMRX16ZoJ53>A=vY)7;20?*cXH3;A;Wrre#MoqqEa7-Qul$P zNEM~_QIiX}X3dPgmK%o+iFPg?tHQ`!B*W-e7Vi&1mDhAqn}ax`LT?Wb9UDTU%JU1d zg+i4WOldi!pt(p z+AuU+ZD+o>Hv#^U*|-x;JV6=o#WdOM^7+no%s7?Ig{vh`b&Q1t)_S4ktIh?e5qnmE z`Ah7rdFXk#wSrCl(v%;4j^|FD%+{3TbZexf5;@rCozBd+bqyQ`PBzF-F_vD}pv0|H zYfpuU(oL8jk(zZ=#244g!p^jme&PkaphS{2+JV6k&8p2TGPA8RRx1QQqNdH4bh{s; zL8l|v3YJ4h;bSSxK)D{0c%+O-Ann*AHuQkEh*w>a=%1>f^3}c%rJT1p0LMv54qqLT zZ~|LP900Q{772zIjaE;!e&(~Yb%sPTQ;`o=0GZ8;1Qkh~v2GB9Oy2*|h ztD+jHztdx`=FxYOu~d-BA1pbup7J;CvbqNIrX&EWivfX+(aIsR$fntl`)wx1HJ&vd zXz0hZ+W0X}L&((Vxa~nNwDz`NJyKS(o!7?aw;=1M%~a)RHm@{HKscZtv;}uI>u8#^ zDg9!Dtf{e@42Az_^U{9l_+3oPfp3$+M(>Q!9${{Gr{`=_`_y(XrkW$j)uFguSQqI-J*O|4$Lf>n@SAw{<9q`}>XYxtu zPpyp9Pn57$1r2Z$I+gCsm&J=`AT^sh-LmtReCRc^QjOOQXAuwr30O}?+4G(S+&T~j7owLg>$X>bZD zLG>55&6zr*2w&_6S@wN_xkzWQV^3n9?G`KB*0PQ%8MGno25t5_(hDXg4evfRd7y=w zZs7g4hhIR3)mAbj#n-M7WqUfN<(Ctm0OMgy4ieo7JAHYeOlgrAF>kOBRGK#Y5o*JC z9N+_32M<-dhJn;I{Vy4j)-mt2Jze#>m*_p^z3hKJ4t-NXu zjXX&b18n{vD1(?9G&tg1QkrSH*b9pv_JBKU67{8x9HRcTaAZHm z>K4{aTlauPGhr_`&t;|H2vKi=u{2W}dV=G_FO z{-{P8%sQ>Qz0<9_cg0^T&*@Tu+S*1ID=8T`OBM5&Q_xdcXyV8BqI68Di$MHB#yyC5 z76RJqqaPWtWMom}E?#x@j9Z#!6mQQ1QNUpeSiPm7<&H6W3D?WP1^$BATV-8_kv4vp zNGExd0O~3v^QGlE<*yfqE9f{WvsR;)nFECWx`n9_BG8JB0Wks_H&Pe>6Udi6-e|#u zQBRP732odm%pZy7XKDE25uJ^=q4P3fDNWv{1~Zt0#gnRavA`Z-tDO_8!rlbg(r zugm2r5);?Y>!nIwq}T!JN?+DGT#cn8H+taXlyW>tKO>Ur3)yjmYGGnlPgn7|J8r27 zQ3ck>G4dCh)$7pKng;INU?vFI+S0h)J-z5zA2tuDMTVtfQij;%`%b3jVjI=b!L+kL z99tWBkYU%ove1bvhU96jky~6(hZvc6ux{>|NH-ztYCA0azwL7T9UHE5=q-h8js7+u z->MXm`=@fTQ1a9Rtnq0BU6N|WRcsc3fR=c1ttwR3+>8({83Gfa_^*Mp7dS)|>bQr> ztqxDyfi}iyGCLVv(Qabpsavm9c4U_x!0{n=R|ccvloPjVjmo0)R`m}hd>#lNBur`! zXK$^k>tBe~PwyHLA{8?xZuiNtpzCnmbbhcsup>fba=cMtR%oxywDzyYunJ4sF{lltf zzOMGI6v5(`syxfYS3>DCe4<qDHbmTOM)ebmKeb<%#Og zzo$rDshrlRZAQ=pAT8YJBYE>z3&pncp#W6$VdhyAG8%iFTa~iAXsiLuDR_Snz~mYd zPmkX8*Y#5#qB*2Df;e>>pBUFRRjhQqM7LvzIs6i98B+%(^yGw9)cZWdG-~kqQcL*BX zZCND92A80VySsakEbi_O!P4}xFYQccI@5>zi*sl0caBBF6sU=oF;8xFV$Y8oGuH4k z5nc0{4QWHz*?(%?2TWMrK`ja?TfRy&U}>s%(M$nm>Nt5{Uf9kWg<^j>H0@I9PP?lH z?+t*x-YiK$V1k046gFScj|G>WB}%8-5H&Z64sk`hM}2KT?fgC4Cbx`_StFCrn~K87 z1E3J3$vdCg!2Bf3aK6lT#L!Ncg?N`iovy#GX6*B3 zfu76fGAZLKR;)c7!lFh0S6~)IZF;S}T*;r=#ay%Cb+@uf&owG16*h(>%BJydj4Yo= zul3_ez|l^C9pihMN?;K!*%;F_lXnRBS<=^!KWf~RvMddN7m2Qw@Y7pj-JRo{aY687 zi%W~;Is>aod}G?yMh|H~A6uvK_*1Pu#z_u8Ed_z>zb%Y&vbao^xj9&Ob?ZZdSeBKW zTx~PWS1qfXX^x*&bN$#(-PCSkHk)w3DH|M!fBOmekTR<0xE#S35ImkOmnXoUW_1cN&6Xcl+ucOt<`d`koLzlbA z_*BKNk(mpV7Q}P@8=%{?Z29^+e1zXjROTN41J;6-i0AWphPb)7vq((pgokf=uG;zc z>Do7W=JG>F{5?h;74p$S^i8l^tIjNKy9w^pw3MDC>?&wX&hA28o%KK|)k@`Z#Zq4s z$n#*@t0#bV)(h~?SzxiO6PO*~A>B2I0gB9`HI8^?exPv8^j)>ZWvMs~fXuCF`S}6F z>M@zDwHe~$_LZ8@h(I|%o`BMf(=YylGZd*Iu97im>6r+|$(~u(xqSxaHPb`n0OqkAp5}(!q{7HES$q4n9fI`g9Vu|VbX;~^0yWDQ~K^vGC=Q0kW$NI_X z|DOzQfg^baZq8FISdVBeehOR^C&&}ETT-_KBd`bne?d*`ojc~`L65#1d8Bd zr6AUyhFVgHgTV|*OCN8O5HPTNB{H%$&QQ*5NxM|<9#Yzzi$^zuAHZ>znvYx?HEB}@ zj6%C|9dxwkONZ?`s{Db)6QzwqaGF=W%u~?wA~v$ zt$Xqeq&QMO1@imkLW{;8OmuT?!hz;iXhJvCaRkY?_yzL6p>N_b4)p)j?LT$1g zxBt}bKXv<0-TqUz|J3b2b^A}<{!_RA)a^fY`%m5eQ@8)r?LT$1MQn!DC{M6c4 zo04{eBnRQkPBlH>MfVC?YQ(Uu#Nhb}+O#KED*djb;-)hTgMkaCX|09aliwR}8}Cq9 z*{p>k8i9t^NkdQGr}pJBEaHGLTWo zxYejSd9AALF%8 zx__l+OsVyQFb=6pV>U)3tb=GHqJwCG7^>eR4W+W7ci@8F&SC^roVO z$r4+hYUQg^7Gl2itXE5L5f?1>A zsu7uy_n6G|+6TQv3r)*O3e3BWg4)#Cd5p>#evcUrFxD8eY=`)4-_?7p`()VkYKh4q zWbU$@55tsS3A)($hWHmu#|m4-_{11-mhWG|K1TZl8)FcAfSv;jzpiqT^CMYVD4;e4 zSAVOLef+cqg4^KL3H^~3=HB$BR;|4ts(y~6rv%}L=cP`M>v9blMDjaG!l;AVM5PbU zrw;upyuL>si!y(TN`WIR%hGP*s}kRPX3Ggz&iWE{1HgzFYMh%Iw;MBmBsE1zx0dTwf-rRqrh-4x*B^V`p$W3x&p4%0^_Il77zNhGLI`&<# zE?Ws#;1vuwQx?|#z;{ZTSO@z+xeO7-JqGsDRhce#(_A&o@@&|QFT=|5#^tvd6QadC z4_!7K>v#m%29^46)1GVfTzc%dv=yz1Z7w9r;lvN2q59|qHIoLO3pPHML zOZnU}zrKT8bCv#rCYLH&fg}zqe#<_+5~+Fi=|_|bsh8>Tmd-Mhnj2Oo2@?jp3z1(P z-$M;iuRD8ncP1I%*G2eR4Y`Wk97QIEUt(~3sc4P;;T_3ljY-UfuGl!VbZLl_!93SW z;a{;dq?2lvEs_AlDS|0<_z3aj$|!?9%SYMant4E2TGos7q?`2N6I}P_*?w(%hJ^EQ zT%M=1{P7>qKjeB0^@I4>4>EhZuc;=%Sufq!>EF~*PtmvO@@;j!n^M%}PSHDQt4B)F z{ZaNl+ABHZHQ-r|u+Y#EU_*Glj`qUXJN<2cAB{1QjXewSoZ~DQH(h6-l7^j(+p%du zy{u9YGdT*B>_z4QoqKE2b@VB4@uUtsN+BZL=|FcW!w)Iy$c}AEEZjGeJ{lbn&vClx zxYy50?_n=bGPE~2Q z8DIxZ_03Gve84pYwIQc`;VO^J`U;9DZpQdY%-49bEL_xd5%kvZ?IP7Yx_nc;1Z|d! zkXvo@bfbaGPRaMpxnt2AxTqkVXo>e$1N%SraZ$r{>~T;c`fh|anE}S|>1=U`7rSd= z)eR#v3%+H9~*<+RFXvltv7wPgh_PSXRWa z600Q~^|b2VV)%COy6=ZQ8LopH8{x(D`Tawl0R8tAb*lMjx=Nf6fj%4%f?Dc_=5U^T zoMphT!jpc?ys9&}P!_W3EmsO8F~5t=e@ zM0NcQ+C&>tf4~cnU6eApT~@n2yc+SSB2m4gLNjZONkyks%WBo9oY$=iaRWH;_N21cHWLsF-J>kw-~-&SGxTJ z0j+W(k*b{**G`EOTrV_t34AYlcZx~ZoLY>I-0|GSHl<1F6(clWqtZMRvnawCBSIGa zIXoP(c7d~JQ+g(7b-hAumiig-m#uN)hbd3lk%3Bm>|`JM1TW z*g2ALEx?I$YW+r|FAM{W2TPts$JS8p5nhIqoAk{$5`)8*y!z_qe2_{fJ^?ZJGx+_ z=n`2{VlI!TiV!HI*GCjD7IFZP67Cg!dlZ)dg>hFuVMyS!DGk*JPCN&oxV+&j7h{c{ zcx3doGm`vn8hEGsd}A~GQEDGdH5z+iiEW8)%YdHt;*_MS2{Zl@zBAT*z&WKTk*Qqp zPA4G48UeN`JX#oth(jgE>sB`uM~OGbCps@8f|&Bsu;~&XB40$nAJaq6O?9puY*kxao zx6o%14~xMFZXH^_U5O-!yudnqxNqf!SfZrEzYFZUcd5wUPv<64)&^WxP#su3@5(!x zuq0+RC_>Fd#|jcRDNjWKWypABF6bm#YN|T)gSdPpd3?~q zb8JF-eBpp)AJK!57w8P&Hr)xNSE(OJ^An{` zz|k?@J&(*9LqW?P=dBYar<{cJ7XhhW>aP=d<6eEz137pC>K0!F9+babySrWrQ``AW z!DkB%T&j!}2uA9$QRsJ27AoH?%yqL=TOJx;fnbs8IgJc&&JT9`g)`UVIJWcLZ2ef5 zm_DlHfK_gtt04>-Jy}j~Q$%E^h`0;~wHj_(d!{1CWgU#d0XViQ1X>?L*rQK~P!jB+ z`Y3bX1^qSqdL#;z0GWDi_$M!U;;oLCv7M}MXCc@5y5hn6_~q6lPVjdHD48~Vi1BDb z7NgN5(MTdBe5Hna(GEaGJ~}$d+sjf8c_pvj^owS1v0m55o=92M$lU6E%SCYt3Y9+= z@m7hd#d!z2rj+NWw*?#zNh13xDPhJfHceS=#%8Sv^KM+(p^5uL`7BzRH7PI`1#3G( zxvIU$yMqRBc3++)2@w!pJ#!0T*YWF70d-m=7wG5c#4agq@ zAya-nxcI*nI^$=@*8&3KG3dY^j(FP}oQTF#+u>FpBx4KB%OW)-qCayXtFI}_>v1iK z*8|XjUZsz2RC?btUFHI-V2Lg`hr4ZiOb{SwG!ghdU(gytbD!N3J3jZ&fVdCP_W|!n z)HLU%>wqPROM|h!fiMKF{D&A;g(0Sg>9lp=0YwhEC{o3nRsq3QVaLO^^*wv#vKDnn z5)(q#P9sqMsG#KVBV#1OA7)G9LH`Mq{>m=40HkF$7;ZQr!@M_>PHl()e|uoW{VEkW z6a_fr&>5I1r})~nte0i>L`Ner4Ayu<%9F`;Ihpzru{N0&-Vm=W0N|usk~Wuu%M%mMOJO zFCehyT(ocGT#Ip;2m7~`amkTiyhWkp6Q)G3BCJ*D(7F$?S(w#6)tqFW@2OT>$qhaJq$gc2Q=_P}DC12)6pYM1^6jo1u z3?oPkR#0hhj--zU{~5zG_k1)-_5y)Kkj3QGBs_r@+geqHK7EYfZdb&XpVyAiWC1$#lx3AAbG8a%Rd3k;K%De3-DVbw7FGuNlVH0oidraUx0jTS!9J_a{XhLt?GeP?osBg9zqG z&cU@6YZ(d&#YQg&XaatoX4i63Gb#YRS$GyXw(9GepF6oaWot?wji!Hk8Po5x-PER5 zm-Kv#o5Wb7dUeevbDkZ+kT)`cs*utrRzYKIk)+@uJXKX1(7{{xZtniGUpY=I9{0DW zON6&C#fY+LT?Ef$g2BTN1-NLVK{Mhq?#kmeUYMgZV!2PnXCgT+LTSDjEqqUcUiTMA zZDz;q&&_=H?On1ME6oA@H!i=rTT6oCG+n&^G$A_3<%4Znb6i#?7hqc@tv2dU5poKu zkHHyR29Zv!ZF{Xv2~w=A6Ygn}@w#O0eTEBC`~~8*?D#kQYBK(!?s^GQ?WD3|)AMWD zx8^GuE^`hQ<3k@u3}OZm99I@9cN`**CVXR*Fbncoaw?4hnA_VVuJHPMnFH;083^Vrr8b^CD#f#^;z9?(} zMhnSI>=sP!4aW01&M!WH$_CzPZxyN4XW7M|RT2Z)lv#1`ap+l{0nrsd^b!m0GkZg; z@(gW2VtNXSQ0DcU_QZ#q4MNGxF)@ zgT}VvcKmSOLnM6f$cqf9<|NXOYxHqfr~YeaCEn&?z0Weg4}}FrNV#4E0qqcHpAwnF zCyxjlCj0>iug{3_)bB9qYq1k%pBd1BqId+Qf0BoO1!i(`US#y6uIMN(=XlOzlBpg^ z)Qeg5$#H5eb2#OP@-R32sP{QXtbL7ipRPnS{vN{^K`*mM*Lr`qz&})@hq6Z7Ed7v} z5MeM+r>ItoK5Mt|Kt~0Ql4Xw;^YhR-hNR<}6ZWvouOqFhNVZv8R*-!4EZ+btp&k>r zfx>;x;7q`6s$=Oa3wN{61&-{82X{Qe?~H_I%Uoa1nR6v2yp2|vP15p$%}Ksi=6GG5 z>7#L#I5gGT=dI*SH5M~sZ|QuQ%Y904y72n4GCOp(uS5M%X>B9!MIj#G-oE5yx8qeW z6WY1f79w{OzV5yz&|GU_y+*L79X71-g=9#NxQV#i%w#SOmH1u1A36&u<`x>`SAI(P z!<`?ob&zo~fYBLi*DYS`nV^#JHaT3wCzzds512FT)Q9W5@bU{0(CajZ(QYMZoxNqK zEZRov-`Pfqjj`Z_Us-d`$1XOji!08Ox)Q&lrE4}q+0cT0(9NC64>1tu$&HLclE>FQ z5W<0v@NX-j-S}{V2to8i66`&dS0)OOOb!^OWwoAe=F5n0T;7YTn~8?8cwJ5~zcANf zIo%25)$GHhvIkfqx)ELwKEtuSDwmd0!Za&K|^M_WY&AKqCG_^`M{r1}sYcwVnj(KM^`5XztjFo<@_r z7`MoUOB-ZB@1HoKsArz(1RRcPv(fO@uRs8hCPdRx~;VCWu(*^vW( zln42=v{y8ho66*k3aPkrsfB)%cY>k25p^e#o4Gy6Dl<|wfI8rlxq#E8R4yHn;?j)D zA1urgr`Z80Og!0~m&(dLyAg+oIDC&Y9z%2x1AhSJyu?~frCRW7W#2&XhwoiTbk1=6 zpI3f|{}7C@LQ%w0#>qVH+AB;r3(%)IF+hBP#pjx^FAEXKqz|_A%h)5%ktaVEH3h!> z>}`;BUyO|GS_W+`Hz8aWYU1os!+6E*))-FaSIK+!!--G3OTyj@e!@g^#ZR%Z-~JVM zLOR5#zOL1I|G}JZWTO<;;cHt-ds7u$;b`tZJ z*D%*YV+8-NQje0^TWmNQzM}HG&}(|?yA`vBM~0DM;!i9tsZ>8!=%;Z#Nr1#C$-TsI zA~Gw?PfUzg$uG~je^hWIF`+l;cwp1;OYTUS9_(>TvOU=h_LWIkP0MkMt6bC-+)tP+HB|%LT`<}Fv{mQ2`mPnq1=ad;Ya#+WSNXJF1SN%bD$9i3Ij9) zv=+t(+^~s1uOGu#pF~BP@PN`Wfg5K-kNIkSuN@0!gg~D}8T6Z5P%Sf#Wa3j4T? zMiFLHLs4_xaqink2|N>ird8sj;s`B}z$6&ESjAT)a=SLVX<=!Gkr@esvKy3!&m8$T z6e>MAqT9k$gfRVvA91Nj#3Chmthu-DRKD@mZc+h5aQSX3w|S-CurRkHVBS*^!2Aa(#l=bh literal 0 HcmV?d00001 diff --git a/tests/data/vals_ai/saved_pages/medscribe.html.gz b/tests/data/vals_ai/saved_pages/medscribe.html.gz new file mode 100644 index 0000000000000000000000000000000000000000..eb105c8813ed4c1508f7d8ec5368e65a70401bd2 GIT binary patch literal 98297 zcmV(_K-9k95kK@(-wgl`H}DRrtUxOO zlfO)fn_PwylEjn4!8}cu$^LkJets^`qG%RCKeL4!x-&19C(&X&gXi%t$yfgL@Ehsr z(|`V?e*g06HxwEh^VR_H?8hgBocrN4I+qbaFWpNoCWp;ezx_tuzB_=zr>k(1`cX)* zkKu@XfOmux1?A-`nGfA~wpzd-lMzIDgb0Zsy-2GD^V{qW>938U0YWb!OpEZx{6(`dp*=rUaVcjX3tdbvluGdEcA zcatbd6XJ$bvh-r$JD|_RBkm-Ql7zKJ5|3drk#l&LaRg=4%fmr5+fV&8@LBuz(O@o%YRswzrP@wtanuvFgE+KYdr#9PI|1~j+fxSbC!mt~TjdnbvXdQ}!r&;2y@ z;{6G*M5%1DT0jq%#{tYo@0b(6wcZW!TD8%ShjdlAuRlp(0Pjic1t66sICvx>{Kj_2 z$=U47iv@58$mYSRd&W!5WAF5EFwUN(DKslX$x2)9av6A18m%UCiPuW}Up-J=nyOsD z|BF`Y1shPg0gN*Q5uAgDH2~#^m2sRT96-hthl7m9mdOelkrjH%|jSG`y-tIWb%1*da4dUtpOUKOb!bE9%tVe&7#u{VsEyBnQ-GBKlS4GUf^B6io&S!ApN*wk0qv@d7&4(X%v?j@=21$ z5joRkYRc5EV1MF<5t}?Yj9ZAF%+6?XL@@pGmjw9j@!JoBzcMH;(!qYwbTXNH3s;U} z(B6Z;GI6y(00j>QQVMTy9Pna2?}N2SfzM?rv46j$!`=YJ_`Wy}_Xc1-GGgt2C@}Fu zJaa?;SJwR+9l7M}>tk@WH}I1ulZlrk{z>4yh~gJuo(}fWa0Sf41b!7w{ZrqYLJO** zni5qcszIY?ZtS~I<(20qs~Eq(9Y8~?*qvO$=VyQg@?A*Aw(0ZgL3plTmNicRFSsnQ)}|OXqP2Ex_Wo@Zf*5ue}}o{Jsf=i3IPHAr*<8TsN1QhfvEQA&E#QK?eB)**Ddmw$vVe0cNr zUHKKdLO8erP=Ajkd&JFQ#s93Gk;*Xmf7oEKBRK(CmJSErYP)3?!ERq<;$Tq3YClQ>>B<%R>!EQ8TV#y+6L0H^Z! z7x&D~Kb6NaT*^%J{lGL27DokgxrkOtqab%c56P!W{R=pyeh;bym@5GT`>6*Sz#CS2TUJROmmf=nK;~K@bFE)NTcq69S>#WNLOum9 zbIDg^$cn#p-*HYS`()@M!n2_dYCN`*mtF*R926;jGAF!VERBxR*biqFeoUJTi4c3` z4i9PtVlQ3AAsN@+CAS56G7=Blk#(uw^gFFiBS@BN!0zYHigyyM4&* zLu!sF19**sAqCz6s4{eAaP;;W)fg8>%LLEdC!fdUJrLO^=P`L6lZm^8?mY6N4?lwm z75n33<6L86Kf(jkRvz|bpFJc0W{Q{;q zbp0Z5XSK)I?j_1vknGvZ=OL>7DdCoCsboS{OSBlCyK!*IZHl5w90dW+9BjP#ota%x zSZON?k5=)ScbU9ok{+Kq#NSP^k?bS>d;;V<^Io~>WDXR{Kfpu;8<0PdsSg6$z097# zg#EGnwf6jkNg;m<;QFVRV6COEy%?<15*2WTZV6q$w{P%g(pCzMDK->VuJdc&Mpwn& zk2wBg7#;W_ll0hUsL3G0FBy@u55*%Bn3u<37ZR{vT?WH3UmhbM3F9dYuCLXi82{Ou?C<;zo#%#y=>H{MrZ{&6U>RC+Qz*Jy5r@+~% z50mYOm|Vu>6{y@L3^hFF-5?vdC^754j z;$;GSHONefridhiy}GQczCh)wltocU9u?L{<^7jy^*1WLpAA=gV2y};dOmsoZ+iOt+xJJH-nqEW?SOq!*uSf~#^-=ey;FA;qzRM+ zU&fFWLsIqZ+mi7bvL-Sc7Re-lkK7Ulmsz>>9Ll(Uzt9>BH+lbz6LtE%m!3!Q`yajk zuDm3D5$>1tOw3=F)eSeJmoAuzhiv)!??1lG%gJc(7g!7;?la5Wl}DQ+onwhL&HPg` z91p+R|G$6xc|1C!KZD)>^B9;hOr`fA8sW>&3lXyhcCUP6hf%djcePzW;@J*6`bt+NCTp-;+Y2AG3DmT6@w4Zt6d zaUr2mvQ`cm4G`FP*?hN3=hZjR0plj{qpBYkm&t#F<7K_8W$d4U0Bp3Jy>Eh%M(@3_ z^y=L~4KW;JPZ4=#n+LJEN8ST(RjnVSJlglsDMvWkWrKv-K!bu=@Jor7c~>B1kIH0# z4y$C5lL$=bA#&}A96qi{F>Djet(Vcc7e8|oZ#V++FbP&uFX0tB%;j?D(F72E@ zrgSV;VX@=(_P_)}LG}z+s1G^HybyoyohJ!h}wdXN$sxYcu}&>+uZynZc-T zQ!}3c<{nqvOa){=A01{b<_@StsFr>F^2?^No3+JZ$*KVP-%@RlP#ykn!2eD7zXkud z;s4INwnjMba1g%C+j(3CWyF>m{cz<~*eCxF8_sG`*JoXBP_-i4rbCviLI8t5XsyG` zKh7%jl#!&CZ|xZ1kgy>GB%A|>90PAzTkl349Rccw4aixQ5%~*SPynQKbxN`>|L2gX z2OTA_2VUwiaYk7M>5w?>6|P#}B5}UV*sHDHRTo!4y}uBuu{S*3=Ef|=VTIryiN5UBL^75UHGx@G*1eEtDN>Qz};dDH(?D(+|EvjDmT zXRi6PznRoti|}Df0gw)vwNVxl>w?J=?b^Pk)+Jb5{IS{?Lta*+FatTZvp;-cu59is z=WBjNCg6*OWgcG6<%S7A937x>0PjKhlT`wQfbS@fJamb3>$%*<+KU-}K$j2w)Z-R= z4cV=<=5@6OA76s$Iyw+o8^!h21(-umK-r^e2eZtZ{@wqZhxi@jU-#YeXKCfjXLNAz zNv(;=BwB%+?@w?@jTZP_c`~%((%Z0{e<|x!<1tH)1A&S6ZL+z#UWlf58tPW)aJW8V&#)7MGxF$aA zM*oNphMm3nq;qZP3OEQ&J7QM=RJwOk_>fl~dLaUoL`viXqh2m2r!) zZr)_IKsE(zT`}7oZ7<}a>BRTIJ4)b&C6vhk?ds~(i(@Z-9R>b`O~f#g@|U%Ci=p5V zE!kZzy>R+$?g!Ih*4n6phpH@;G0O2V;`*pTy`#Ao1V@+QB-fYgMn!Al`gL3}whGYQ z>6EQ8;${=>1P=!;ZvO6{u$5~mId_)>+^E1ZjY-UD8+sS%*I9eG0*yswaANa!O|E}S8a@#Vfo%r&|3%m(1>2UCt9k|OGR0`cQf5xVolwgzZ2Ag~Z zw>nAMX}jf2D@4zFYpQRaSBw*6h8EO(@kJHV`V6b!==y@;ig+OXYJE&S^zX0HRmV&y zEhY`K-!%oy4v9S8^1RTaF|SrW=)m||!ef$B8oMFiP)#I{uVdksHw5D6G1QWghvU6^={YG%kQSc;viA8J|1{|iHGgbu7r^kd{F*!*yOZq z!3L(XH+_R3LVZ-0L0tt6QvBNNIn64S`uU0s0)FmqztS=oGlS$Z!Cms2{$5ueReMjW zz3=5Mm#1t&R6NXb$-2UI&*oAJyzECyeNJNqc5WEuGAN}*bmk2QNW!{~WP@1Ka8xE} zuJKyVeI2yeJ?(RJZB%{@m1|-$qXCYC0mpXfN-r~{qYPWKE$3D*X_yeai+)^y!nz4W zFk&_^(JuRbPg=BE2_^Qz88)i%a67&~EwgrcL?uErwf|)9hBIXM?tRfFG+jptb~QKF zeS3oTuLymrQlnSYR0T%U;5q-1i)^oK=XA?$PNuBdc7h8xP+f;q(D&}+3 zd9h8^(3?f657ulrlU!dqM*|o4w_W+^YIXci`?Zp{#@cwyZQMG|3@y*}D%*Ks~m75~XRh*jJYqS6ep0-@9z{otZ|5`S(Wa8nph|y5U*uPkHU7 z=e?H#_cB_ghd|00e!e50vA1}Y#SK2>2jBKj{R=Q6(`dPG0^JW{oHqN)LDqwEz^(`E z8_Hh}ewEmI>At4y*y%nDc33J4O?&S~AFyvjHn4Zz8C8_OT>X_z2QtLlJ^VGXOnZ4T z`f#J2^QKKzHvV$3?h1YlO-or`>>(_pgN1vMeK8PAS8dQyMb&Q{yqqpqSD(nofj0w3 z_~;yk!!dQ;P(QY_0Z?;!alo%;@ZanTR5FiYO6)6F@^QjyeeDIQn^mRyhxMhX{n%T2 zd{)3L^4u6;cc(rsAW+Sidb7RHUT81W7b;N*l?^})v)B2#EdGW-x1+t!Ce}osQ0Q^# zhF*~MrJ5g3q8QwZeF_DWDDbCvs%a=?rZoR3En^>VJ&b_+0C@jbTvg0I!5e53?`fr3 zo0_&}Vwm(U&X)rp+GeimKL6z%WJgr?5g{BaQ$OYhh4-QJ)gnBYxyyYWSnD90Ui%tM z+jbZZN+_!gV8*VMjQ~a@I=fJn!Kv$OINcL~Y6Vc`b4j8Hc%Ut1T@uy8fB8=bsaOI2 zmsNrXXQa$0+6U}U@QiDE?s?(CG6KIBfMlx#$$l7xUJeWpG=s%3bADM?h7!+ChI}|o zH8BfnM8-s>1`a8X&a(zg?L!477tFm-!c(%zK4&?$*@GCx*-)O)@-dqjFWr-ZEi5Gi zN7a=&&mQo*d;YH;Zwc8_$~atOrSbbYe_<%f0V70(D5tYEN8zM94We`DJONu?K(ash)r+G6p?iZMTrJp-l>fkSxN(xtKm7wjy-!QRgVqVx&2<#@ z18cBEBuDMW@x((Sv$}R2y?9jKV~3RTh8*#R=I~l#90ja{II546w0DrvkIk66%wgrX zgGFcfU={f;Q7){gwU@pBd+4XdTYC?Mg1;IG|FjP8Lm$5*c6<5zsuelq zHkDeNyb}-aO)xlO1*ZBnYe8a@+T_%#Jc9_8a;aFEt50$WP2)o@;S(=7-3L`) z9>~-6WTfWi$+Q+nN^TdRih~y23%_W{%JqS|HXoQKtcv0?HK?O$#su1#yTNJRj;%Gt zpi&irWd`V0qG$dY(51xH*h_&ThkmHZQbcPEDlbk@+a*DiDn()Ag9J)B-Y1`FmigkD zeIQAA&}Qkun!+wOr--^B3j|g^R@_v|x%_#)m4paytyf za&xFqd&C?0wq(btl!`U7(N5#NaHvOMt*JFz4UG#2Ewxr~zQF^31x(I|2{q?`m z->cOhjoHn8@`xP*$btY=@BN=kr>h_!h0=$&fxoN=0U){ANEWF?RT9kh@rRDV0jB;L zyXQIelJ{VQ<0c5-{V2i?a9M+uSl?{+NqqJZ=2Q`^r;xxqP2mr7tl;mj*c6#VH#wB` z&;&f*0bd%Qz-(|&r)m02@*beVs#*8&6Oa9}zURClHhh>7dBeVWY1c{otj zHAhHqRhVAyV0eh2#qW+ zftrOY!b#kqHzj_^)IJ+mC%+jA@qptcJb;2=p|ZoT@M0vNPx(Dh#}2(t(;lCh5aBwp5Arnzv)llLg4 z(T^WIUSE=YG+ySd0uN-rgb2wm(vCsFLKhduui)=1nBcE8nF5;KWL_H@FaKwvEj;?{ zx@`w0LFu~0>5Hi>`vSXy9Vv!UF|CDjSqinN|v#4i`0BpbhA4}%SXN{V`zNn zrf7=MKS0MEp9F6Be!$nZ!zfzf4$q&0;ALdpBEgx0XXktv!9nwri$xG7JPJd0@%voE zYxHCA5|lBs4Kiy$)l1d@{6_jcjV=xc3Yy`qPh1h7?C=cRXlZo7sf;x;WB?4gZ%r2YqBT0iA$#f#{~54@J8VxV+J z^NkC`*IEfvNopA%?}_cH0E&?<^(v>QCc4V?iUIbX?e2t{c z=05@cm5ppJ26&?5aPY$t_U)HEP)L?7BMqGMrLnuH#Lreq&#QxJ0{poAGmk# zoKqb7ZtnTqU7W1qnHRJ0oBE!`e9J71p`$%5cu~xP-xeK!p@+u1)yMyA@GOKqO}G= z@%fz16udLTg=T&Np^$4;6;quqEi*N*kGp_#N$DJ^@{yeuFV7$%_0S% zTWuHxv66{*Db0T*tCKYFwozZ1wk-5(@5Yyi@kG&hGV*p?_A$#wQ+_P42ET}H#8E-p zO(Zmr&`e?i!J9TLf{$!VYa^vZceehh1ibhARvRg}(=OE?iB$J)V`?MGZettjk3`Xj zou@zi&eO&@vS|bA&nD&W;m*=V8m>2({%q8`cUwmrX|?5c(I1Ie_hN@=vrkppDEhNe z=w56DZKTjH`#ygpD&4cKp3Nr4mb*KDD0AW-Z0M9mm*EtUyu2XG6G`jBU-tqP>Dw+M zmC!Y{2y+~w z0#)R7S1&N~Np}R+wh$PNKpf=(R;xEg(-LqZTdKyGK-tA3c&J6<=xYbVUet3!WOA$? z<2}n7sx&|3JyWISBZrXfsO^2ylOLl6we|2E6$WFXC%YtBA8}`F7Iqy=&KKn|1Wu-3 zI_k3k%Yv0EoKpuKSAGN{iwKbXK2od6Q#&wbo#Z)--v-sUq_-fYbBJ4rU77b~Nmv#D z$HEGNvm>1tx7{?@7)h}JvQH5Mqr{S_HJ3nF0O^6vFM zY3yisL_NG}#7~Q6GY(&&t!vs~_-ScuO^p{$1B?VZjm}Zyb;O87fy+ZezQ@3}7)f~jP;jiRVV5(#|+8n#JS+yk=R^U(G1D)51DmB~_(&|0O z3Dm6MsAB6BHdHe-ZWA=lUyu*Ub7`6JTTDFFaQxv8b+ld2R8^ zwIHQ}24^)gR2~g5Yb@8L{dn&O_BwWnSPExVpDbfXQZ?iL1 zze;K3;vB;BDTJr;kno@tVR^Y($&;D%wieyMouKbCdPfTBUfwQ$TU?6W%f7!`Z=b8B z?ejt5UDSe9wqdXmV-~3MIzz;UXtbr0MfBOK{J?9$Is12povfg~#CvGmV#=KNEPq)w zYYBc^uKQLFwhluFzU|VDGill~L-_z$P+}Oxuo3_4Q!f~`7@{nD(09lpfT3Xqex-65 zY+13(pS&Q~k({=uBI%&D#QJhHMs7of`sWavt7a-&qYpz}=1>h=5>%9+dgZ2b8N*H$ z@4h5MN~EI9i2M(LPWRv+N3V(NEVTsQWN$$nM;(40u`(@#`A;^O&p+hdU5)-zk@c9G zLMh9r(H7J%%c2&dZ=~LiY|%?MIvQ-^0%cH>+Hd@Y2h+Tjme;01!BADe0Q0KRj`^0z zdzo;(o+#z9-!hp>RS>G|d}guSlVUJ|Y)Ryzaf&Bjib>`!NG<8Da*Y2qmlwxv0khuW zvKZ?}UrSFMLTQbqB#JSc6foD}U=~CtZos9z7h}W*9-M;L@RI;k>mGl=5FhzZmX4CW zDI%z3YZ}Q_ExyHLAoJnl4=nlOA5LB z4_Bj+jEsNi9b{4*yTwQwY@%T3O2U6-Q-#eEFn1Yk57r<&IsD<|7jFXYfOiV6MV{8} z0Kd%+hyGr)=R(;C$VmLJp0F1mz{ps5F&-(~|5VxIDI8YC^Lgy2`EM4FX5YOU?fY+| zcZV)(ce-vDjPboJZBg{;;UKwOoJ7IktLV$YU>{2yd|<00haZ%EK+C5Y{yuluu4h3(X9QdchPqB!cSNiQY{_H2< z42K?OX&Ge#8X3hq9G)Dq0z4{B2JtAv3CHFB33#L6mL>V*$fot%B#GYvRM4q6`rvhH zodVhtWZvEfOzH_N4`P%9bj6llIW(0;z$ZbFcZ>pKFxm@YLhsGHissRz-j+pI_733% zInNSG#h7emK4LQ_IgE=zvSw4B|8{hce9Ca~#TSUgBY1_`Z&{hq7hep+w=tS@B_Fz_ z)4}Ms-v(K3zQH@_$p=!z$RT@i?%t0^R~5J>Hw_m{s@)05h+%(ukj9rEmK?Dd&wpJV zQ(A(7`qg6ie;xhsJ+e!}d#iPml^c!rKU`h03?#!iA4*H{NzU*RCUHmb9<9RnA$V$! z2YYAzHM|S9DKk1q=NPQz42GS2dH5gj#@Pq*`G+&Go<=BdlL(Z&7sr6%|GdpO@Ev)M zF9;3=^II~=S?;Z(3uq#_`p;hXF!y5kA4A>hAX7!<*~iJj(Ns+Bz2#oG_W_(+j2+oF zK=x%Yx*GA0IfIl6z_c4=--W`pJMJk9M#o%T)_d0QXm6uk+f}tBWUku4T8qmHuQN(wG~!?y zubJzZYyO%9p1U!}|&{NW@4^B-C-lu8K3Psk@CsJN7+f2rw* zP!2)%yaDYB9}Zb|@Mk}~diDJKH^<-q@b&XAzNpq_rGG+02hqbRh!K`x{?-cSvT={e zM^LhI#Np#i{DBP&5yD8^$0u#Ly1L@lWDj~WVzkWC%#r4E3oT+MzYe@FzTk4eU!V*E z#mGs(!m$Fnd5_ss)_T7{#ROq6xArE7J+cCIFtc>uPTm1hIbATQGBb4>==cvu`sW`e zAl;DCWf)sDGin7sth|%q+Va3bgl)j_4An3(8EW?4&i6iKx@UiR1-30~5De&rpMdcI zn)_{rw9R?~On|Sl!r^Ei!C;_Vy&LVV#FR!!5i+Z3wa&shkfDRB%s!mIUA^Pev;Ltl zCr(jqud?*UZEaK^-=4m^%F_dub;jYcVkcegdF*+VDbtXrXDjQovQ3sNwTv%8t$qGs zGdh6_LYxkHfoCj7LU|sPK>{|unhr1b2H9ks77ZtZy|(LeKh?lOLl+ArMQq6G%_Y7;SwvXm5fuk5SCg_G%e-V( zuV|;bmQuUzi_16&8EMTNK$?jI^2?Qt)B!Mz1T%9FYFC7_J7>B7lJHw*IT2YiEE4$P zU>37$v03#b^(XHy*9$}0vp1c2QuZfb|0xX$Tw4qWx?bwZLTxJemsNt_H6(TfudJ0~ zl`pCtuZP%u5bIoetJ@0RkIio3vz0Kc)12b!zw1orKlXS^4Guq?VFQl z-~V{?^PEpfBvm#!-_pIeZ(TPR>-BRx4vNjz%x;$(btU1+U)iZ3ZmxW^S+rSTC_W`O!-}H=fK1L;YsJSTDzM zR%;vhCJ~0X-U7g%Fss@#usJzwP z>jY&pcB^@MQYw5qcw>91?m`f%B`8TFd*DO;;g320*sdCby&-iz72rjxossr ze!}v|bApIt?K80^ZH>HS0!nZz=&IP0j@lX!z+koGTN1=?+>1=J3ZgP~FOFTT@H@h67uV86>$MRT_za7065M5C5c+M*xvsHj?z?^$-kj^J= zybXW7@}|$$#&i#C3varu)x9e~JmInsNJ+WqEeVKBI=u2QMX8@G1OVeT7PT6;!fdaj z_g>63q=0>fg}HXz2Bw9Z&bPA}qtk6>N5F{-g@ZFxSyUodHg_k|I z0W4-!ekb_;;hm5pzO*L@BN*1lwPn3`?+QzeMLbzoxp%64w{4rog7XerH>Fkb>ZS+F zm4*Bc(8?E_^QHbgd?hZ|@5kDY zbbn9_Wyw#(y2&zHu9kpXKS@ub3&9|jtjTY&(R>?(VH-xUh9H3^s-Go>{`ODoifZf4 z=j|c@(NgqJ{C&saEvqqr=25crQy0^*X&3>h?ZS2*n{`9zAF2}TOmB(%Lt%{cOwwGH zU~XdgucQ;@OlvRd%%#>wY(nW=E!v>)S3NV=&l3FlRxdwG>Y3z}D@f{GZEsqoKN%(P zpyao9`5C&RW+QA7GEJ3r04L!D4^%z=wvbMFxt8J?r%mSGSDu*iux#XPlJ{ zD!~9#?7CGi_belwikN&?&pokJ@w}0o_aHK zZ_T`gmVDsdyW?H{uh?OUMl_53umwLu@CpwqRD#v!#|XrB^2>(}F1;X#&IfI~`zN(I z0$|N~!11gwX7nX2tuIWvM51QLkpcLMsfR;6hQNN~@v};X$~;1?jT3ZTTqzz|4ILw~ zKriqT1MVw>zVPZS^7tT&)?Pj%^ArzlPU6+#1iwPbMs+-u(h|O3#;DO4x-|+T9uq}n zK~aj#)YVuhaeKf$78$ue6bv#xV!Ji4ZXQt?@0ahlsCWJWkGjM-JD{O;+}JGg_&D$% z3j)DqM9Xv;XUyITA_H+M*fGGFkltsi3@tzcS*l}(pUYfLYg~yiD$sC2(CH7c4 zuESGtlh2+hx}m-32+)(^yWT3lYXA5Xyr2~O9*p+#lcI3`cTOvTeG)JBPdN{37r{7- z^Mi+l@RhNCRd95Qvc1aFYr#+`80c_LyGYVLt=Jqb!Va!QQIk_f2*heef7;n>6nqe`Iym9cpWYZi_=C=9djAKOs(5GZFI)_ zOIlAHG14~{4(Kk{SSq$<<}oTnJ`kiGsr(cuO~2%sJ2It)%Qpv#J3ybotdRSIfnLEU<`p(nn-Bb6T>;af)M!>u~xfFom6fZ#n5VQw~ z7c4HnAyWp$5IRzwzl>Ohlg%9@c)&m1Fw7%Xs2C`AL6yIZS=SW7nrnCv!A+Skr^5O_!guY)UhF{{5S8;*u~kI)&L0xCJ17 zzX2gTm#On=9?g<}-If3=M)eb*tFaom0DVp{U-o+H@x$fC?nP2bw zvXP4pp}A&J-`XedCsJNn5a_C=G-NoEZW-r;(71K`48z#&xD3zX;iE{SP8-f zCB>$iQ0Y8>O`|XvupTPvWE=$U!qq`qf`aiU?;{7Fy6&|5?pURMgx7Y>=U zTpE(5oJg+L6wk9Lng!l?=Ap45shv8bqnw0yv$iUXv7VSsiW&?U=~|V$Ik2mhU3jOsX zQgQTtjQ>Ch#ZsK!0)TRGRB-P{iai%3U@M2o{dhaj%G5#aH$9dN|MJwgZzR%4uqVG6ftP)Q;kI*+| z&T)6C<78R5lilBs=`j^3qc+R%Tx#PYms;u8USA<=bD@@@Gw(BS{c-k73NT}oqzjbf zndeXjGzw$jStvPMrkO1QidSm!7Aobqa1|^Zx?v<*SmrjiM5)xG{VOHatfUHs`%CJX zSExyfv8iMZEGteo0K1_zQwmn0yQxeP1)GYbUL5zfW3nB_4MjEqvk?>%sMAv<(lRQ8 zL^V4L>1F0=ExC-MAn~W(IB;hGvb1QKRP{0!iQJr?DIVYiCwG3ScPi6dN-f+8{PL19 zJO*u2tE-@tEZi0mltwcz1p|AGaztYG=lCW53p(Lw60K5TuBR~i=hAnXE!t?}!gr$u z&f!HQowW(vdVy9`r&g#-uQ!Udwe>49La=Dn5&1Wxg<9)xYR@;V?^^}zV&eiNa~gVa zd2OipmdvB`r|xuCTGwLdFN*avo-Jej755y9Wi*iclPF1#mtK6FdP&;RNW8)rTaA^o zz)dmUWOXTQQCxgjmyAO2(dJR@&^?~6^v}wxc$H4GQFnYzqtp$Kt7~S;9#x$6f|}r> z+y^07yj*eu#(_I2Ee^81g1lRdmKK)3>N=GTTP?Ps+7V7=YUtE33+$ZDj{R`CN{`d% zy%&}Sf?gb)Lr?xF?rnX`9itL!0aWBSRUrg9E zc&gCO!E1^s+mvn`yrI~pqTh?*mshI#hYmEejn1K4rfdk1UDZt0x)Z{8n!cMxt(a=( zs7;UrhP`prie_1sb1&#!TT|{GyzR(_-aU9t)nv^vH;-P&;orsRTO7bm1J>4%Hf5Wk z$%?U=$rVdCl)Gi}rYE^^;9B?Orqqxvx|z?ZLv4B&v% zs@gq~XH;y^wrzLIGp)=AJ@QO9Rr`(!EjZe4ZB11Q9I#niD~6?N_sUQc8@i#YJu}*o zb!#)F4I8BVooKFZ-)+;lHKj*{TfpJE0Ea7%eJ2dQVclrc*cGSSLbE`i2`Dee1oQMR zhTdEw-8fpkTa;O5H>HZYHt_C<(v_92{y_r~6}#6$u~b`NT2sfl6HVQ-HL_{o zy4pRj@$*_vbyHin=pDHq9c$s62CW)BLL3Zm$Fw$2X{uSeeb)xKkk$hK+3sFkuoP&9 z3dk@`QY2AZ2bXgPeI;a7r++DM`;CKzK zkG*9_*F{znsB_mEtGsKnY0SFOy;@35M>eT|4jC8GEdf!kSmxcVDQr0Yw`uT}(Y>BxtFk6SZo|1F!M6EC+NR-m z4<1ZZTNPL}1Y6d*lU2k1-P272*Ol&}tGX^b#%97((=g0?#q^!Wb~m9zQ@a;nEEROO zx)B>XxM-?=FV-Nom;j}1j^@C1S+}Xk;v%*6J2zuZQB4nn*1DCP1A|rsgq~%Z4!uWv zB|V4tH>0C_r4s)nlB_dxQkkC$y4wdVBfG*DdE z5a4@NwbyRKoejHr2yx@!?Vcrf)s$7m5OL#mRlQFJU-=Z=CJdAenI_Y#j_f!Bz7BO1 zYTs9}x0UYGd(X+3mh7kk3mrDjhx;J^EOGg#fdcGtd+~sfz_zksTX(eO*mYlX)2MB| zd+W(|WJ^E-D7tBB>OC;@4bja;;EW!FSCK6dBi~fCJKuj2)pJ^DNbPHRiYc_j;`zmHTb0UGerzLA&q+ImI~%wgY)14PBO+)cx_dXBUknoLcRVH=8V-MPiydP8^9@Qt3~s}>$? zw?qyU>DC>Qz4wCjrorodCar47njvD+TDorBmu=`qe3nfpDBVVDI}>Y&V1VkXZrvLU zR0GLu8b7sqSimr3+Yqp3bxKw1o`^k>d2cyj*YwSMSrs`1ZqU22;YDI|Y#P7SbNt_y zbz2ot^UB@aQ0)!evT5L^*>giv6PxfLUG7HScgED)G-|D9*NrN8R$j!8ca*!nsMis= zrhoXhZ1q^Rnj@=@5W8!4W8~}>QEC%Pbf;SYYE;&2LqzeF{J*>6h;?wJO+%(Vr{ygj zH`E2@faVy+9p8UZg^m;BfO~dbbVtW?;Q}kPhI2Q|@mob;>qmtpJ3Z@`sw&$;8?bt5 zR@~Y0O*6F5rm-76Y@p(z9^K6N!1t=H-V^D+wY_7>y4hprqB$sH@PXD>?_BM7hVk8m z0;=|0{{x9HVgf*)>OGhNts$#6jlbviqpHY;YKd^bJuv&q@s|6?Y+@jl-rKH*&~YoL z6w~DnsNND~Yty(*qkFpRCYZw_3mdqub07R0@qG%6v(n?x8!!n3u60w>)b3`7rZ=AA zrjhG?Cu?+&e})LJ>-N1kQ`r&^b<_AM?WVlx$W-74tm0Uu8wmFz@{-utm7e*I$BHdw zvwf#)nrYvwed+FS$Nl&)=ZjK`{-o@FhV(uTBUA&4cfPQqnd)JbMC{1k!@W& zJd)Ob>Q?0Np-HK!-cK_}1PS27^_)G`a0_$u!iH_p`>>axZLQ>#@(sb33kI?y;?hy4 zbnM~o7BIJ1&r-_aUz!IMz`_^0QKIXraUUjrOVXX*@tcn9(4K=TnwTq3b?u%67~Yh1 zY!f!LJ{jQ1hR7WZL(}!Uyi?Vc4`S2UOAB~y=aL^A44=^wvU4%FDPw4n=ynx=+j>h!*^Xl^9}Oxf;>!A)g@3dLnM?nV)I zCVJ@~wE=3NFMt4*ZE6aIqgK@WvAfZgBCa0~T7T3qQ??wT`w7%B?s|o@BlX#)@#{U2 z1i%b5L?Z5MRJ)r?#oZ~4`iG4fReI7a*yZhxKKGB_&1p4(w4=S_Hw@Xddjd!(n0P=W z3Mf$iP7u(OOmEY`m7d)kjGLhfS^w6Z412p|to@h(fzjg#sYW^4b)V$Z?*tROrUvWB ziPF2*2TD;Bx~+CM%V66j-0ejK@J7#V2C5k4a8rFd0P62-Eb{;6JmRN8BU#uhZ+bL| z{S&X6Eo|x~@Ro^`dX2B6NgBn(or0-BmM)AGu}qTHav8<7q+=5RENx^mb7L=!Y6YXk z(w(H=dhWEGgp8$6yBd%ri4O+@G6{SyOb-V^( ze^oDnefXu56%Dh40FehgpFWF%D9+zXd7H9Pyh8d-+~whb;jr@hm&gx`_xWH} z6EH^!|06A;Gfz50v`^ds=3DH$(%hd;akAkYgwZbDbWWy+gI81~>Y1v*t1-S?e_7as z+C<0yGVW=9Ht`F7Uw=g$Jo|M=wDzjC^kC>nbp-TgC<_#uu+LYhJ&C#y7# zLgGX7`?2vOUG5J^dbtF&@h|yM++-OoSAd+rPtudp<7ffM|u-Xco+fGnsp5AglgZl*tB_ zNUNnZjn2ajq_GG!2>Y*Ov=G1|4t~M!9RwKjDVU4r^1r2ty;(D#vt(^+-PWhPxqwen zFm6hUTsq-@E~*;6)b*Kv=1;p+{YEX{sl+uj6Y{1Ss=K1KDHa8@zH@j?fn~U0>U}N# zS|r{w2CEYV8zXC*dP^1{D$(#f96SkA@(mzjxk=$Qza{5xI++t>#Y=3egwqLf^ZsYl zw5%5|29K+=uIO^jCO0=2zVUJ!gZcW2Wf^A6V0f1|KA5jPZ+YZ-?{8x)`q$>u7tiZs z$$Px$j6_jlvN_%2`2xs|y>jTgq#`bdf{Z+tMXywtrFirA*} zoYu#Iz1{o}_$JeBi~>mUR8bnE;2qxhKz>+y;gj!fg&OQX^>Oh2ZhRO=OLyY^^L7UE z#CfVdeO4bx-s6oA<)7@h)@=;r+1F34Ck=_k`@8XB{CYdYppK>)4PyLSrbCCx7sV_C zuOEZ|RQ1%Ep~OF3`6gn@LvNCAr>BtMh92W@ByHWgF+Z`&~-oY!pR&g@AuLy_Q8E& z2utVQ0%$yo!J^JyE%8=J;9f>6@UOtnjNx|^g^Lu;bY|(N{7|(CgT5pyryvdx?GS&q^5%K&`AXl%%|1wz#+LkLp>Kr|{KY750~1yS1> z#Qg4J35iGSRRz z;jZ9uC0n@)vk9Kb5k3S8FxJOPno>m;+r<=+Kz1K zWjGa?BAAM7R0*tVtw36$K;p5IEO&lsdBBE_11{V&GR@jw@N}fuR1Yi2MYPEwaBuivz3RF*`(joqsHie-~;2ToWiU>JzwEaCg$@!#MF z`CA*eDMlF{jR8~Pe3t9I9T;;5Ol9fV8Hyy6pgZ6;z=HA9O)Lg-wgW72q0g50j;s07 z0W$VXbn*nBTY%eWw_%wM;Ka2656*eKPHK+4pZ{)wN;RsCZ!)+Va-#z-XBSOr6q>Rr zCNL{kzow*gp&RYyt+LY`H1dhp3XRpU#Izbi(RPWTUeg=UC9`TM>Oh;}_feRi+#a=n zYv~&dM@-<|;T)*D(9$euX|{l4X#KQg>9yOT(RPs%JOy?pCBWu8ih>rXbYp|%4^m!f zhf2RTO^bu~EyR7(Ls_bApsd>EQmMPT1Z6X*0TJHtFG2ME_(|2jqqH3u^ST(c>xi>k zV6%)3*kbvf+aVNI1bMiS%)FRVu5MIjhNbhl~-{3-yG0 z$5}$X!Q=1vj~3YUsy5{YrBXqozEC=3v2$HIx8ygjQ{L#CGMMOilNXKIj@y?Zl%37Z zjQneZvy1eqU2{?O%*C-QFk)ijb{OeS>dst?NO))!k(efTgRR0+2e*TzUem>I&F))s z`Zv%<+JUon0&`9Hh^1m}UwBq_okmt;8d)9F zNUN>AwNE4D)D9TwuHsX*PD%A_pAPAR8A3O&hIcwW`c6w*rGQ$Q)G8p<^h+=JI%woq$pYn`aI+D#&3SNDe{Vq6<+Z}4#=blhI6 zS{gy7R@*8Yz@}HvcD1{( zn1QVWIDHqY(++istN$AmCd-z#nc^&KT06wXb`^D1 zZ-G$RbeKn9bolRqui@9YSOjiSct9fKHC_@OVg@l|BWT25!g(^ zpswxk89ObQ)pEz=3udj>1}2TE%^ag;db8{?9YC5lA+kC}RIhBplrJK($!@pkY-=~; zvzncyFg(HUCpoLOfNIrlLo^TCm|b@vZ2S7>1oA-Zjl%&Oaa=uf*W#SAyejR0?{Fc9 z<~xqG1s2t6oT1XH3ui3vN&DrXT@TP1rH%F$nCiZA`=L3OoTMEtx}z!8Zb&gRYDg*S z_7-RreS`Ug8=vjE+)-~%-nWSPUN)q)^RsP|o+bZihg`d+rC$|NT17}TU{z{YvD()g zSh9|GX!YGZxWX&RT(~Zu1PpxvlMZ((+C{#ztKU>z-YKzcwnt0VHm1eo7wSfzQ|q!o z9rNadXZI1S9xH{VtJrim33FDogTL)DyJlJmaGdql{W9h$RkadrIl_TSB|1cxy{&xb zmr_B#x11WlY!6!N)t~Ct$fDyyp5yGkf45G{u-?LZ#RGY2O{tH088m?-X-em^Etu7} z;p+9Rkym`aZf>n#cWRdom|>yg;-(_3&+<6%$LlNz7~TMXlUGgs&(a@sU+`e*54vDd zcVDh(V!N_xQ9^FV*)3CR?CfWbK;r~s$H5Y+>~dD3nOmT62UVqS?}z0V*l^_y=Bm;G ztG+A!F%Lk&0RKE{o;fpXp(bh^z%l@I041!mc$EI*_9*=rqJzCBPrj@J$HMdX-9PHM z+D{c>4OIltKQ0B($A~HHfwb7CEOdVB3Vqdfa=MHWst7BfB9!@Yf1GV{ExxrlHB129wy{zO(7FX-R@{8y-c4|Sr&G77ygl~!@E?)_=r`%lpM%LpR@$xoM3 z@Q>NwROq-|&%3~2+lPaxcj~T!bO58mXck^vZIQ=-m!JLbD{po5Ok<6JyS7}1f(aUKD!m`?4HyD(HY3~n8@>q2r@X;cKn z$91c7>8+_N%Zqmy*ViP=6G_#{5+s~Sf*D5mRoDc=*yn*)1cocRn1Y1#gM_dQ0~pae zA?2{dyoYLpVhqo#m&1+a!MQ^Cu!NpMFHT5T+JY=_@QI|eP-f~N!(Gv2BR8DPq_^?xWT-30mUV0>$KzwrDWi8}3RKxI z#Uu1X(5Q*WH9%QgV8HHVe>|Qq5_#^%XKp-|z3FN^ASs?ygeh@+0&Mzzz{PYJp*sn1 z0Ft~@F9rcu@Kr|Pc^1xxJn|PalEf2`emN#tH%l-$i5o!E@YG(LL?^$n7~?!S1vbXD zQEL)|J4GA|b?&Eg;L_Bad9v%@7Vh#M*7(cb?101dpIr8Q@@O89@!u>oUx9+bQ;T0VXbSh+;08DcU7a^@#~3Zwk*3XG0~Oryza0cdCQMJCT!G2aanj7fvg4aqdq%DCQA&fzZQ8lKKlTLkJsb0<@l@+v9|!)C4dPGM2PsAxESdGIy_m$AsU&TZK>~09clQ-9C?|K(2@xYFG+s2HH4~0Q_J|z_eft1pZTZ*LsO2Na(o-1gNk$ zNiGuyV#v-toOm$F8CYQ^%P?~3X$541#hxq#U#(@P!VXr{a~>IIe+9o z3Q<-IW6363!ti$Ld%<)M`U!x~VQ@f=dPvDV(g@&Kyy7i#fm4lESnx?6_7kQ+$QjjN z;JAT|2@j)7c>b(}fIcBX6wbW(|NDRb-z@wVM<>V{e-Q`uPOUQALLM9Jx)vZHOvKuBe3=+r3k?!iQc zLN4mOQlWLGyyfw-J&PLNH|R*VMpau;3V*02rD(aG;`6b0dK#4zpp{FVVNhI9`j_pW zk<_cyYcx@STJz%ATrcNNh))Z5=*V&a@2);R`uA$hbqCZgl_9)IJx(Kx~WUiRq?lKUiWnJ(C5A+wDD571zM} zyl2PROkKlLHS=K--_RuPS{@T4wxOy$IvKNK^i5@b^ISG;FxQ<2_xYx5Y<_EHT*)#b zMruvzF|{pB7}G~;O~8Rx9NWIld3}G}DUsRCb6ajjTz4+x=9@ml_?H8LjT>KmFlXy>`&X?S~3<$rK~ErFD}VbJqU|PiVD-~9 z40E?BPJd@J94vq5ccb;28rg0{thc037GncV?LK(I5+wcubyfS5+{h9k`Hpw)cu?7KDTllWalsnX zySJ)=TjXB*T!gxy%c=Q5-JH6dPB(Icwwy;?jMxS>dTz^uN!52QK@~X*ted*?KXbj`b0hAnVsLsjkddi#S_c3rEiLTNQS3am<$3ewx|Iqa>Pva0mVYugYxDQf7d z`VrCl-XB=#T11nQg)4T&QZFx(G^M$U;eY!!LqyJ(ISREOR?=>FC9TC=+sXsedqbd@ z7MNyzE2;uMxMHKYf3Vwb)NZrH?KZ7^#;D~~gKS^-F12C+ADAD@X|(%px7EFE)jAO) zv#odE>9HNz>Qm#6B@zb1v^4c&Ii&Ra>hN7$@7kKUA$mOzW`EDBsUlmnXF8h#0$VW^ z?crSbjjV=?`(0XzZtZ8!l)lw6-4VHSX_yaZ&u+)>D&5m|tyDwqtFsW6oPgo1Ar5Kv3Yq#cQnfga0s^4%L?HKYx z6{Bb0NpPy!vSz7$*0jLYXEJQ zw*2IEYP@ZhMAUNlgO;T+*--novJ{bXg}VJGIbL*IPIAWtw*(2?bI-%H${Ss~xysbC zZ2jS&@NG|PF=E@M(>I$fyN}q4BQ%@MhmGl*+p0`~2=4W1t6@62UY}V&Qk$w_+Yeh; zx8gJl9~4wE%wPA~CY5P#6hV=AMYcvY`(bV5<^r6o@7PMGZjwqZ!|f(Gl})WL{DrLd z`OQ=l3J+?04BFTkOq_Yt^s&#J|Z)@^$>_`XNv8Thc&Z)9lG)sP*e< z+Y}%$SPYi^aLDsE4tZLFS1N(bTF)IUs$kS_^O|{c%u=a`)7m?n>nuxay8Ys(Wy@C2 z;-}`ws?$4{g(4nk4@)RFDXiBg4YI>wlH%}1l`T#l0OfV3+Z5BNtl5409FCB;RoWAK z*sI)La+hbfVkwRoU1?7=Z%fC!%{}gvE#OX>nqxd{XIyJWmr`t~LUCb{u03}ybVtWP z&D|5$as)Oe4a3>}(DonFf!Z;-hM>R}p}QqJJqH6-Rkr)?PCBZPoG!`w2RVJsGbK8b z)|3=YQUnQX^h|6OPms}0iLJ?+K(25Q*sA?-Df*U<9MB!tNi116dsaSit<31r)Vj#F zv}LOgiuzjT4$986bQVa|S8F+x+dCj}aq+P#jnzywzSpoe4p*Kwq4B0O}% zsF35-5~FfYo;y{M4YfDEn8>mxeb}VF!T55OWigXYY!Nz}veUCoi5V-6-rz)3U|mdg zROjK${te9MNXuk3sA*G-x~9>slyy_qoa^ph=y=}bVItpBemiZ81N%U2qi6Hk0FmCC zXIQ|wqNb^R47W2rj;)N!vdMPqou;h!g(lI#NHdIU2KQBU+j&?%x#{H-O{VNbiTG7$ zAU37l^V)P|+MBtO3fzTutWx~s~-nHWORdG}G1} zuGro7&Dp?Dyzs7^oO^OBtG5B9|Dl)f>4)B+iAQZPM0%;wyJM z9RV#TwiKGg80^>SIZp&?*06fWOhs--8>;gVPVBlD?OLN9P#*KsR&oS?LAtxOd$xK;WPsrjakb>pG#Y`>BU z?nmFtE#=H%=9ocsF*%sJ7jq!oV{9@jiP)#R(kJr_@v*_NQ()1mt>AM1_5B%!oAajDJeyaz<+Je41M&B-mA(GHaF_niT zDL1#mov)SXLXn3|TQ+;{;7|<2ZuNwjqC)p59c9PsR`g-1q%~~d!khZ51sdTZG`3_* zwR=t&;NADWgCtbI!!j&WryomRx{o1n;5KIgw^H4vZ1)5XrBpVkea(y00FwEzy>Sg| zZ5c&D0&R^0ceV;$X(6=mZ5m*d_vH--!<+WGUW!o6ZdG|$O37*aIwRCIkMTFBwg|cL z3{h`ZS*tgFwAf6hj`84w-FVmhIU`eF?E5wqqzIKdX{Xv4~6n~M^Zd>mg7QURuS z>Pn38(!CkdMmUrF+3a`%eU&w)t=clRHPg~`)6s1EdgoM|9T*yJsZm8$ zbj7Bord=oET1)dbSe>S(nueyBR0TI<=dkz9#vKjhqU9J)h9GO36wy5^o6VkVOL1fb zU$-6B;Z5DFZgyqa(3NdjMoyF+@}#z)3fhu0*b!NJ531&_|9-W`m9izS0JoYYXY7Q8 zr`Ji~+%^;k#GAez%+2cFW{0|F*>c9dtXq*e$dRWwsv|TeO9xIaNDhL1mtEEt@qt0( z*V4nXca3A-(zI?^vhn}3_omH_<5=40_xme&>)sDpF++;j!H(&PHR&q-$sy}Z#VYMmEh0rwz)4}-ldxoefTmhJ6{9w(`pd*fg+SJNSJZ}4Uv{z(60CJE zEcVNPE_pzt9#GJ~TB+J3X;H;6t%nE*v}&4 zv?W**g}jKwTr*FpJ&xUx+~ScO$<7r`$1@?pfQzPw)gc?nKnNh|D413vn)T>>aw(Ehn~V0ZW41dQZh zDR=5mYR|x2@oqJa(*zlMO^ zRk12Y365nNG6^;8seowu+K85pWQ0nBfv$=Qr)%>rl3MrWaCQ|ucdSOXHPsTk1L#Z{zDx;_GdDs!(GsL$nNku$~`M(b^k1T{NAJaI%$q zfwyq2kNQYYf4~vgmU4aclnV~i+=^AW16oZooyJ-pFI4%+>;1u*R1f>Y$6%&8RVC;Xk+5PnRefK7HjYOlJ?lo`YB}Gg^~7A$i2~48@sW3>9BW#* zGO9SyH?=ic-UPKw1aO7&qTN8ZL{bGTkxYqNmLBND z;4{hyRm29g^7YFc)u1R8tUlJP$@qHbI#cwp>-7n1Hj*^>0z9fRg$hS0_hY5RYlcxb zZ^-I}Y3XRtwA(;mD?~Vrv-?WybOog4EV^uMK(n`G+WKI&<^2r<=raV=Y< zm|Ha;&PT~k>0@$pIhYQw6Yp~}y-laE0S8-o*wf@reYx*nO7ZgBtPfvxGRtn#gX@xb zTxJ4|;_GCzQ!hV`e@`dg)#?V`<>n2F>1grf`_B0Py4<{OHhR3s#&gJzhus9b>cRD z9L@V+IK?0X$#nO%_F*4TzxujQ%4BU$G=5HmrX$oDI^)N!=_f zuK8zovJ0n#WOc$z!*y|CrSRrIfo6gHWj0EqEZ@eXS)wP*%a@ovK0JVw%=$W+#H)*A zm~4MHyt>`7!wx5qMM+E>VqKC~x%^@Aai)0=jE{X(Vs@8`c$SXoVebth?|#%0^b1fc zn#FuMv;OuJ`#V$ITkdg{Ko)+rCHMb(wSqV0Tj|?$!{4-p@;4-qJ|=C?N_MAeWGIY@Aj@CqTJ zQdbbBs)9JMrXZZ6;84M#LN|)S%CEU#FZ0Da|0yhqYL#NblJZO+6Y>Ejs={)H{n8ok z4GL=xv|UtK|IWd};bB3dbaE5#fI=DhUKu9jv+GyQ z?ijI!4uAmc6y~s?1Pj7CEGX&`T3CP`7HH}TYf#86bu6y^;anRKa(4pORSc$jPhjoG`n!pj9FfV2QnDC@Eh9-39uP?yX1}ktQEgx#XJ9xSd8Ih= zpWl*6_J`sV5%`@IaH#Pf+`wB7&W#Z74ct%fy%w{BrMCEs!W{b5({B`CR&2bJ^ds z!OXLwj#jCx2KwcC+>f1w&~|6Hr>s;A;<$5THhCW#}Sz2aViqXX`F$6vGf1 z=Ib)=qZ;YGfuTQknzOqz$GtVxP_UCQTwUI78O(%KGQr!-+(pRw&l_-In*O8r?S43e z*IZDTJ!z^M|CNvOK5dL@NzYL8>X*;UQENGBt<$I00_}L&M@x0+@{u%B;1zX$hl+dY z)Q;i^Dm`5{PHLyZI~Crk@Z~8pDqY22W}YCQhV0#588c`nP^E6!b?S`nTkHu?q}mln zT#?RkGiS24PcCkCXF60F^sbQU^>HD8&J^9bj-c;QS}A++#su)((I(Qi$aX*3^;6`0f!Hj0f$`% zoXdbS&VW2FPZw&-!K5DJA@^Wc4-4OjhWy+5nrAaL_!HrI1G zEhnMja#~*6ZTsl0|9!4YgFG6xyUTQa#aDY6|NZtfy9y6G z-@Q)ehjZR{@4PR`2}20}w#W&~#&w<(R+yd333EANE+_0nbHaXy3H4_WNiw+aJx+#C zF!$_$o!mXfaK>(Z_BxPxt8V38X1DVGw!9Un2KlYLi`}_ZY6gR8I!kZo8iqw-{{8y- zJD;HQ37)7=u#OC#>|KRMj!}iyJbhPy<~rTDs*#Su_&5j`sWZv*gHa$KMlRBOep-&~ zB#>V+Lzmp+l6zbU>%czXS)FCXGT@|$MR{%e0{DqXqm%v=L6 z)pyrez3JR`TW-5@>mm#y_CMW8=2PA*$y7iW-JLp=)18SqaUC76L8}wiV+rfR{hp$Q za~n=Dp(m*+>cZ1rGLe1rU@J9eZX}u_a6a~f#>~;i6|Fki=hMft!il4K(Qk*-QOpzfN{EE5!4c6wh2NV4^evec=cX+j1PrRY^2_-v=eHk% zHjgZw^jC`;9n4!_HS_&R;Hq^B;avBp?z-oU5#EXK{(Nwy8=nJNk<~LLZ$uL*QqmcR ze$|xr)+8z+fCh#&K+Jg|oEPGic_FHF4bv7^YvrQr7FQ*6>{5*|ts~qgUD)Xn2TB2u zE(O3878HL*roH2p7G?kqishqwW20n_j_JcOeHZ`(clJdsE+|Lf9XzEsNuOe$__H~D zqVdMuntYXNxc5so+h&uoej4@qh3wv~6B)i3bLILcQm!vwS#@9Mv>!V{>F>UbA*@^- z6H~UP@4&@@OY7j`;*`yt_c_4^3N(ApEJ^SAL3%@MP0sa(aFqAQ>J71=q%{_k1N4U2 zoSN$m;RNNNf-*?&lWEd}R~*Na8$b3#l4*IGdTBn)d#fcR@3Ut$OGrMae#M-QKvQ$V z5+WU?)0TvgLv>GKp=R#*X7?1;A@og16ayPOR0oySR9{f(_Sx2Qy6Wizld`R7TNK^U zY@SYfli|2GfsZqcd!rQr$|M=5^JzNq;XRY7Por!93HcN!k}|HDnz80``DJl8?^4~( z!VUb(m!CiU{MLZgW%+h}c)Nc7c73zA+ZP=wk-=q`pK`4U!GcT2@^W&JW9THV#-6Lw z1)g?2$qs=W0=cY_T_W1d$pqhZG8;^X*U36H!=EOzhjcR2Y%aVYs8wz;?`m~BDi zrgg}ja@4>twn#=|su8YbWNfR4}k9S#5=C3AHU;vmF9$l)obGIu!j$MI7#9S*+w z!)2p|>v*iSD1IhFwR9Q^k;G3JJ4jQLgmcAX>Q7SBCnR zumR>i67PJ(e_AzYF++^(9tUY8Q$OGRNh4}n$uQ^WI6iPVG?Z0jg?LuxFX1n*vO==EqL%Js? z)UTfYwLzwj-L1781xD$&^8ta9$vO;h81UjSV6W9vq9G(;n0l+qk#>lLUHkWr6^bo1DX=;lLrQ0C)3uY^TX&$9E`fj> z#vL8NIDl~gQwuPYc#>KJ%r7vwOMoez+)Lr3b`awr=0!tHO4hQg$;XqDF5skOBF+SI zJu%vb57!pM5FhB41)&QPw7Wh!6;nE)J~~)9&~TvPdg-u5pPE(bGu7?@UP#lm607k>>_}Zv2q;zJk66EEg1eOgZkh$0O zRZ;)9&ttjEL_1DyhL2;#@2Xjgx}>7L$ObOBXxU-7Xm=*tT~HW1BHH#@*BldV`>lb} zhsT*u^S(MvH_*E12J}}L>(K)@7%QFdzK+n!nsQ-eDxk{936ib@P-m`Cf{i?(U@Q>l zm*@~g?0SnW(d6n_P|~gg!eNTT6z71jhLlCWKTFMI(WCItx=F>PU3bD0#gXL>zn>k=ap8Uv1^%EEQtu%{tlymc8rmnI=$<(K+;b0TBGIF@u1Rmbaj~NFw^Ok z2WeNUnUS;X!zm8hu2vi;7x>(}aK7ID@$EO|T=NIO#}r{?Ko&qdV+SZs#qs-$EKcan zkgDzpO=>$N{Ig@(-<-3f9XKqV(3%)9O((FM=JBxim*IHm{{7_8g1+yBL(Uc3_U{e@p{ImM684yJRU)NV$KXZ7=rw+NZ3YWp9Z?kPVHbcY-B4Qrm_sGn=iJGm z)GfN($go!xcgLvM*`Bn^uXp+NE_d|UH9Kpm+^1?sVanC<0L4}AIAn2^JIAIDQS`{^ zY~~{zYVw9C!sfqTUV1R%U8}r_Yu1q6N$u}XVAT!O_58)M+yrvR;ySQRx?s z3Asb8DA1J~0!F9pp(-Y}KZ_u1g$T^GZaI16c)=M8r-2-W=1Qi>(PiJKiiz!SU$FMF z6GDd_r;Z(w!;Y@p5jRQlFiVoJ{u6mXda&^UhFroB?+ejvb-Ns4;*= zM_YVMzl16>Yc4|zT^(A7X^^Y(^#Kgc`^jrULRQ269!7Z|VN_?e>^t+?-vp@!1k)N3 z`~t~{bx0;$o%zw~%ol0EgHif;(;M6mrw@IiDX`f{@4Xu7y;mbWVvwPn57f?6&L-I3 znBbn0t9#^~3R_F@?qy59m!Nigdn~5cubAHc4*e>*;wMLe#4)0D|3aGAbvkv{+#R(yU=?RMjbb}sGe|FEMm+xxX@u7apXnE5Zl=}9*wM|_EL>Jw9sx( zcYp6tOSYhsTPL?&$!(EhFmr@V(uu$Nh$e)CnWrYJY4Lwuj{mVU&rS7gyK~`$#x9(f zLrYlGWD9l3OBgmrf3_}he%wR5jJ5$}AMiLFa5&I#py4!&BVooyriqHdJe%Z~yk8IK zBEe)(c6*o0>at2)R>`5WN_HIQrVjIt9p%nJz8BM!Ya-jCbUcmxhtcC~ks2BKm>3Dm zmrvej7~ZA8RL!ue+Mv5v1u1! z_SI>&q2}tCIOE!iIKXil&8hWX&zWe#2x}%AZZ%y8s5vYhZ{@^Y%XQas{rp<4qp+o( z>#p9mQ*gAS7cQdY)iEKmUERxJrNhb>kCjf3tnIJDP5P_wF8Zr**I(rTad(Y`FD#{# zxnnd~QPezlFuC6C=<(UVOD2h`$12(LmGnVswD%E4dwJH6P zyPRpibf%7jz;O_qC}FLEk-np(_%LLl36)f za-3&_I*73`na-FLA^&t*7cv&@FTb&tERngLUSu*1>1IUd0B<)+S9HQtY>%H z5O_P6otDdPS0?Q|b_GniIwGL7+!D+oi$fOY;L-t>ufy@Mr~dH2VnYP^3ykvqg1xH> z(Jg0QNz=*fKAn3yzArVee);6O^n#h91bca2utxe#y|#U?icEpGESkwEc}gZAdb8ea zFdbee{U_XquQ-|y$BCYZ{*>L~6WqeP%x>X5Vei+CTQtb)`VP(Fx}I>qYQoN1ch>re zTI)r!fgo>I>m%eNU1D3sIiN{PZ^JiGG9^bl;AjUN?Z9a?!U6F#B`66{#}p--b8=|m z(88g`@z7$k1cLZ*wHd$nqS}FYpQow!2BDrV+o@e$L&*%gVkbt1c_-MFFYKQ~S53Jl z%~peUisk=2L}dmyayX*b<#7DhV*_V_ITG!`9SQo@0ofyw*NJwn8fv07=45su8K+Ti6zy2)D3~83P-rh zv^a96#nE6TQ+JMI>QqgsYZ}`|q_NBKT0_p&@$jbI{0#>@4tP4UiC!P>ls9F^=1tkL zg;KL`i*|f+E!x4rk}kZ}W{dV_6NyS^AlW!=C+Q?PB&HZDFk9eqby#F+JLS|tjDr{# z7V4mr=%O*_D4yOWKA0Pts>`AasP{>Y(`-X;(>b(adaiHiv0Er?rnx)CA#>?-c7QZF z{U$vdsdC!AdiF_|YqMjIOPO;ib1r3W|0Dh;dCCt_{nP!!Vu**kI@n7lwfC7`8HB{{ zMAs-wb0_-liH%FRucTl$jz?AkdZ`9>C#LnY?6T1aaWd;<_9SF>LkGu;N;2#t zWTJ%Q?o{vHDebL$i)!85umKPfPWN`Y_i5_h&xfgQ!{*g(xa;aR+-h~3g9`^2&Qx!$ zx^1H)j$wUU-pl6=!kt&RmCUj`ul#vf1FyYF4}YCOzM}!h^Ds`wDuy>s{U@ADq;B}` zBK5xD94@qA;)ZUv8bv+%h5hO&?w=r2CBgjEs2mSUiTYwn{ktP84wxNX zalpFeDGu1dn2Jd|Z02XIi8f||4KgZKCtd~xa z>hiw*xatXqWF_vv@4GYl=#Jmr@l_YO5YoT??|;60a^?kQXwWk+9MALl!|V^e-u*(% zgWJbRZ`%{SNcQSIrO#QCR~FvEJk-oY`;rgyl!@wHP2)*UU*1tlTEAkJjvL)^qn{`@ zdZ~QPSAya%N&Z`qxn*|(uS_sHkiYu3a5Cv+@?5$LE+|(t8&z*|!|>zKQMJ?3+L6JW zaMHy=gGI;Yq1x2-JLZE~mnj~GhKldSKE;o~KE)mBl*5@baRt6Qm|5nHnELTm`Uq=3 zd4wIP1t|#**X-bY6G#XO1$J)xW4P^0a!8nQ4hic#hlIHtk^_h;iZ~Btt*CBX;Y9Ul zIV1t|A4Z_|nw_S35=p?k&oH*b^pWz}bx4;G7*=hCv{HJHBfB+ieKt~2^mA&2aq=pz zY=F*=-PtFKP*bk>>#gQEP>13jPXTUmfhJpoN+kgaOOmCW~*2|K=#d81C+I6{hW{W-h{Y^ zd8b9|WwUjSSzTjR*O>LiHfB9qasIZ0$)GxaOa-9f^->#{Xb6$;L8(UjEIYHMaxW&@&>Db_@n)Cjx zkk7t{hU4pUy2j}=<=N0qsEJ-eyVFQ$J5G|CemjOvSl7!#&r2e3wU?wi85ajFrQ@E4 zpM*I*>)hCq>4GSQ_#yMxkJJQ0?~NMrV~2QHK0Swz4j&ypI(q1X!lNNFJ|7So9V$9h zbg1Z1u}kmVH2vCBfA}=OA;3-;`K#GzLL-d!)M)R`ugj5o0D1Wk50c+>AX3e+Uq8*R zG7$JVBp}%4aoc49;FpsHfScoqTbDR5U&gC=z-qAVF58O+dORe$Rbj<0e^j$D%(~0= z_Vo5o2w>G20o*nCJ8+xrmnOldVv*%sWDDB|2{W@ym1TBmla)?xZ^5okEuf~hK!V|4YEIR1;Y4dzHWc*6jG=|!==oh;%O`R42P)6!dmDdbv1>S9 z_nZp<(nrY{w#%jB)Y=taYo=MBeG!iNc>fUFs_qljjmT-@64869w#s0%LEb0u|Aqlw z|4ebbmx#}DULI7%UFo=+Hq>>fO_wEPa$#AP?*ex>pjVOrjIJhq)Q`b`to$hG!|0w9N8BGyHOuTc#A2s25Hu8uD$ z%Yf6f)O-DluC(n?wS|E(Re#btQ}JDO-soe+tr$;F98?L1bQk&;{5q^9=Y$_+EnI^i z<-Tq8QAK%0JJ`+TlaZH+K4n>2BPH7r1RF*hwJ4j&GnIH`ZT}YZBl0vm0?%-U1m>y# zXu+Dg%09W|#Pzk{d1cRcXf+<$Zh5HojwLTfbY3M(vGR)NJ#XxeYC)Ca7L>~c>d9_V z)l|0BwyqPV)})2XAgFwukmODNQZw;on9dZ{4w*nc2sS*0o#YzfLfcB>yw3PxK8C_nsT(+TlG9pz#)dbwc>a=<@;ui0J8#)|*#j**6Ehoi z$`ZBiB=&WMiCg(xRs*HFy0)z5x3QBv+XR4pHL**V2F$gYddb=0^QQHDOb7hR(SgYq z>r zLV7`G+SFtV^z4`gr{Y{?L{DxEF(Fcu)aU79SY7(FgA z%)13=Tm^n-rxGv^8W;=VTKQYy=5$awYrfW`@++r;t1s%I3uo}AF`mkJR(hNa&H#~ z=Mg8L?vcjqOhK41d>Ed%aGdI9{2fQ`)~lnK{zceSxw!<}iuVWoHVbq`?)rcQI{U)1 zlsglvt*WU0wOQ@0fGD^LvOecn9e!95HviJ~YJtwt%@PLvk_=(JW3v5pWfdOYtADQ3 zAEn&;)K?$5PUaUd*tp9JZ!4b;_DhG5J}S+H>LJt}IreV1O9yD>4P<`@>-iTW^d#C0 zV!%Q34USpHaQwU^GX(k5aT|;mgTY?2jg^lPjGNPOb$NPcYfcg2K3$!4dYQi?F*x)` z5v4u5B=SR&_Rq9;3A10D9;NF!zj`0vd&T~qxGKb$k@{PL94U?V>OO9Kav{1$FW=;| zg-|42yhtQnE-4rBi0??sd0I*RIGIen2-fd|KGpGK%fp##neDsl*q24#9o2Z`)QbN- z+0EDVt#$V{ToZNalwA{L$@jf~+dkCp_a_T|9leXGe`I4;7P&!G_RN$fFTPG8H?#96 zKbYgj{V1lhkE7)LKDKP*)r8edClysTNuFx5Oq(KwQXXPsWnT8QL&I;&SNmo_d4?D- zWaAt;@{1Ajf6`Z3o-v{8zS0?G@7MLw4tWp0WMftX^YnW^Kle}Drq7aQ8+KX7lrIk6 zA|Vsu97OE2N89e6DDE;67(W$b?lewYZ%4PDr%@`3@r@0!AC1d90K=tIT&~;HxRLON z{N_RqYbllL`vJeYuMhWByfd~>{Z;9-_VzAJMWET<`JjT<`WxShZgRw<cWO3);NcvHzska5J#>YGXF@I>6>?8@TW&Kyqp|5ea2lyJX*gMJ;AczfK{D8 zQ^sJW|xtqlpg!n(({2Q{0kraXz$O1t}Z zoM;>?wSHOhbVygb?}gizCVRvPQ7|C;jV#!ZzB5nb1b-0gk**b@or$ZxCZlCyCU2f z(C4`hthLfu58Ph_VBp%=r<50Jf z5pTg{t;gTo(gC+$)rXasKPw?_!)Ud}QcN6J`MS_Y<7Afo>6Y~rie2)|*?jAIay-t^ z9FBba7gvg9(<_xz@T1e)A}COgBKJg(5@(INd_H;~x<75143`a>Q!yBsoV;?Mnhz!H zsJSsrd<}D9ju;|{lj52!l^U|7ja8xhP1Vgfg%$D4q1UCBI(_Y7zuV=gu|b>;4kFIF;H z&oI`2mP4Y-*n85wVyAN~?9~kNbtBDIjo}o>*4`*)E_<WWOdn(R$;`_W%`8+}RtbCSkQcpYsv4ax|-r ziLUx`8g@8U@!wDm(B!xo3UT%Kqp^62E45e4NiP2Wo=G; z1HRibFZNq1%HyKZ%hvB$50w$caQKyd;e0H4G)l zCo9ibYlvQ!nH@2MI!!C~TFJz(PK8wI?# z1&o%TT|fwU@=Wd;!B3EvKaw%p%wf}CEIsqovYKnVq|anLe$MoC9cGu__B!RV4OARM z=%(O{&?D)_t_2lorVhkJHM@rjX?&GtpiiJ0UEUo#ooLi!T|LZgEET{)UkgNrc}O09 zBg4ytZ7>ry^Nu#ZaFqjGNFw3C&=t^Q91a$&IL_r%w1%p-gm4PBggPxDRjm%oXk|#P zBL^D{Hc=y8c0;Yd2dyjIif0CFDj?N01TdVYw9Mzv$dScPDR1`Ex3h*4S7cjRX@Z;H zs%VXs^nuyK=} zuAka$gYiX%w^H&ozwcTemk>Z%Rn)g27Awko=nrn@4efT)0@hE;%fCgn`fz=t!?bca z4)*UyMH5sJwMaC13;P7;S!!5jo~imv3X598i!PJ*Zyi#?>XsgKwCDd}ADQ)=5-kJ) z?iFKzT-EDBE%4#uYu>J`C)CWg8tO|9fQ~W+xQP(NL6lG_lhhKJJpCmyToRbjDV-A< zjYTKisw%@pRZ-8ID* zQ>2M>2vgJuQC&k+bRY0X+rC}$D;8~-2)yval7Q<8cCPtG*xLVA8AnNEj-oXj3b?$( zash%3BO#a6O%F9%mRdA5Xk<}+wVr8YcN|Qm9A;4K;?50?6L_WvAv}(byWDboT^n!n zrJ=t8z*|3ue|Wx?n{nz*SN1Vi^}w6z_>e&_{>e-8AYr#$o}u)>3ULEuf+R$__y=Ty zAi7pJW#=4QCK8Tyzv}VbiN*ba#jP#1G(!|g{p?TkQQH3i8)j-DKwh6icvKVW-cPT< ztDZ)C)EFYiqJ2h&31coL{_TB#R0M#i`}Q7pG)}Fzo6H2w+#79=oHN9hr|9P`_VH0| z(GvK_ifQQ|ztws5q~zV~LH%WHp%_LsWmOe-LAhm_WK2=HBMNJ5;Ry?cRcF*FKm4H! z&%cC$w8=!N-N>E%NEG(m$>BXBSqyOEJ-Eicv}^tj!%%4Z2|x`Q1WVy$ zmCW2_{kQoe2P4(bTXSb`)J2}+__nQX^zsbk$%J35()Xf~-Qi^ofKe?_`SH?-=yfnJ zo)_~tG1zJNkDRtC)81}Xu3Up^WdXL0AI0+@D%ah?8;T`}3AOo*`)Xv2hbT>kT+{hJ~V=?mRa!Hq`Jk z_wi4@3@@>}>5GWoN-M`e&@?$FI08)eOwJf~uk+}<&>s(+o%E|W?4$f$+uKz3SC8_w zCh-O;p()>caf4_QzGYhBDKO8xC4#t%JG~QQmU8+u=gYJiTl;(I!K!h3*$$i4<8k`Y z=}82;TYH1ioTRoZx{Phs>RmYE-Lav%xKYI;{whUqNFw{`8DKjI zj3Tbk)cVdGBsM)%wGOIoM^!FzE60TUG<-_^Ea;3XvkU$LfHw`DOQuY-qHcPIiJ^lW zv_W6FQuv}HPmUiuBkAOs8&*xUSc@xz)(3;+t@m>erln=(MD3fps!8nDpO4?D%^5m0 z0CHq+CHi@lCcUYAyLFcjxOBV;NJIAN9a!hOhTeYdxo5$D!lm49PW0%Phdp~tnyrKqvscV;}=I_Vz!H?$NZo|Hx+NM=M^%UH!rZvB; zD6Cuq8=NZRSh6)-rJCUrZvH;7A`dp~9p6>g%rvFIifv6Vjyc`pqJ2G~*akW(-JM~5(3$5%*6JIO@>NbWG<@t#mU*^%`lM}T zZZ6V=o-ZYtJ^T{0b(1W&{$1rMq3k(^ktIBu92_3wi)iHWwmO&n%L=!B{2oo~k8%8; zd$3|dCQLtv@I@@WZv)c!Cr$j{^&b0>e%#n~54faKp_{JJptJT;py;`n72Ixe)irMtD~iktX91-b5P4+mDeC?zIi z{BVctld`%#;oGt54d1sv^sQb7Tcu00+_$>n!4;1uD`w<+0^eEE=|5^CCPSl zKlctlOINdAd_A5kUT(L^cYd&TKMA|mS-w)g#C?YV|4`MVe4~5feF4r*sCPoY(N z1)KeHOdZHqr4ZUKOFy|J6=6AY4qF~!|=bqh&noOeo3SVa5pd= zhFJ2Tqg}7J9Cv+BH~x^EtFX5&zo3#kwv)=8R2$z)OQy)vp|W9YIk8(pA6BH=cHivT z#R&H9w?lZp^a*|Z@7tiV)^&WpzkYzyT)3mYZcLwdVY}V0o*dern?5|=zxfxw0XnSt z!{wvr-OV_2HaI)v7K8F6`cEaALuRa5)YvEH*M8XXlv#vRuxC?)yE{BVo*(NckFq|R zU5jVNuWsvJ-?Zs*R2jcwmw(%H_HZ7=*N6X?#wu9-c)rrUjR84-tSz%T^L7}4-)Y;Q z{``*PCwLEX&x5(SK7E^wY=3UL;^(n_zpO#zEjUORhu zIj6{>`&riCVK%7pQ_A~;Fw{1&;&YhylM6>)<)`00cc!==qxYT0FPDe(!?}H z<@>Wg9h1mN>UG--fA_f)6vrpFkc9U{@y`nCXGghDL1cz~ z0&7F^B?~mXB+iD-oB<2LDMZCC<^RfB0*k+`Yh){aHde+P{iak=e zC(!WZXDBy)Q$9Uw z=qDC1w??Ao;6Jy=@>=yx-iBB7Ny}oc4@3v!=E2F@^hx~mKDez#?KUIkh@TUBe&1Rq z*49T!VOt(gl)(E6Zrv5;t@%>_i<_g{Jt9A^ExB{uE&AR6?7kAnT%Qz_(Bs|a%|M5@ z_MKRqzp!2Ad~-Fc+dZW{==)wD)E^So+Q-uO=k0B1*1pRFb6{7ri#mzF?kQwEE5n88 zcKyeuPwcz-q!|+2D4gTMrELsBG4G~^{+<0|T4jIPb{0z4(f9F@*hb>0gp&@$RBoQhqo>D=}*yI;h#I_pRT@_s``A_R1p@!XSIhKkuKPGNwNs-+3WE zUqfEE1i3sT%=Vd;5wK(KpkuJm>TtM;hc>>BZ0%_=JDGCl{G$`x58zL@OE--__hC(n zJRNT@KbO0B-#rL4cew_Fi5Y2fV!J)-60*R);iyGZQ(d2zo3pu22HPbUg?=Fw(>Fa0 zPW)zo%^5H8yOLLjjyyfsatDYoKdEx&5oM|0e80oJ_IyK0evf9JJ9c{mD|xg(lh|@# zMPu0M0|YrRrQb;vvP>aiRkF^y6TPHxTa20hU~wQm4>_3xa$;wGp+cm3dgLVbS(r*GX}f6GcK;=gm3&v?1L z)bB%=$1!}bFB^>;rWpL4Q~k$}E|1FWpNGAnJ^LS@kKJ*K*Ns;`s^fL%GTAH$Q#i*yIaIR|FJHT~Jcc*g6kxkQN^J2l#1QUrb+pec^!f0-jGyK{po2Fo5*lVOs8A}s|;gdsBTkdW5DKv~LA zYkM%c*o2Pv+98ZqQPhaX$wb4L(2zyVUR5|oMfI*!nRjA_5(tDI7*0U=2$;4gIGT1S z8~_5N0GcRMP$sR4>BnLP4!4GoUCf0ED52OPn)WCpB;+iCeVByJ`v6kE#74cPQw(SU z!dsW6we1yFK*LpSfO{4*?~Xp!il(n30;=Whheg9AzUn}r5{a1y#F&0H0GS2kEI1n?BYWfuPd}0drpIE$Rt6NUYiy1PfoU}~;E)@uc6*{tWXi&NZ6YWSsIkob2$bJL9Eu9& zT7!Wc!PJn%jmU*Vl}eQEp&B7lJtpF?;L^SXdk}^ZG$TzgF@%f4T6yrpB6}=Hn-~VB zBMK7GhT&gkMH-uwx569?`MZqQB1N89@4Tfv0H(B{N5^a8yBYCM@}J*+Kc5fI)aEWv zJtz%3JP@zYdApv_ItR4ZR(vqaut;sLUd zl?5D#uz9<5xh{o_Z26oyAR$ha1}6d>vd-K8YK^JdijiW>Gw>@lZ^$qEf;l$w`WU!7 zc(7%AMjtVv6?rIzLD@e!`a@sHh_d)Y&$$6wAcV0qJFkkW>IuzUyukib92N-5M+Bcg zq}`7aXyJn1%S_I%nG2VIkqXA_RS5+)3EC8;+6*uEuT{}5e?*xOp8qA5-LT?HS9;MI zt}_~qgpdCu58o1S#cIEBd~=4@2uC~Y49=5-=35TaE^X4uT48Q@>q7urD3;tJmc8Nt z0ikGL!*3sa9--t~>D=^LlU`yy2?E$5 zlH$M&&`KK`3q=3`jVgp|QSh8?lXPiCI}F0#kYrg>hDc^wD5ljM37QiY)Kn!$(q!y7 zaTWjh`;ImP;m7ibKN`;*8Wc<*#3fg2Vcym6raGyZLZIYF1*}TSH8t4;7#F@l z;ecVY>>H6qU5W;`=3ty?b3hM#Gfkr7YmS8q@>;AX*4rS%dIm;8A%z*3?Uq(Jv|~$v zECB!*YVVBf-Y&Fc3T*(u9%wnX@pLGCC%e40g#tp6_#Q~ujp}3#Pi>NNI5Y0>^-_8Y zj-o{WOz=={R+r-qAeerNeId~|qEaGJ+77KugwXztpvYn;a)AZwID$gB{NvtpBE$(i z3xNg@Kogi~)6#3>W?36sd_*Ty%77d31{T>a9xEt!5q{jKZgbN?a2oA6{6xNvy zp{4qw1Hz9fTT_Za#b}5<1{>!llL;x*PRl$8f1SBYfk73ml8TQb3`?4&m6d=b95)CJ z&WV`?Sr4x4#KHO=Oc6>AW`72%9C$@9Fa+o1YCoGRkHdVdka{19;tMLawweD;iF#<> zBHj;QDKp@}^}qIs_9Yqf;J~l!G1nGWEae^KoKy)%h)e~rO?e#+bDl^ff+-@yk0zY$ z8V&19bVcf61cOdlfUI?uiU5dOYsrz45o_njhO+~|^3}-#@5|$1y8hme6|}$xt7wrD zraJ_~=FspouhaTGau5X}%XuST_OR3RY>Nn&u8NI1H8mu^oIW?fuIpbeTPF$bBkFef6x z#9VEav^PRFP%qErpZM1+31VC`oIy+~tZ>OD2|}9!a3JsobhOw1i}DClC0+7T#kXQf z9Ox8&gk-Ro4$%;Y|HCs$aY+&h2=;0kzaE@o%}M7?b5EO#w{_D4!UZ9B^G|ZvW35_t5 zS?8;%6cP)Li^C$le%44S5ELPrq%1fi1*36}4LZQ7enLpK5pxhyc2>~bbOp82V53~f zFFJ&TTh+LaTGxc-*>X;BI85RrD8yhCGJu%~1oGtQ5f#BGD;p_*Z>GHvau5TVc!^-8 zgHoviPsa&U^n#=4sR9)HE|<>Z;NJ@*1Vb38xJGEAlH>b0sQ6tUl4YM_I*4{c`LH}y zN9j*XkV>1eB8TWoMSQ)=?HvdrB64s}04FekiS@k_0HeYK0sxeSu8?z@%lNBm%vj7O zC>k^}M}W#2#@vXEdCN?6urmRPOu{f?z&s|0M0E{T4nDwAd+g{mEWH`M8kl&GK#!=9{~8 zYnf#zAxHyl(5gOh#S1jZAxhdBh>4IIug7_zCF7~sp#Y&3MX3;>ja0tOx+8r0sn9r; zzphTC>W2zfad7d3`dLx}5!CG9g5iC!tKsnt9`W(A(bi>sCDD=o8KQ(rcjD?%W_ffx z^#k_=hrwrxOF)9yU?;tPqy4`Wf5nM5(;bSVt7%|`O`9B>9!Z336hVS9Xbx=VL1L$L zYNs}iq3Xe{B-Q!`s+uZ*aR#4%%agR^9bRaIO0&J@MD>wQrX~538aJX4L*{WyxzyHQ znkjJ`Yf0j{P$8)IatJ{#1Xga_^e;=`@{00#6j0Rt7K_H>!M_Bcu6x$T&o!7t%eVnS z0rbQ$)D{^3npopFMI(>IO??ccWT8E2VDSPG1`SY&6(}sPpPBwLHJA+o0D=gTBvR-D z48-ug(>;SX)h+ac(0H3tnPIw`(Ro>u8N*mHp+!0;_jmY!b=yU6fCYI{bU+2|(NGF6 zsg{8AP)qqSg6b_7%#XqrhO=n=6=dG0fkhRxby0m`e+V+_h=1{r{+DI zv6pXbd`y*^(73ve$&iOn+9#pGoM(*6jT%Xzf+SLsE8H7Z$v<(`QDqn;k~l!^ujBX! ztb=&$yc=;H2*L)@pGKh%v75oDnXIoI`X0*a26P}_wkm}Pv#le;l{&sl0r3!+_NUA=fYOwMR&_)b&JFzZ^C{AFND(^QeaCCnFZ9UBEnxvU5&v z2K>wjbwsb_F@Ur}Sl9`5Sc4>(I}G9oLsrys{REX${fsqF5eL!wHmJ24%w_MWPlBZ>GCb zZwrDRYnUQE-QO+duyxCPRu)vGas097Y1IbQJZ)43>HOUfY^Q|`0XRA!8Ir+K9kF!;UmN&mDYA#$0P*OFhBzm4%!we356)fvk>#*#mq<~U zeep5%$aU!AkGIuZDs_$1dpI66 z6iB@T#tb^VF&ILwga8ImdvfNB|HLivjj+XK4$}&nO_P9RIT~ze{XpEp#3_X_$|WQY zVWE}3A20)&r@cf%1@L?naHQKIpyc|)zl*X*U9Xo#%s@*08glOPDn-~yEZfg54 z^}`th&)u+Vt;{FCS}`@6f{g0X2`-sXe4f9(*-k4Km7fus)0kvdt!js$$z@6OgV(!F zLxh5Yu97+)iYXUxd|NRw(8JE2q|~HB$TqYYs0|l`VWHF6CG{ClqHaP?Lu9IPta4Hb+vk!*nmqwy zMMlJYfNaR-1>&EyU_|0jw5t&L2*ar_1n7C-&WxUB3Zmj;uFl)4aP%yZj606@D3-j4 zntk=VtBA+Pl&8e6Vag-vQS1(vqVoHp8d{%wr$~%SNpu57JbyiWF8)41)4ECpA~dBd zTAUD_eAhY{!Dts7b_R$s4Bt$Av0*P~k}|&CDFGA!ghJ zmnwaNAJEh66rt*}n)%UWXR;Lxa&Sa5$dg@m;h*t0kc-+JLcEd5$zU}My7px)l@j1- zc4bUKc)c@4B;krEBfxTEMS5+d9gC<57l&1Fk0o#*jF~-%1_e_k_|TVO6ro7MMgUf^ zmQ66qViLXg-iqvFe|A9uNxrKifaUPT#A}I>s~>0bkGLT(j~tVgxbN~+wcC!ST9?l@ zuz#pg1M%(g9=)B~JiaD#fd6@g(Jj@yzadu60HkqF{S@h^JjiKuVYF&6Ggk^cE^95( zR_>q3%4pCOys@Q1(NlqTZxlP353eUcA7kmGy45 zg9T(lD3qzfmE8_4Xs~Fj^kw)53E4Mej+vV1f+G2L;|ztEs}?A#tUtLVhc0}w44nf|C*j6LQ|^<)BJtoJ z;>!ZEcLt12S3YF9+9tVyn%BII98HISwp1mP2Jg_WL(0->!iZERko zksKvLh%Qacx0OztwgC(jy7kOVk#!d0h?F)_wJHdNSL|U{R7r0l0C%FnvjiD$96dte zSBOOt27u_Q(trwqtyv*9;2vYT4hD(5@5cAhtx2JkYkCSzul!A_iXmL10{H@81^JA* ze^TsR^EhJI%qn4k;xiD|vX$y!7R7Z*P&9)eka?6sRH$g;AW2W>fbTdUMO$~6{5?1f z+Ch$iAyu(7tuH5LY_<>Am}p>XpgR%E@mYWZuY@E;3UR>wO( zv3RJa!B%jskr(**HUaRmF2d}k!^v8A)%+@F^j&3$ORx|=y|!=u!#=<-O4?`3fX6M~ z_>vDZu}bdCVo@tSEmP%)2WkaQNmM|LwYqK>RFW8CP@b@Oz4K1g`_Lm4AO#MjCpcu= zE91=|ui+%pobVv+nG$C(pP{$W6QY$cu>s@1fNR) zR+ML*7dwS&f&WtxcORLc6&z8U9-9qFx!?$`y`mGG2oV^7FsM9cwUsf7wf{QLhj!Yg zXe2)!MN}T`oWkmGXhP!APOGd8QkKlpKq^4VUo>w$9M%Gk5Q4c!uMm0R*-W8ZMJeElL#BfT*Ei0RI+#*j#QS5{Yz&rIAXUw?~ckq?p ziVcFyu;?Js6n^a%`7!f-HZkps7$=cC{UrgH}9fbdg=_m#MXda3074|G`XS@J|d$CW?k)?iAu2? ztCiWc8;MvPv*5Wvx409-KQxlmSS5+L3A+~JGE)W(@WecZp@QJe&7Ehe1dYB4?#f7{ z3CFQk%FIL8BJKU|kuIkjqj1W&LnUqzq%e!P%wis-96ReoLYAO>Z-q@(L_mI&N<9hM z6RazUz&h5BMwn2AS~3xpjrEZV8!(uJ@2Zea`JZB1g^c`ic@DE&p)*A#mlRLsZ3eWr*z)Dh_MZIQpyd+XOt+2nr0V4YE!Iq`tDU2p*8|5tI zO54s+s&@zMMB9Ah>G5Fj%I@;({fJPi+ny-YBEWW|Xr3r26i)l6*vVo8Epkj5CCnfA zWSaQp%^nU|9S8_o(bi00&_^D|EcZpBZa?PyG=4rnNDN8I?wt93viW&tJ{}&enbryN zlypufq01y;)ONvwOz!xq^k0&+UyC2^t8x?1UAeGl`qF;9;$sbo#%E~Nl&AePYIazB zum;;UdvC9tj@jt@;>wat*fzbIW#6670}ovT24*6ODS;aa2{)-{B!(ix1XW}%-7mM_ zy;eC`rOA`ir}rD@jcYIap(P(pcSP?Z)$D8`m74sWEIOpnyG z<|3-`)pK2X;#Sw`>5?(*%Y|zr8C{W0mPF|qbklJr73Igzs5WjiYo?+;kVa&`ZKcN6 zr08mOE|F9#wt(id)o5E48VXd+fay*=ou?~vEqrRXDv3pP&8xyyxbUC+4vfkk!4+A_ zrvn^U%^-_8KD>)nURN3?b;$nXI~({^*WeaDtF=W^?;UK`x+7xhZ@#ucn_giOiX3l~ z#thv=#~RN~f=nq*TKC~%=gniKV3tb78fHyy$wdq}_H~#hi6nBt!>b>nZwukLG}TO{ zV;fmEZ+d^y3ZjbAu=Efs7uW`4)JnMw0(!}7G-YPaHVoL;ex%#GiF#e{Bk-Dx){JS5 zfut!ZXU=BDA(3#*$?3IqB|x=l;9HTip(Tz4S5h;_2uH9BvSOvL+(#hfNIo@U%Iih; z7M@+vHPLg%L&kxKeU`*g!icfGDnq|{YUGHSOoA$(eM;4izc@ByA&|H>mmyrP#NzCe zKyMJACN(RDYmMsHal>%V2?j#poVx~Su98xbO3P@xh?XBN@-M}8C`DOxQ`$VL=Et9y6xWJVI-&UPnHEER$g(JA*y zoS0i-^$e(Xg`_rZFpN$_+gWEJSXC zl%g)_8QX}YLR!p%8Z+Q>W3#@wc;jEBf#ncah%8ImCgsNJ%wg(jrqD`gOHP_qbvzG? zn3GNqAcw|rLOHNg49M@7L0$28LDOk6v#X{yrAk(-uRT9z{z8HEUuwP@|Ei`g$0VYA zRhQ28pB$}L&b~bWEk$3F8AoU9*reQDM-p`bj%S}jqix`sS8&tdZ=87E7i=Y)_z@! z6VJ;NY-(8aN&yXV@h4uBnI`sAz&I;^TTHkz3Zsqwh+X>JLqe;{vHbccpQLCk=Or7X zPB?%-tLZQblk`(tCY^Gd<#;VgFESHA#jGh6+5wiUXNoEfgQ$U6(UGk+?UCQTuqjV4 zQcPGuoOZP?;s^NlP7`C?F1$8dTVh69c9^8KbQg(>1eS0QL36vX^r6r}@}v_N3}+p% z7riIRXIv&91m3N?OAQGWVmO>(`{6t#bo4x^cN~1WqEJ?9cwT6#A5!Nf%1#m;)4W$= zwo|lq3m%0F+BUjF_e4AwCAO`QVT-6mrk3?v%^fWtF=_QrFNkEIgkmIdHLPEiFb0xc z%uF4<#1wo|>JmtgL07bAJ7pY1Oawq`7fV^C%1|xY+@6)HM9{e*$-SatEPq|tNby=J zSk?$)Qo|vW%;ytwYErH{EsMjXfl-g(q_qwH`<)7E;qIV{kcZEJ?bq|;lVrJK;IGlqxaN*-P{K zAk8y&Vj@v^$vMEhX8{76l@n;m^?GQoWP@CbW55k>GL9En~?c? zXR=UUtG>;w^(0=<)K(Uqn2-v%FCjX}p(`{tD-;v7)x7r8p@&@R6*1kbhW%D(n9w7$ zx?^&|EfzNA4j_S7xPRY$Fs0G6S2}nZp&_a+LrN5#P?ZEN%fOk;K4sEAi1Zni=`A!p zuMV=|K<9%WgeI$)N)^<)uDQ+y!;Pr2^V>9xv^%eFK>$2>y)=&dD-qpCfyN7YokJ;v zG}>|Yp4ECpj2AhlL{TL4+|$9^>Auv{S;glBd(=BP?hbte|+h@unye`2tNP$VnM1t9pfU zjlQ>BA7_t!s;5M+EL&G9SYmPC$2?<1e}vF#cLJD zzdZgaWr{6SVhRE#y4l1Fjjd3OjObM?;@5>PVMNi7E2zu$p3s$6S=LjO6^3$-NQ~$+ zYtn_#v~2Zg^wnCcnVHa7ilR$8wDm*_5Hd_?>*Th562KFU9ie0fm(^Vmg_*+w(ZHJO z+?a;6*>nayL+8BCQsr7&*I~o(tmx^>5Nv9gCgn&Ck^(GVA;1R4=aGfnQk(**zDS4) zdWUZ+y-aG(O3FF)&GBovOWw!0C^6>$6L>`-koj}WYg+_cYvl)2>RL-!LwGj;#(^0aJ`-z=MUA5Ha z;zOQBhs36W1;eiDme3X|Y2Y)>GW0^(0X=!a_N(^8zvQ{V<4T%naqO*nVJ8$^HFAj< zj;q!{00h*k3oLz2a1j-(ioGi|Zl-j)v<87yhO~SA!4qmDD<%!ps|`FmqEBh$8GMxyIU)4?BxS1Ovv}nrT8y zO@d|Mi&(3KbiQSNek#31+GoU`6D55*%=mCDx?X+pOS3 zM0XbVC1!HvF^oblYgxof#{^JNGmgnvP7yRJhHWKnOGE2GiQ=}(cCH|9peD|0Q}Y!G zg27@1ou7*>vo7#+2ix4S9sTHRm)g_UO5)0zt z#^`w$sX$IwDD#@mDB)Oi5=24uwM@Y5(yWOGG-zMd8+qxOa)u3-nvlC*2%;@6$`WOp ziUrX-<8E2U4PfKKhW(1N=*bNEhO!S_xQ(;%O8=mkpDW|Gxk}h499YmY#7BasbycHT zhEs`YrB<699CF>s#y9lXAoct2lRAmHp)_zo`NgyfB&BAH1dk>0Hz$MjwMa4+r{1?^ zAgGCbVl;G7g3ei*p*+trik*(H;=e0P89S1S7+Bm*!*$Yq%yTnrYgEW>l4= zu>Z?JmIPl`H$Z@-vSNZ&KKIZifLLm1$T&pKl+bT;B7kQe0b${8^Te|b7eIi*jPpX5 z8A@%1us_eO;4QeY1bILG&tR%nto@7tW!)tYi<~y{-W)t+f19K!4-H3+B}S}Cf1%cFq?k!2!2wp> zbsA+m?9ME6O|KgcXfs(G2#afk_2-^Q)dz)8tfv+KEne}IP z({GB~X(%fxh9ZKvgIpjiC0fQbrCD%y#fdx4PmexrQStwzi7{e#9HrkJp`8CoVL@ox z#s;?fV$G%Nw7)KfI`oj=`|CN&x3Uh?xIpx4U&oF&GPG779_ccGsr)@=jrp}-cgUa0 zabfo_f6p03?ae)B(pkYz*C zI^~|Q&+~_rHmA2#gZ2Bck<*+L@XTKyabucvUc6v_SPy%6_eaA$CwF>;40nF{OxWa3ik4phXJ`LHsy+jl3>{fNlA1$T=$8gIt zsXf@MCnDRPhpaygma-xJklVsG@ z^YLo`>EF9-&+aA}%*OF(2-+P=4!4RHhuQR!OjNy&O z=}j`yG%3}syvyuX-rttDf;Y@>Oo`MY>yHnWv=oLROKDEO*4!;zng#V@+{e$aqCQ(R@})O3%ltS2ERk*Y8lS z^X@!G9b4Jw!exyUnjWPNuaa-17W-}*k9(8!39!L8ba#DN4EKJ?hI?;T-mV`tP7_f$ zO_<6sF+5Gg@-%t#OFCI@DXRxpC(h~4;HvWRo2puVprL@9>1td{4_v4?#`iAU=R5K3 zkd*Tc)*7X6wbe@&5*zhWsrzq2qKNs3^|7VX% zgQL;4{{)s9P9$XikyKa^+nccedwA;FWfd$O`{=dNBYF)IZ@Tx)$t-4op zC0F?ksg_>5(~^y9+bjjV$Ti#rVtBNNME!xNH4DQ7uVFfsr`{USatbL-*tH*Hd~Fa- zfP(;r?b^nbr97$z(@YP#$ObMHAy$6kRPi%ZawOHMy?=$VfRJWN_tpdL6z=7due2ZA z%m!8WyoGXa`mP2gwR%rCH7T z*xMY{h#;0w@kRhf{Ah<+d!p3x(O3GHV?3*X z*8X6WKHl^O_rvK!U)K)GMtbknNbkKGxoZ{e@^w~~z}2Z+7x8nIfm+bH;=3KKOY?ZU zSPot4T`0hycSYrTSJ@+1{x0Ju<{q!w2Nut4jjpRTCwR64GSu@|wqEn&_IXtGV>aYa zZR%$e>3hE(P_x@icUk=nH5#GT-T%uc@AE1FsHMIzCI>a_E@r9qnAN3QOzO(ocHHa3 zERDu}zD>)YWqV2%W>J0!hY^mfOcR)jrD?Q&@P5XTn!#1GPRUJlN?sWyyB$3~`*+DC zQ9zzd41;7JVYK(Be6;t|kOa@h-2fJk+=ndE&@& zA&_?Xa`FB1#1Zoiu=`lM^)EcXa`}th5DlNQv@)+YMy+q1jFP8hl9}FlETR8|`*8S; z=EJe3jr3D?i%)P1?=p{kJYnTlE!?7_{qn-AijYw*C#&_Us=Jz=tYDXvo~7pgoiic!?(U!?>Bf#^W~hDqVAOAvDNPRu@(zv$J8>58ca#Wm-0D(H6qy zwh*p=#fs+`zSq_RE8nUzn~foClg&nS#8+RH-4Nk2jdnf%o8&2f0sX0eSPa?CvRdC( z!Kc5v_tGvhenM^*;o7f*7m%rnGE6>C(_M`r6YD`{Jr8F+XfE7`2$#X;Cf&M;Z}eh# zk$PWne*0=oOPFX{Vs~>)hOmQs${Pb)(!ap^#ZlWMEL(vzc^9fw0^Q!rb^1n$J97%+ zlFngotbWo=8{NEoNB(+GX`Cbu&wxxo|s z#lNV7prxmob@#443*dM!WbEX%7$v1DS`6Ry3sZV3Y#f%v{IDcihov5GTGMwbc$zBw z#~W`$P2Pjn-)eCh5mjZkCQ8w~wf9C1`BD1`hBtpMy}KEMS)DnA!#c~WGFz0}Ws5c~ z#4>)AeJdo)zEx8M3GTm)Az9tOEQ(Zu*uogGMP=@<`m1y-rxwLS~oV(s|vT)P_2v(byj~Xb1kjuhPUxxo=(5- zbxR#0^bR&I8PNL+gvH{-t_rTt5y+H@-HtP_{wdsWD#I;6Dq5X}Uxp%G}F?csfsJ z!?*{Umakh#z8K^EJSnN)y@BIx(wlvQV;v0h+jyp_nO*+ZPpWFJH0P;GGm$o$iPXNk zwOgU;NjruDF*FlM)21-iSVgJEx+s2ImAn!EaRh+qGp5M+7}GrUSG8@#`rvvHC5L`gV?3sNZ+#G{zyF4cDGc_=QqL5bU!~d{!zSh!uE`u#cY83qoA+kL?6>Qv zn$42&_2@fHbj_)tSShqswzrEry#0AirQnkajkCh*$87w3O#ALWm=||ezC`A|dH?+y z4$WWH9qL^|9G*AQR>{-=c*>~Zmh)sfjm>U)wH{J=+YxA6W_SIx8d3N928Vj}C)l#E z`>2mht!ywOcaGep^I{wi;HPA!PRWtBGvnvt#pRkKGtwMul~4mr*9t7^7Ro@UH_F&;SG*ON>*lh5Qal@ENn24PULycjtLp4e zFtEm7SS1s=Q`kr{a(}SC)$WIaAWPHTTaT^Mo}{r#ElV>tcCZLrU{PJp!%h7CdYZ&w zUyL3#7xu$qi1%(aqy-A*2Bl9Cr!DVO2VL9MjK&;HlhM; zwsxYaV@;Mh*4DZtG12}=&3%mCIlo)^3oCN&1c31VJ2i8r{`otttSwndUgl!%o1$MP zfZOh6ZMy>5SQPp<|g_l_ukIRO~dU&GQRbv17dSb zdLM@qjZe|n#{bRFQFaRG^(-mBc|M@b>sAJJ%gDPi(k zD_xViuB@u8+uG7V5+tJ-5eUVQgP!U`x__>|q}TS~;0T5f*bYzNAdv%yOp@^M zGwij8wbl&~)N14#`4{}o3;7$>sCw}`U-_7KBahoV8ar<8`-}gI-}QQ@`|}_F_Imw(vAG#57X3&3@{51E`F^tL+w`5yijzx?9cFMn+hU%kEGJZ#=SF24My#poIEFTe9|U;NHjz}MpE&8@!Y_fKPM z2mh`44Htj+bbUSar|$9x4!qgXz~}kRz5;ZmMw{NaDIy8q+H)fW@2FZNlhZ&kL(`=>QN zX7%&x>T&V*Gqv!Fs?7w?>;<#gI|p|i?8iJyxkxyU)`?nxBtZLKj*{hZsDJQrZf#JpTlr>haJITKW6%^||roEFJ(i-d{Yff58uLF6E;6kIw@+ z`0@Sv^DFuMHT>{bgUkJwm(QQ_SAThlceM4QUcTZlFC=ci&yS1$o1?q?<^Q?(4>#}r z0zVV~{4@PZ^ru;Wir@2gb^Fum;o^RCz5c@wS2rJixcIo%ism1Fpx}PE`1$JH2Y&`M0#_5 z{bqH)_=tPqdG#M(-vqiGZ=N0>H@6p$zuw{bpMQegdw;c3jQ;Sjeuu|eT|caU`0|j5 zPHVh8eWr_t8(q-5)$NP_`Tpv|)BXDO|M&3o>hbNz3!KEqU-g}}76e;&bN?6rhjTeZ zU5`us?O(ookrV#q<@3WH8PLIy(^OQ%je&20v-1`2rG8)Iu_t-ign!Qfaslkw+3ITfye#!B}sns^3hw_(Q$A7 zSEImI$Ed;>usuyuVFdWR0QKJcpTlpB7k|5|U*Wy~S;?Zp>;Cf)Q-!yEtAASKZNF7b zukpJ7TG6QSs{cF$Sm9m&nVwYPCEu%KQRAimc_gbwkpHbiSmDjztxZy4AovEG>#h5I zq5M=B30`RcR2URK&lFc56hbX$`SpnKV(O{V!0?%7Qg2{*O_{aA$k3j%uQxVy2{qT4 zHM|%wt}}c1N>8gWT6|WptS~fev3@EH41J>KH5QLvP*Q3R4?g`(mGz^R)J~1T;Oi`N ztwp2?@BbG6sm6lQv%WyRxBu%h?)6^(Jyv%W280(HbQQ*gQRau0#)_{}6P0ES$6piZ zNEu!?_^L2`Y|}a`j3RA;{5l)XpIsVjY%^?giz*Bbud7jiXD=LRA<+zKUGe+j|)223qY{(^tEPY6Hrv?s^pl zlx-W}Iy>24Ju51VAX}aa6$Xh7*IR%3D!f=>ocLwR{GoFDOuV{QF|PxZIH zpB?O~Y+-z!7^*n4Xn^g%BU1cSuSmwXm{)t%={2SO3IoWt32w#xi*4)pQwSF|Pswhi z)^@i>xjJW{zIWd{g>J=G+2CI)ju_wE9P7<9UN;-Bmw~YJTsjjX$`W{@=| z72o^@Dhw9?c)e;Iz5lfOX?=h7_EUwS;>(xt=eL`u#|i_&>yAI)zL2&c=KsEE3_>=4 zp&z)we{FuYxO%w8`T48=?Jw{C>K`{7jf@vJoBK5$aPzRf{`KPO_HlD@r&^%*jTY!i zHKrdf9`4p}uijt1y?A(f^Y&RX^lJU^?=K!UH|vYX)x#(J%DcPu>i*)ZRU!VckE{E4 z>-*)!-)-(M*1xQ7?ylGW{^EcA&;R#7t#8+gJO2E1|I_-{|MfrrzY9In_#c1&&;Ou5 zdw3Qeez^E~jYnHIAKv_d@2=kCA)c-uRnQa<{%(E!u>8x*JA6IuFYn^%3TwuH`#1mj z?GG1N!}nLe{Naata#%oh+Pz)hJ}#c_umAAF<&Vv8yHsu3KY#tozRJ@s|M=hj?#KWB z_y6UW<^OnS|K9a4cbogiKm71+{eFdq`QhSzt!i-V`|UsVSsi!z$N%$B>!&~ek()o{ z?tXLkO(gg2)5GKD=1=QK9H{T@MB5&IYM2`qE-zm3FMGh#|M+cTzW1K~`}OKIm2O?; zb?ajbf0YN`Y{}h|%ikOb-u<|~Z~a+r>6m>=A(SXZLJSnX=?wqz?4G( zHxa4#P5A5_o*|JhoO0{)|Gd$gb@byTS)aKF9p+^Z%=!hT&sC z()ISK|8Tkzsbue%lHh7|zI6~j{`$}yg8zu{JmTed?LSYY|K)S!7{p`wI0&_N;6JY3 zJY4?m)#}6T=Hc~ z!Ye|6N?8|x%gZ11GL#^LUxFvg+hOsyi&m@tgAm!rrE%~Sb z=3@Qs={bN5AC1l}ii@xRt|t775JhlGG>o;v_j+~x`0?UK@dNM>V(8`HZLY63KVLlE zUHtXk-3I?%*KR~~ldi>M^1V;TIbaKO!26MezZxrrLEk57K%2(p*&rJp#RS=e13M~& z?@FQeqFWG~Ga)t!9;X540l+cDHe?us*d#e4VoQQg#BwG@Y`B)DBDNqV#HR5u1+Uwk zBxf=rHi|aH<{g3{XI#vBAxYXVV%s-PumLu21#CItl>iL8NC+7v*a2HI0X9aYMKa?k zCY+Jn&SrK*Y%!TuLLy=%5~|{36mFl9-4UA<8)9Q}5jRwcC2EQ(6(TmyHpE6amxzKm znXaTjC7BUhh&IHAbTSv0ibHD4vE(U$O}q`TA(f|;kijL!QN}j3K{jC{WJA7yFFtrA zx5(^i?}#mB6Jis@_E-{^fLt<_Nq~)FGy=91v_c>Cdn`=~vj8^7M!*(eP z3YI4%2cUun^OFl~&WP7yM&_9*3m`R4ydADZ8(hQJj^t9o0+Re>udl^yL~FQ)B_Uej z!qaqO3S8?(3~>D^o_$NrF@^ zi5aaGGS=6CRq)!8O^a43yiBUEiLt&Gkw|(b7Ag_J!k-vi%VaCC7{Y4KIl&q9rkr=|h6Y!xU`} z02Zu4#IFNrm#|l`7HqXOM5d?+K1pD|bpsfSYtdR)|?!UB_jHR`NLrES_1)QYCvD0ma zvRJ{I4jLilF5V}0P2aAs8M?W9dH`~Jo?rt4ro|T)*KE*@oU>s!AG~vTNTiYu;2}?I zi;8WKjXe^$kV;DUKBgT9+1Lu*G%Q6x9*0eb=rA3+B^z{eNpjFhrIdkPy>*bySY`u) z^?K}sZV2)%?94s zXV>62g|@DQFbTYMj-d~MH}bXl$)>cmW?!oq_fSX2<{}Oi%?;gv?Occ4Ober zeaWHd7OWe*ku%oaaB73YKQ1dFWzQ@GwjlFOKM_)zZE%D zJkw_5gWyXST#wkBh~m!iq`F(S*4=#2dD{%&l(vS@cSUa5hTH((0irTOq)zTwNC?gd z-JRlZ6_B?F33N!9|hdzql$VXM#BkYthFxz ze{1`qzUFAQVl<`5GHcolbSh;sQ`E*gP?~1IW|EwgC);Kt?p9$VV}-4Ft)NJV#pkWp za6(McgAcL=8%6_a$GJpgDjKTWlT5qzMvc4!r-YcI^R9)7r`6Xwxer4iEfy^RYn2T* zZU_@lTC|}wi}MKDFt( z`+{j~gK2=R!Q);^B>+ZGY>p;teJzWAcuL6fqwq}4QEPn-z{2Zji7H{pU~lx}>gv(Th=WF-y(H*EbPcK4I6eDco;#0)#$yfzu>rh*< zHb#p8trfvy07uW!7_|{bnG~)SLeNBJ<)@X`Iy>h>xQ(*5MuDubM83?_7WGzv59}J} zM1X9y4w=TaMQ6uq*~k~w4s^zw_DzMQJt1J3VQ@98$oN`oR){aQcS@qb0*Xg7v zT1N(6%603p9jCg-u^=@uBDLU>%D)s`^+hQf=Q7(QuqM_qHk@Ti88-z2X7T4Lo1x0a zMz9tT?6?%=)@wgt5?W&`Uo?_VQF0>EzO0;@yvH$c5Q3G1paX^(|Kz|9sCp*CXt0*q z#OaPnO)-RFwiuiXT=MsS>kFY)CrZrHP6k!8Xe#D(<34M)i*z?FQj|=fNC;aFKyqwZN5?2tifP zvbFBO9T8jxilIn|PXRcg%>$W^;PzbLF~K+FgEsi)Gilh?Sq;KVEOr8^*yMDB!z)m& zgNS=5+Q-;z1BH^MsJ(#d0V$Jo3Uni5olS=aRCOd5UZYyZ+54DlAqPCb+>NI)W+M>c%Spd#adOvbuecWrY3 zr)90o;9rC%3iKUR-9t;%Mm!LJvy=cKV+@{TlHoz;G}EBr0S<+V9)$*{&BHsOY!ziw z4L{F%kE`bF%hK8_h6nQt=yr-(4vnDo%%aNKrIjvLTW;ipw^qa zI*I-U{LoI=M*YbPZX_x(#9VYb&rNiJ?<^y@eqU1^SVRV!IQkHF#ciE)QCaekYHUy! zB->Pq6hi=}O#;54i1U!klPSX_V#@#GM-@hGHS0w<& zON?IZ(jL@lQq>DzBhIAh!0veo#X2gIYidu@|fX*VqMy0b=%R|CIS`lhyjX34RuQo$J+&wPB0#CU4 z+z-tQm*yiFM5_ZlHee`#Nd=jN?sOZAX;g}C(|Bmst)WaYSi=aM7WA^ zFX&{}mfgk4)Y-^bXY-Abqa;?do-7qRz#jueRM^@P1zwb<(x%B}yV$dEo%ihDo{j;!gdD`Q6zDzZWsK&SKjD?VuDj`&6mK#zt^Q+Citg z<{<4LAtLLX2I;w^%C*?o$~p*9XZj>EzRi2~IU>vM=&*WpIOp%Xa)Ew+OjRe2kZ z#-?ytjRt(!c37-Yjm;m(rt08gY|%|z2df^N zH-fk5#RXOO&}52AK1m(jn60z}Wu+uXS`Bew9zC{T#cnD;;xpb+z#TJMkL@>es5%N+ zHnxGbm;?SzT8UTv|6QHo+3XCjGtCKChPrEP84lN2a`VnQ+7=r+M)BGX^Z5ih2!}U6 zu*rJhR65syC0~>)WUsi{N}r-r3jtdLNo#OIQnx`>uwk}1Ib1zbnCClR`}CXOU`5@aK*gl_aK zS&I%u19-b-x6jEoXPs=z%5O)eML0klyBP?-760J(TU!g>T3d_ASPO0-Z1)UwB$Z2g z93x7z&Aqd85^^p@g=V&>!?mKc1M?5|j+PA|&1wK&3et8B(+H8|v%Rv$HUs#WMcLFz z)f05EvOXKpVstscZ&cA@Tn>tkyifJoQ;AhKSBYs6C`e6oOPhhTuL-td8!y$Ow^ICd z#FXPSj25lz1H>FpP6cN&6Z1SSqMb9ygFXjI+g2fXCGyZq-CRVmH>)?R%YXfOeanlV zt`~RLPY<7m*vi85nJT(}H&JsJG&VY{h=jNhwkoTX$+flK zgo~=2xROJ_b5Z?B?QHl?(b@yWCWyY|=yc>f7kfdia~lj4)SOBmar6=2CrUL=upNb$ zs8~&@)bc%Q9U+zCQZ#l!gVE-IveI2tv;vVzx8O;3L7mm+ASPsTIGT!NV=D8IMV(dE z0~aWOtO|}OrL_!oLGSK2^YuJ~5u)MsN(3fXisW__sKkp(qScG4u^2+isF<5#sU4yn zq3qiJhuhE9v|80X4iC`|r;dAQHIJi^UF+7T9YAPB?M!r1FKM>Wjt@H()Pk{~CP5{A zG{DRDsOG-vY5S&YpBl%2iq-BtF)5!{^-h2zQ-?Y(|Bi%l57F+FPH;Mi&~V&w^JT&Z zZ+XDgXcl^ua%;yilh)Y}AHrw2d7SdU8bzDbF6Y-)L^}c^L9`B~so_r&3PGn|*wx1* zID#4)S;x|X-bs2(7S(5-*6=jaK#{(0{S{Bnz~JyTwC)`;tc>e*9<7xSkNkxlFJNJ-FL@uvI%lGgpngq67zIJ*mjn zIoUm^*N3A)vD`r(=S-V`7-tWQGjG^vu-BV0Cs&zME~ zv4VGmQyS-y#O*9a4*@iZD(#a=kC0EP$G)It=RT`_!BH?j55X!3*;+^oO&$_OaZ-|4 zIZ_iVQd5uYj889#s4WzAI_kGaFtjE%Gk1J0l6PI4kW;9eRUk&3riRr4F95jJaeFgV zQ$-^qObbqr0D^G9X~>kSZg?JgWbIh1XvLXe8sc^F&8;NLY4p=zS|{OV2&N&10zOOD zaT!V#IfRaszCEN(=x+(!4xxc7#av{gsBXa}13P*4wevyGEzq9RpRCxesIi0k7C29FiSYeYGM8ym$hywofsf8Xxos^wqqT z4;nb+fZQJoLov@Iuhyw_Ff4&vlm-g)mPL7$+t}^>)&2VC&Hbl`%j@ga%}N$GtDn~Q zS8qQpu5KS5@1Nd2F5awg$Y#TA(Mm5!+VW6(yjKn6iH*^0E4JaymKX@>96}`T*ckO@ zhG@i&PmLaT)Xksgb9Nt4tVk*^vDtE+eVlyrX4xfr(tyBK(dho7YQKM~vwt*z7BCU^GE>F7yHg6<|BAo~T ziyo?ukYty8sESsA7KRRLWMFX-C9H$vCsdsR685 z0{lxd39&h=PiWfy#|YWHkszu}C!N+oUQOFQ**!tFU~7s7hg&i`HF!!*)ny81OICIP zP!lrAoUuWGA<`@wYt)%)4G*n}SES!$o6pRoKAX_Z-P6ND7W8Zqn%#UZQ({GHQKDK` zt3-2A!q%h;n{A9unGVQf5^f}Xzcr^gN^#9hGQdvPek`zx4)OZRNe0_%1AO`*MtKE+{SE3Vtwt=gHN%E6|m_9d&)_p zm)79Jr1q#-ji!9#@~ChbSwk|C47znyhQ`=D?neBJ%b5cCxc?$sFpjb5Oij>468V&$ z#xf{omVs9$sLqKoa%(@2DlVBDqy_l~PEhtCi%)S^#P%ELcpQfo+abhqi`gL5t531$ zwV(s0laX7nAva0Lrm?vY9<+>WMQ&oXGf?2}1LFkZ;O%#;ZQlZ5)$$JxOb$H)JA=dH zHVXS-AC&r>r;cK%jjTg$qJm3JlenQJ89Sq7wFkvlk3~?D>^41oU!aYR)KK4ydy39A zk({s!o3mEg@Mt(bN}5rp1=>M{ZN6@3uy#WUK+Pvg0BAr8p+46s3E5m=qasBEu?k^& zUw1o{yj5<^IiV0^!wB53>r}Q$M=ABSAK%NXtQyDU&;Z$}?qSeqX+jWk8!ZfRg zds23x3c+LHk{j6z4NYZ@$kA+AjhvBg;91rFiV`ZC$zrar8ycOpqz0_!SfzYo4*Ssb zI4D*-N~Y+MunpKMxQ3UKDFZHHi>kVa9j|2*Uh_$#bYKEXiOLJH+Z*h=ExkKjOI8D_ zfP)bV_JXk6nxNYiuF+xK4UT{*YNZ=wy&JUKCb%U-nAQu;#;TeKF%Wa&q$9?`DCr^A zUZ{@(s0PpmtR|86KfTsT1{rWSV5|z#S)?*We3(WR6>Be46sGC;yH5^R^-PBDRz(kh z$fyeI&SC%f+#~6+d~Sd)h!!-Xle;%|K5+~=>H!G&Th)=O)skK=t^X)-om|HL)6nlH!zr99H9{q>6#s z;eDu*ReKHBf+~OMRa>+jTcK0v!I`0*Y@fl(S`u}T4gf4#Vw!@~qE*xbSCe?4#VDTi zIgDQJ*8ZncYAA)E19Q>0wf{~@)gv2gYg9-FGZ4N3hqB~(YHOXz&LC7vs+olVrvaau z7EKpa^Pi!b2U?pCs#zV~iHTecKx|1H4HZ_@yj8&i;a70>5d|DjYN}kp<0wmNHjg9a z%Toe&Rq1p9Hb1$fX0)3kpc8KL>7au`69hbLWKgfBTMa_;MTbevIIY;^iam~o@(@{F zWjVB&cm@HFVl@n9z%(QiL7Q>WF5uCr1TZAvfqc%XD&&~*d~&zmYB>XCUtIZ_Z4>KfjT3dfHZl? zlvE9Knr?>}MREZW0hn36YlF)^*5~Vf=sC4mtc_0{7*!=Njle=&vCi7jvI&xHOilSE zz#pZlHBg-7>JBVp9aPiMfw+NGD&kpY;(#VApTH>#2~g5AQ1%O_sZUooSC{(D;*)X- z=GVd%BR*4^$DkuP*#8;|Dx`agSqr!16tfpblFU*$Li5g8Nh?taRlay-p0+{A9srt6 zfEDKn=|Tp2=1kGW$p9@{%|KaYFgeB06g#C}Dcrq79UlxJtr$U?vWm2%9#Sbl%+~2NZ$)WdoC@5@Gqo|FBI&bJ zdU6n?6aE4@gt0=gsPu%Rk}!KdpXQ{K+rp>vblhB`9`2f?*?9pNWdE z&#*mo&RYz!2MXs#V+E()>3gbMkt11aXd&wOXHfqCR{T31DYe7K{Q7GL{Sy1bTQaaM$1Nsrg1L>WgQn~rrQYw z*~<4690JKY2cu43f$OydJ!1ZVt=-85kQGTR>nt}DP2&$(4GS2dqY;P(5=z0%qs!r~ z5G^3Sd9So}=gGwvx*Y$e6KU5s3t7(R0NR-x4?!~*vMO2PZ0I!7#O7wSxwSK?B#>+9 zDRFBccwBeWTs=!p`ME%*S;_ufx6Z9ARv8ZM98oGNW9o9+uPcE~0D=qqh+vSSj}+YG z{-!e;N$I#Lt{o1|bA$=51cw*8WurfYj>3_$qe`GsIW#*IVc_u4ELmHe$WDs#;t+qe z-#G=Db=xxxC~DGeN%ytBbSD^@A z3tml}#jc%_RWdCDK_=G>STnnkW(09Yr8FZ((^4Q(CBij?FRBd?YH4-G&MMvzMpI1# zwfW?d5^mm5JFhbsr_of03ZXzmvbeNoXZB+uc3Pd;)if#U?gU`N%c)s{WA960(BSY^ zu{7;fWOYtTtP-QuaRv@g?K}##v;%8!951;u+i)80g&e$^G~)@?9&bvwGZm`@M=GA8 zFHSN?ALn5W1S_`_w+@mk?A*kHr_3Y4+Hb%hFd9bbIUpp>oYYlesx^COX{jHzVYRbb zS49`~1iIKGGsMP+8U5P!?u%yZHJe*J+L6h{>6q3sLkp~z$^f3O`SM+N3VZC2ql zFFIVC3La7geR>@vssV>r>-R3?$~0_O2rlTL4C4fHJ!6eS?2{Q z#daV(8HdkCaWhwV#cQ-ZBod+#cZgOVGxaDYl@9xiFg$_d5%s{K#htO ztpU#~j4nv2)fKi{0kvqO;YubINK$tgA8DF^8XLQzD(VNo64VDKQT9^d(At6VS`h2V z8d4=kR0^k*Xqq^OH{vyBq>?3R(j>SZvvFq4h}YO9B#po$2>qnIr~S{Y*{W)in{n?L zqgFbm>D-c7b#8Goim>3m2CU8V^g=0F@fu!H@kyDxCHt4e&n=nTFLvxFUQIj1$eIK* zK2uJJXwRRJE4kVkpjL%j#uMNcR0UHNyWJ~3+jkkqh|k1`&s3P2DUovUl8e4#FLs-f zov4wSX9O`Erb*>AF*bS?Y0%lFuFlyl~-6{oFhCGnjon0nu_s ziiqSe`#mr}#RkzL?qF^BiWX=ySs|J?LNpOI^enMabLG?Qc%DrNifthf7o%$E=ycR1 zfmxRaWh54H=M*tb7`I2 z>mh_DF#(-Loy*AC>J-%tlZOgutYl;C|H37;XZH66&}_Wnfj^s-+mH5-ROw8tn#HJA&(9J?@;(u`IFUI!bnKuiHhzHOtmD@yY=lokXC z9*7kwRN3Mj)>cS!MjHagA$2aXnib(aOmHD^`_NI#hDusAR?=KitF9anX-e55_w3r} zw8AtMP%YFjd3<1=R!K`%b&iA`sNCm*tSuR>qPp$6hAL@5=t!g+;j%!q|4Lf-$s7<; z;{@!8mYWL#zSV)WB+UWEe->E5whV&Aty{M$&88=_c zX*8qiZM2T0C0s$G()?0ltgz)7g4KLoZ7jL?sA`^>r1E-N&y@UtWt3MspETJ=+;hv* zR>c8)3K59|R!3b(WU^-zJ7Yaf4aFoAFq$&WO2w8@hvyJlE1bqzhHTf(cm zg0;L4u4k^X?HQewMS%PnXpwTk`*+jq2*_$jpabBkp}jWy(FoRzTu&w_51}20 zWz<0wIYQ^i;~+!}>dT;o0_^kB9tW$^)$D z*L8e6g0s$~5oPb|!)M)=-vb9TI8}R2_~KdlmJ|G|$piCQjttSz!&+A4MLvFpiJEe<@L z%j$Vn03@0FSOal!cm4D*-$0ttH;OgzgrM_C2+)D1H9I?t>H`yL!KVz2=J6O|%Vc{L zYoL>3{#!RYJJZww#WY-iBq3;Dsl+yGY&U#n97!V;%_9JzQuO}Wl6p89!L+mQ`cb$M z9Nu^RU`4dUn|hu>npn95s+XpBQYf-LA$vj?%~|ErBCt?ipxkH` z;8iJm7sBhFo?%A2_2cMn9t4{CX!eOpO}#KPKvAA%%x1lyd2`YUH- ztDHrE5gc-$g2e3!ed7$8x6(O-63GFMT#8(#$ZbR$EK@R)G-FU&JF*wfqOHj+Aqj3i zuXEDm+$6b;PPW8gG%#Z5((G= ziMU>Qv?I2NVx!y!s}2qhXJ9qMo@VD+%wuTHC5hyX)uLr4zJRlip``|qIft|wr^vk^ zbl$8VSC=j^QuYb$u5VMt%GOkKBNW**!wC_c(#U`!y1Z>CIh7avno|US81BZ zfZ1)Gu{wALLH34QD$Y|h6DvGpr2y-U9&%gNjT>hTl}^3AX%nsGm;?UBO@OetrN1{$ zaKqZ(MC8pyXP?DKp&5$C{D#uLpY9kzk(ZkM+ZinW>wq(L}JBiUi}xc zuNS86dk)Zm(#UGzshI?>Cl+>F!nwOv(bxvkoX*CoJt4kN?IEy=yr9_14?u*A+Grsk zOU_&|PU~#&4B|Ae1Ar>%?VD+|{o^zno0FA4#SMX@sA%9coMs(BYlg?Qzls%->}N8A zvdV5CzjMXww#fxpb{nRBy1Kc#)NdA_{4$<6HamH(MFX$X-mrQT0*14_^WdiCT%^D+M0)HH8gLnq2Yo~xS>?53z2M|4yU)e zt7~_RRBAc9&q9h_IvX>})Ecdxbf!ehHaT+E*>GQ9*(@fjH778&Xj|GB)mh41`Ro~6 zSph+vbfQF+ht2LZ`)QkQdnlmU^ikP42|1Ud z$-@?9q*nQ}gNtkIT`|}|60mxx7a&AZq{y5C*H=J0v?FJ>QUpR!HaUy8n8$puI)za$9Y z=EU8xy1M-Qqs0xv%G1q)XRM~>Yy@dZQ*@v+#loK3)q+|YogA~B$VrhD6CelhT)8u6 zXQg1sP{82|M!*Of6}I#XPRE%uTbB;K*q?S|B=`w6vkL?80tP^NjLkl|Updc|tK21M?X!aBE z#Q>N0M!^mr7d+DsP31^SHN2kZ@ z24z++P7hRya~ak2R&qGO+Syt5tjCpkU~l4hATe(Elh`IwwKna+I-G}>*a${khIbE4 zj4zvuQbZy2v>L zCIqU`->Fj}We}o~i-Y#h45xoGJ+2M)`I*ivINvx%Cc6j@MWTgnCpnBCv;Kaowev&Y)bqFnO{}oSd2rb)8 z(~;Pxg3N(+beoB$iJ+OKX^Zv_u}29hR4MbcIFl8lB_-V@m9bNcurN+!BhQD@tU9$$ zl`wISIK{&3YBYI@odZ~da%jnAjR0D)p{+5aD(M2r%H(vOT*^8ENGZyGfm*f!HKb^Q z8$zR0P|DKeNwi>JupHLF^gm-1jimn#d^n=Txbxwf{ z0W~EPDQf}YczgdDfLgJ#1qu=tq}4g#1hTcqnXFV!)=~y%6=YQb&Ha-FY&gy5Tzo?? zmqb$hG?|TLt){u8lgzjvA|goGBKz%;EWlQ>0L3ESm}h7^&UzU5;}twp>e{g$F);Zk)Up@Q*EBugwVVin5075SwbMnW~xB6LNs8Y6jbgB zAT9DqBDBK`)}Cu=$!e7@A|;080FNky!u9FZ1|y#{s?8x~?;@h#mIl|32(4I6o)CZ$ zuBnMaXq~bM0|<>YZ}mC^ zAd$DMiub8ko2>v1m!>$C^X1@Qx(WPpwc6Y$Q{ZN^_{q;n5I72>C$deVsTdpXpOmA{ zh3ze``zStYt8E#pXW3Rg%hJp_G48>vfvLJ=RL7iarWCO|4j3=g&bv9ph`l|MN z^2SLtByoUdxhSo7i=R-95ZY0AJ#{8XLks~uhIr6K1^XZeYnzjd;7rHQ0L{2K1jurl zLTD6=nIeF{8XNM0oyt59MZntUOnP(Y9DpE^)HNrl2c6BTvr)Vep$TPV$O*R}{@xm-^=2;lBjHyO{FrE-jRl=ZVCu}MF?k-(wp0!JW-b_8OzzuOMyF&I7tBS!Pu z)>Kd2m~-5!VV_2av%AUAUq2&V8&yOj4P<3f`K(hfZwR7M32K;FqTq^h^Gu?NwZF+y zv_Bt|hK!RtWcT!b_2zoLn6Hj@B(|J~;0q*Ybxc=3YEmz307Su7uUrr94AQzmn$=N& zRU}jNKE$6`6fJjX$xl*bWt+h5eb3q=D{G>FE=1{dQ=h3z~ zk3zM!b5M1t&6$(BJv4qafNS%xIy)N?gKbV+A0!I&0W8AHKIgxwqrmmeA}sx>I|>|x z%2~v0#OaZpYU>0ENa4zn*-?5u9f?#eTgzuS(IKWNcpl!Gk3ahf>|8#x+NH;${1qKS z16B_qS4ibNJT$YaplN$WT~`2KILNdnXRxZ(1`Q3`Tfi1*OBJ6P0()Z!n0uTA$f_AE z=>WCN>W zM$xjBCyT?N*UtIM0GhxLyScdPqPKhlqM>P$|t71EHP=sPSa;rsS8 zi~?xU>YDDAKk2n`($FQS_DM9WD4NpPT`5W@RVlJFlXDg(=b=k!dzYu%9iwHdEE+aw zQrS=-*EU?0hiNC7*K_kShD8u+Txo;=8Q&50rl$jT#8qVA-s=pHs_q9%}G`^ zC*Dbn-X}GBPV;y_1*>?iA~ReYNx7U7V~?vqICXq>2wQ;|tpzk}Y0b&>^3P%V2wLZJ z3QkYqL^}D|t`+Tu&b-w*y=aG96bU8Or`#(%v#HrLj+3gGBX1R_`1`Dcb^_?^ z2%OXNfrK`UW>3U80x^kdCTk6`*x8&`<8-xNLH3+gX}sN1vuN>;km9ZKioYo!ZI!%@!MCYK8$vlE~m3fZUnC^ zVNo?Sk&r9bt3AqTv|}L(kkvU|h}a=K;3kT=g6c(SHb>5;!5Q!bD#qg4sx$_W=B(pr zt~57oQIDH2rOz1;RYE(sUTrXfv_inh0I9`A$J`#lIMX@zhS6;1Pr&Us5lydCd7YCK zH+oReK%0|sKf9zuUjRhG&4ba{YMw5Qd0A#Jxv9c*1GIFp90moTQQxj08dX8e47Vec`iW!)NM>>9lc2s#!O&!P%KT z4blbhW|8E_qGZmHd_{V-FCg0W)9s8UG-q|@RGFawWK@tHl=>uVV<_e(XDYswLb>QH zZmOKl>8!?_5$7xUf|ITR=p>CfixHqD1O}uJxWchTSt>RYID-1wp@lZcXeNMsL3J;a z!|4pMROy`_t={P&7nhJiX5v^T^*Dod04=GhVrq!3Nys{Y#?}EeoidO3*8?2}^6@l# zoX4$)X04y$|C2*vf#e1tHox$D`gC=3b*bMhKJl3!FatjzS(R$N3gmKg?wS}_dGRb; z=>ZXsfYgEoF=&gSb~@NhjCKMc|-+?*q9}$9!in=O$81P)s8?Fjjg5vE<{{YoLowH2NU|7#fH?BGZJt(v)5Kz zsF5s?t&L8`*9@#GE~}XJG$vudrI)zN$^>KU0-wa{-7s`gn(bJjvyR*EYl!NbORu7vdIqHXS;9!}p^;1KRliH*Jj zNR2{j?7IR1%uF{=_cl&v)P|0z?3Ici=Tov3I1=sJWOWq?+Ui0I;nc@9rFy5)_79j1 ziW^qC08$8*f6|HyuBdi_wHlmu@QT_|Sacp*apPQ3>v;xtR{wNvZYO-akTN`HDydn4 z8WK#0okQx80nE0I)84JucMt3Jr^_!tSo|c5_t&e3j|(IxGiszAgEqyPC2qL?aVK6P=!BhcrS9D*(}Eh!$*YTR?Y&Y3vPEBH@%K71h{SRAa_EG`S>V z0Y^JVW1CrL#JkT(XK_D9)s!`G2$B(_m4f6zunKtf9PCyDk*yd_`5XB1aX-b9^Q4j* zMXS|7Bfydy7RA;e?M_YuMypPp)W=t@cOz{#UiW1*kc>tH&c%omp>sP?l1y?Ma8{>* zAXpfjPsTl++&mH-6s?X>**8WEIeYmWTSttjEgo)G*E0%o1Y+ARl;QPj5Em@W)8Jc{aj17@kxb@Oz;(M-cA1@np#!L6P?dhXS^T9MW!ZeLOArv)A40zw6KI&(PR_dop z@B|T7<^Pz^njCxAuJHMZeY?`c5Oe}H93k=lLLt(#UfcL zC8>~9ENOeZ80{FS1l}r{7L-YV$PsmRiNZWRPqOwrLoUcMQbafrPMn6UJN>EC2o566 z(OE(q!e^8;>>v#RHd6E6voy#4r8$m8*VbA_!)JS)mE^|iJyXTHwtIJgFfa+MC{6Xs zikkDVcbRyCcc*bOO+D)b`GQwAMJiR6(vHHr^T661Mu&B;R0=%`3P`qS&-)@0cpH%* zr0DUtsx=^V#A&6p&PX~$B#0?D3=bv8C&}~h2&`>SoEbr`-H`NDg78Hp?fUA?>dor% zUw>ZT@-i=OR==FO`@kW{o{_DCX`-&p;t+ta)k~=xq>&M%=^ie~fud^b^EM1P9guc- z@<1|yG$ueZomh`av7!%WbErw?P=`{tJp%nStA%I5wT#O-0-ola(xggSwst#(aQc+R zfqUAAbaL#0V`z6KqkNjyT1#{Rd&CE%`(d7TXR?Ac6|bitiq-Tsec`XZ*?hQOUw&BM zT-{zRcp0C2p7Q}UBZYufMKS`iXs)DkV$&!D*~aojl2VYV;~Gc+l{lvN*m8(;dxr6ZX*&!(W$+laiUZ^mX>V2XWR^2sDW#GQB-cV-YGFS zJI(XpEzgvUKuxRdiqL>m6LC!S%BP)sp8XbTMyqvg!erzxrsqO@4^9i8XVb2)9@n#hwQL1zc*}x>jg`)_rN>YY-7{Nf(~{1+;1wv147nQD zJ)K5uEUS@Drs1TQTzty=t*Y(2lw(6`WbAx;7x8MMi$!onQ<+dy3$~&fHUu&m#A+2W z%k87FBUX#H`X*44UJ!r@5?KOGqI(9TCWqFXHLGOg#N>*ra6awee9vq(B@n?^uR<6e zufF6k1*?fsLQOY7j3pMWGA8qUnm1y#=*0!RLD3pFeP^3B9jhfH;j^iz;f9bRQ7BHr zYLu;5Ej2SU&+mqW=OKI+D~BKf(S}$WPmMtI0;}C`K3(cJ3t7f9zjOaS1c}U+Q6(h( zT%_pQNvV7;O|0}zl_bnr^(%?mejg`PMmzXeTDM0zD#R(T9Fw!m$5~oQt@-h^US1)wTi|`qiYCWr_jmg z5#qLY_nU>zzRp)i6QgdLQ$k(xo)FKrQ$!0^3nr}Qi_4A`@Zu!kWbE4YH0!3RUzEo+ zcQB<#`4KNkI96ol>^^P-@fSI0DZoD_V;>BFG^G9M!Ek>1t@6 z&WP5cw%U~7q$QIH%C^~x)wGx8N-V)|?><8j%?j1B6deW8l@E^OD zq>TSV9~7xd4+cT5(0|}K=F-IKKj0fDZAHi`p5#xCK@e;>EjaHu;x8vUCNxbi zEm(1y$1__k zJ(ct?+FQV%Q6@OApM^Mji#bDOG+YZwkV#}DGx*z1@=W^>;z(6?J-wCW836=R3LHXH z+pgIo>UhZmM}W~{wst*5#ikvppN|Xka6H*tsRIeOOCeIy5pMEri1z;Ke*N?2{?o(d z?fPc(c)z(_Twi_o_;}l>+}?!+yEls!&xhNLECP?Oi2zzsOv>1pW*X%TsBJfl);SJ7 zF^MLVIe?n#H=&{)7IhK5X0g)3DLNMdfk`yC*d*Z>%LqNY%Z zy9gj-1_0K5Cd+{KLXN;-xR2EJej0nTGdmelLTlhaM5cgv5n`ET?2L@POdzl3!t^P- zq~ki<2<+chSC=2IZx&%W-;5b`vSNo|79;Tlc#PUTb6mJW3AD}{%pf6vIHiUdtUNSV zd3}tnGiEwtsFa9+c;ip9eM#2#r5f+zK2|YjY>`rN|8n1+4hBMi7;9td4k@@>#k-<5 ztLE+{9f%Kbc=@vm@16F0>6CDaFCg08W^?^;^=^H6z54KU zyIR~VX*Q6Sj3fX-{BeYp4yg$tMs>6Os0Q+VK8U@dcO---Wli;!{Q7eF*Ab4i$Y9c!50D;Q| zH|16VXGf40Y~@UaB*>r;rU`(VC$%)KYG)jxXDuKSvKV2hQ0*)mAsUW=s6CAXG7VJh z5G@#ep772awFIxXa}0h8L}McZfU)_U5DuaEh=3K0MKmi!izzY^MlJ{wvFyKw7HyIm z5+gZ|AuB~?QeTs;)BlO6AdwZ0xFTYlg{9feUKv7XNH2>|nr4^mUxKsM?c>M$&E3`8 z%eU97r+4ec`_-H4^&%|h!)T=#>u3?z4-UVumP=imhEq!&Ef{6bTqbQU1|2UJBAPs4 z)-7$-9|n;Cm4-8ljL4UGLUpwLPoCG>=Zs{4XjM=AC+oyNvt(B|AR-38Rk}M{Z;tM3 z7HeNqh34HD2s}-pK)2GxGY&M(NhW9 z%u6ax?b9?|)>*C{Y-bjqBEE(kN?=YW2g__c0H1U5*@eXF-cbCsd9z^EpJg?V0C-Qj zHlba~-ltrKm5RyOH#)3@rvsdbFPH$EJ3QCb<%he+MO+||+^ zh#iu`xb?YJ4xAmOT*k2&1SG3mhW2?iTR=|ew$$dTlgl`Uk+W!}1h|4nKPk zCK(mYkg#~dB@syP{3>f_N1>K+Sd4bO!n4uF)ztfuku*?loDZi_0-Ti+kiAa9lpMIx z9-OCgW)+^ru~qv7ud3C!#YR{JEa~@AK063`v$HfZgw{lr1hdXpdK|^t$(x?7ZcadH zL5&%7-YdI!lHz7(e!vmP&mI9k;80-g;Oettt*5ctGWhHx;$EP9_H&v6tQ`e`Gm2H( ztOO0A1tCPIxaX;-S=*d|_N*zdDzC{ixM+d3D?HPkzMjjQ_5ItY`(GD+8RqkGaz=gt z?j9A&j|fE+ruw1VdYmBxp*7)Hw7C#=^h5WZ2z8JkK!W z5%76Y%TLNgMu`nJnxQkGVaB%O6o?A@xOapWjPwAcut;9;TrncMPE!C{HroS`rqWRZ zchs)XEChj0TF;;rdQOV1bx^^{ODl9d)mwc2<;P#uX0y4>PP~TJ+4Uc2c%~p!@Bl>W zJ{_vHa5^Y`^X%bt+%3<87tszsm3HlQ@1ZdZz+27gwteh9@=2@?B3LC@;Jm9P>mHPEwcy7?krJT zCphaQ$qX_%i$bL0aawcO^36C?2KI)|vQ?tC5ORl<7W(D1e4d3tSfT zE#W%N0~#BfyQhamSj^YvJQl6mXu@heCS*(^)v3jBF{eU}8!BWf|?1eO=5}GoaToA^k?*t(x<97j8YwIf!mZGoO?Mo+Oauy%Rf_-ZcQGi6}0dcU}TJD=}UXG>#fCe5j_ zOY%B#Q?>;eYpJVAwz{J0+Tq-*!y)SsRNc%0GV(lCrq+?lTtx9D`cu^1@K)UoAovjR za)9bfoWdJOCaA_*rw3djO<{8O6Uu7)77yMXs&%@z3_-P`I880Fso;QQd(YDwc>_SK zgp?*BDy3imXt!7MDi3XaUe_4X%L?faB)?gm}YG@`f(O@ zM@8sTvKnccN%EZ+aG&8F-*@$lkRp+EtF98AZ_;c@iGKVDAAIbCrS{R#7wn zz=#x4UEz_lrD-Hiqk({rUIlAO-5a7BEuC7`EZeGPN?c%-`a-Hgfb5exoXJ=;^G@du zBD0oPBhLfJyjWd088K8x%}J-ZV(39*51Z@xM$o(wj0ve)bn*a1K`m^mQ$!2d%H9lm z>N7ph4aOvA3(ixu4&Y#w*a!$uIZSnX}$uHyL?tXLXinTWXO|K(o zoYc@mTg0+!xhz?GnHu!ez6^<7Dy5x&)XCsHO3~V5kwH6Sv|}I%IBN~f6$HqfO7K{b z=+DLil^P2i*eCsnh0w${dFGwVDtq9xJe1l6P76)IYWRc=sthJQB1v@^W`SwZ>MY<= zRDlYhoklS(3?Q|c1#68I0oJ#xS}IuVtR4^*>eZSXo9Z-&{e{~HS8)AW8)Bt%s+n5O zdJz+~)i_V6n8w!5X3&8J;8$d;yj4p&OKnux>tq{F(?TNP+j0`3P zmwlXv9w1gVG$kNwjZY_2G^T8Ww6~j&SfIU{~rW7%sLo8ScDs78{om6X+(OaO& zHC)8YZ*+(DT2t%nwhU;cY71EJW}VBYi8zh2(O19&EFwr{@m@vySnAcRO-9YyK$<;d zN~&VsZaD4^)F@arYm<@)gCzEN7<)NQ4n*YiM#)-*CO$c(nIl?ydo_)XFioq-IDWw? zoiyf2~@KA^bqu(`cmKhCVVVH{3t*s>zH zgA?IKilM^hXA*A}-B7}^D?Vx2wp2f!c8nHK#fA@*WM8z;phN}^9O@?voO_=~VXE%W z0%^ZrfwW|muVw8!apnTVoZ5@9ug<3puHl_zW%qDY#1zsbhXHm*xTdWs?RX-mD^983 z^Js2Z^=nBJEp3`nR0Ext`n8+QhwJs_hxN_X?bSjStNRbHysJ|aM{_n7K|rDmtmaG9 zS&&_a)XX#jB@Q(qs1V+OtG@M_jX*8pYY#z#A#mDOXJ-$+Pg_BaRR<<#r1P92rpuY@xjRI&%A!fbGfXad6^_piSiaIzLw+QFe)zWP3XHIwnL0v$y@?C3}uRYX& z+7Xn}idB>Y5CwU>SDy!@^IWqi$MGUHTi26S+TR1erlOE7PC{yIRj}2eDcp@&6@F8e zd61gfS3T%0uH$?xkNM$a5UTUp-N_I(UZ9l3`I*ZA;|_p2uhk zbu{y7(W=dX=WzN6!1Yp6#o8QuuWlM@YGSRa;Ur7SPQ1=2oTWKxoliTkGRM){s~-jD zvsjfmBDv^O$Vdma3E9VQ33ao6TwUHgJ**c$Z*H+xi}z30Gxj}?!)E;u(`m_B>uJgX zVUEfQaSj-z@`Bniirk(B)Q+a0b_^(ut;1==tO6lLEqPqAdv|s+FNatJKpsU^tD*D$Su-QQh0U7`_QxPy!-awZ-GX?LeT+y3PrSNFHLE?#_ecX{>w>do~U3r^z} z+QV`^^MpPMlABHYgogf{`1}H?fSp}^*`9I@7@yn>ub&3 zk(wwhTb=!vYwefr8{+C*Ybw+-4x9s*AszqLS(~`n-yOzdy1)NTxnb8gi?HMl_3~Gi zI3q!holBhKvh;;$H{#OP%E7FMd7mNrw_M==>*Lkk(}ndce}!T+ZkhA99>VufjN1qO zH7GU$iAikH`p-ygPbYcg;&x}x`^XouP!ya9;lsgYfIl!^N0l6m7M21*T<7ijXBN#y;-T9 z7Pm#p4@*)d@~|LU(8!zHmF`Md0{H5ygoz(sUsc4&GpEe^yt{wX!2C~GyB)9YPggj} z`px3g*cFy;;Kh0Ar7t$8;cS`6wM~&`IK%kkTYdMMQmD=5_Tl5^ao6~~WvRYWCg+%? z#&LDIw=$A^o_1Y_Or=mA6|moaxedp8ocP_Vi!Hu8HtYZ9($}#XQ5lg$^ENT|dai%d zi~Bz{%eGtMj0MKKxcn2o|I_N1#h-ZGGPedMUdgQ(OPTr-=Me=G;w1U_bF1G7f_+{e za}j$~5*3#@cTX53hd+jTk(Z1ph_H9LmX!JUi`ZK@d+Z{|o_NfDnIcUY9MU`5Tl1%r zuX-7x@#{kcvWVlBJ8}1N2U#-ZX~`5(AW!GI$Q-@LkxLx7@7iu(%&!IGIA)-L%K6fs z;nMC>EPQpDC3H`GQY_0zJ>WW*;Y3OYv7J9BQuANki^q>wx1YAY?7o$!o`v^m8KO*s zW!_8T^Jv&yqhVtfIV@X8@B3BOdGFLWK_X+C&MD^KN2LC3M!`Ox<~(kFQ|O)=$$rVb zhu9C_-aeEw5ZmJ+ez#uVJ*?NCF2DR>@sl82j~*gHLe>qud6|9h;NOTY6$LJH5B_nM zN}<``V%7ePRoiLJzl7zR)m!{%{cwq&UXKjZVJWVA#k~lyu=L&FBdVVy3eOAab-5S_@0|-Ge8|BVqP^Fnu^36V;NC-_b??{k<{sD8deR ze2Y@?{bI{fa$hlR%-FBD;ukKgXn`WK%!l3gr%FY;b}>|L!mMpt+CzO7AG^TFp|4(0OYH*oL9AsO?bvbK%WT`HH{+H$ zOE;${mu2b;<0Xxe!H9`{EkjDbZ49@h-Y;t6VJtllr@yCa>XwOmj&mGwVLNqtKhK5l zqZ&Rw@8Z_Fy*_*9`l@A8#&9k}JkOrZlXpF5T`n(N8n@c9N8m)OY23HkJR<3G6wdt1 z{bgKOqV?_L<;PzI_jIG2JaV18Ntl8|?YZx=9#KmP?1g8`vZ7RURsYNL8W|-o{J8b( zn*=A$OHRHIO^={Xixl$bd}*&j7vBw>j9cuM^#1of4PJ6Dr1m2OBQwQxo->&%PgYRf z9g8Mm;h zN2WXQlKQkS@;AZIo`nrAr}M&2(5=}jTt7qVovPPv8G07kC)MZfQz!DHWNajz1qAmO zkZ-P3Pwsl94aeq3i|f_>hxJ0nEx7cRJ$+bm=<~uFLAi30L}m@hKEz6w`%lj+wrdsi zHG1vmIPDUTyv|#eSFb6BhTe4_Gaf{0UwS_1d!*RDy6*S{T=3)MVYPtObKFB%=1?Zi z3<;4>=NaJrt@svWtn4NFkkw<@!v7@%0OPucx^vAgU`wI?dEo;gc^&SdeqzV zQu?eca~gxny$kWY%2V>`mqVW8S<+bF7<-%yxj#u-|=pNGR1 zt~?i2c9=ZC%1h~nVoXcUeSMBHD%)!H#Lm6S-EZ#p>2}GLaqCJwYTikd+uV~*B8!|_HZtwhzd1#AJ|kzG$unR_Mz>QOrVVB8Y3_4G3%a|rDrhE z`(^IqP8L7vo^IXs=X-kMOo`~y#)adB8@a%t=U{uCAfwJ3o0mMoD;yWL&u3EfOd|(m ztdpdr#GcBh*2+Rpq?R#l>6HCBP*0jWXeZB$pRN~o*G~_hTbQ`SdE7#i@3BAw)b35B zM;vg62Re`OPf4U-T;aE?+s)T|oYJ^e?(wpcIat72VCjBKEorDD~ge8U}x0P%GfHK>{joLOX-s)5rDK1CW`xd$>m%=*?0Mb zm()y@jHg0lv5C6Rbu3Hjb*`ilTJY2gZ|RvQLYNJyXPIdXJ39DsmJ^#ww0l+UfSxr!v)e80;$`X4{6vP`@4GQV zmB=H(_U9?GeIv4wm-p7R=WDe1ma4&PnbsI3NKR+PgFSVpzcx0;Gw_mcHzHqAY4`nW zJg&77OQds6bf%iO3nKk>(-&}&;3do*_HOo)YGu! z=$D-O%;ZPZOwWf0#f=!!CyM^upPE+!@>@}4%$n0O^=LtIUb;SJh95rFlv5RpUX}gHuBnr2o0h$z zx4tZ;S8j^NxE-}<^yjm^PkQVQ!$WQvsmBGyEyeYi*NqYTbRzyt%$UTFeG#R%XgDJl zIhOXo=J!yH2YSn8l%#7F1MdMi^sg<)S;Ar0lQ4{iRkSqm+kYUdA2{4yh1(>XDhEQ9R1x zjC3nHmr7oh4ZK;uyL!4&5I$y&(=z$oGhv|I_r5_A8j)dz6oqQ>rK>?WaFFt`LeeE{CS}!j`4qgw8t#B&Jgvw%BEiX=y4~RN;v0WdzYw6bwJQiGf z48tieL+P!ojOa`=@!5EE+N9vi%lnsyQNDJOmeMm@0;HaOPaT{`xsMW^m8Yg%-fuS_ z*Z1p7eD2Na_T6IjEPUV^SGzGebqLFteNPipnK#r;%ZW0=cT`+U{$!3*T$Lx~ADbon z|J(c0HMflTNfIs)Km%a0=x%@C zyu<#!+jR9dF2k*zvvgjEAs=OQqTs zHSDgsB_yNY>i6{rQfSpQy9YhP9V=J(_Lx}mWwL3twtDP@eBp<;QQLLxnQbaF8|LT? zZxzP_&22u6m4O{4R=4fD4}5*g)rStqVfG%JwY-3Pi%i+LpN!+ffR?E^hM$fA)vr6_swWFJ#U6S@c z^1o+upP(&n`L0yIUPQmdq3uP(&>8SHJjd->nO^4F1MaTf&CKlIe2?9v9kA1aBx9nxZRO2`-n8KqQdq>`XIWJlQzT8qL&|E}WR@ zxaJ^Tb;}?d7t^h~mhQHtq^5Ogrd5FK->D_$w$d>Y+ivAC?rDtPea%ts#VFWrZJeSt zNo%R1%!sO2L@H{>pdI@$^jX@jWFgr|v=MrDpg3{mQN(8G+aR?vK!1<^`|I834xCx{ zxS8xGYf$AV7%ez%+6(2K6M-O_N)O6S^>GE_LPtz=0;f@{a~g39ef5#2(BF1e|7F2F z`h(XLUH!#gV`6&-9k8Ua06m;7z4FkHJ@Wd%E9!q(ZG@NNarSmscjejEw)go@+Yp|D z+Dx=hl$0IXjP|hjtldtMliW>#bXt?>SnM^nWM|NHeVEYyTYE8eB9M4wrL?<>i+C?u zu{VWQn?RI!MqoB^MeEgq6V{TcayGBp?cB}@e>GLFxTcrX990PR9X5w?Gi7tfc+y>P?*Xr={e}@Y)y0Id zY6{{6NfemMaj)d`YZ$Ar$LXhb>!<%WP|SaUP36OBd+s@)2uHLWZ!))GMKgcI-HIwD zcEzr#EB3ERSFl3S6}!s1swNd052o}+B}%{Qxsv2H;`c$Vvj;8Z0skgxS$3BS^0uy3 zJ=M^F>D&6c6GC&{E~T8j0pEO0;+kHTGr6=|Ut8@fvsz`jgepoyNg^Be&Zy99sMYNC ztyUEU5RNyAv95=Dh!=xrR=_cId z?0er!P-?G%H#^Q%ox#n<;ftc0U}##aQ%_dwRFVUwh*P`oGCW_q-&T$Ce1lt+(u>OC zrKKACh?gktpQ#42d26$-fqXZLeXs2~fa-V7$g!mz5!qpC{g37-!I}MmatE53&U>cS z(dx3s?`3c78hhK0n~yTVn#j9(rWYrg~kMa=VrOb6LRZ_y0XK-Yko53B1}MhHmO>cO+MNhh`CBWNOplx|o9#n$yfjoSoZ1kh|L{>Tb7fwf<#w!L8Tc!sx9UXG~EuxEMB#$RQl$Mnd@FKd!(Y- zO>&U7?nCaXBn7TEdIc_p!iyh8il(cB>JrNj*VhbMyx84u?QTnYE#z9=l4f;dzt+|s za{^J#Zg)wAyEDUX>AOrKELAoGt+uAQE-Pnrd*}lylC-bYAl@&b6?<(*8%XOfL)V{= z+QX?62ESC*gQz8Yi7nYnY`Lx#t>^396au$R+`6*wHz}w&naiH6NnI7$+f~vuk-M~( z4sut&T<$6xGPQTr9#L4lwhox**%hj;ZsFjB0YvqbRM*=*+(U-iJWSynB2wSiIejm) z23xSLE|zWYutFi!ZT-J{!+uO1fF7wXOdUoI*k$c0_0)??-(ylYd+Ux50!*SbJ_I(8 z43^a??V%P4Cam`L{$8pHOl42iC>!`;$`Q`sX17e)sUuT%Hr|qr-TA#T;T}gq&b^wK zWUrw$`&P7O-*^+(N%WF@z!?Yy?x-zal_;*vye~x)c71chw4FO497mFk&?;YFO85)M zk@^U*KJ0$`pBsPU_UEf1)*L0#GH(j4vaY$y@3=>*x429AH`K)& z=z?8~n~ysk&8oRBHk4nMR#i=_<;!VpwW?yTbvY~OC8fNzPU_cfl+F+?SLnpEUDDAW zAbh|gc8MRgi`iEgt{_)cRVU@4(Q2MmsYP1*>9Eg3#Ad@5<*iC$ED z1fdq`TcM*%l8HeXb-=T$*V-gguSPPr=i%H=G^P1a)62+aG2MC{+vt^Z<+FFX=zg=L z)vY6Gb=Q%@a{zu!BML%_ACXm%et^hB&b?@d7HdZV$KZVxcD*PM*6Oz z+i3Cewe#1$^F96;7X3`}A2`Paob=pkbumY{l`z+lmsa$Yv$8v!6=YsWE~gzd=Uu zNTb!O?!A>40DI4Q?1)&5l2DaO8NRn~Oet3=FiWF8votC(%W8SkcEKrUY1&{vF096^ zvkr56SCShU7-g3wN~zd2JHsYVDvwaJ^ZU}?>t!(8rVi=qHqC3Jw`gZSZ_!;Fk=!2< zth8=DSnaw&g8q{`q`mqtyG1IY*xlL`ySvf%7hm!`TlfH1C9BkYi@)Wza!#9>R*85Q z4mnq#vp9rB#Y?GY)nhR?qOH`mHVJu!$aGUNw7Q|`CNs1b?DA$i4er#4|CJ=TQ_36B zJF-2SM;MD_hqo0`1D}<7 z0zzvowFbF*ht2bXXWD#&8qldH{daa@#h$mK=;FWmUUX*#u5Iezy12L6U3IUNw97K5 zlvPmZ-9b$d=-n4(hW$!q*>@O%%t{^CkgIZCR}iggF?DNuSM{Z|T^)|?VeE&?2X$E` z5puadH$Bg;*11Nd^qkxJ{I~0tUfHkHXzlCPX>Aj}j9S9hng!AIC(aN~mKSr^btc>! z^0w{rV2W9nc*Ud^7uev(T!Wk-F%Kg@#2});k9eq5rYa5CpS8Mm-6dMxGV}7#4r3lUb{ixu zrD`C#Y{M?SlK9;bus5$AEr6|qVm`K`ijvv&`)@F6uGHsglVsA{M>6SE?tO2w!lsed z6a}NEnqcfgirNxuw^U-4!Eb5Eo1x5IAc%@(VKP3;s>t=z*fxXxE4JSW`LR84xl+$L z^fk6&R@J_%6Orxx7evf%4ID}2%r4ngHmhTL9d%6K;yH@F1@(GkXK7}?bwgDY@!7y# z!bG#23Ye9mvdDHJd$_pFQYk3L28nZn)z8|;&SzEbT>TNk9QdcD3$TWni?4W_;s1S=Z$FWN6!T`e2b z$TgOm7%yh_5Z>_!xujd!m~okZxLk6yYDtb;h0q#=ZButSl4jJQWkyx`t#<5!QSd(8 zHB4tO8MIR_B9==9d&Di25uLV6*=^Ij6?&(tFZ&zJvtcA@E4Caj7YOTK(b;dH;TC)0 zrZXXo!nIC@um@My+7-2HXnU&eENi<-dg|2{se7d!8*8*k--8xy^LZHA%R$I(&@bG1 z#f^O>o3IyYlSfFwyhG^2#A!)C+elZZ7TM}lwI3;JPc!PH3ngAnTW+Q{-My$ycLTLq z!Nj_TwO*NFEt4gUI?|u9rJq`B7QK}(7?X2zAcWJUSg%-0*n>80^MUIqb`;l4J=U2^kYo6;rs?KN&{^oQNl=<7E%BR-^L zZS7zAodP1bQFVkr^E;M*zMQpmYq6H@j%$?897J^dcc(ajX zKO@RIT0}7q#&fTYuVe_S!OyRu4YMk^`*eWN)dL0)xDs#_8LX+;;l%f9u~(B_WNk`t zBBA*BMOCX{LiAtd+LUx1bIdt5z&uqvbY>ty*%cvj11*{P0&`TYv@98BRPnftw5tC0 z=048$X2nRRx|l5DmaH=VngD>dAyn=(4D_igjEl_+**j=ee$y?xAG&<3__M0XqF=(} z?1$O9XHz!ckl*8EE5hLwJkGMm%W&8khJNIaV^>fTF2YJsV3j0+zMjc6nJ(^C+|{io}^!z$t`QB$dSicKyS} zkL-Fr7}_zP_`oNB1Vm#^G1UWT?g3QdBgSXQG@wG9Gy4h3_UxI z5S3zxZwI;lU)bTxshyO=N(yP&nY}Fz=Nv||7q$~C!>Qu}o$z4L=g&sY9iwx3-thbw z#d<(!K-zoyqYQnQ4ea49C@6DpgxMH5N^npP;9uJE>7WR+5ji%ykR*?1cJwy#rz)L&e|rUFNJFkybrkqiCjbD zdDV@7))s?3f7S+272vfB=>#sFB3P%&Pew-#?tfH`+^dCY&m|$+B%I?0|r? z&Jqx#gU=Y#MlKb59{UV5Yu|MTcBo8&PzWT5%W(mBWiXFp-(&GIfcc9LfW)C=qtG5j zd<4_7-H11mNdmj#06d%%i)Og5f$gpQICduUkeB}!E$nzWWpF0rCBiJXA%MYt`1B8| z+Os;iA8l-&ysRjR&)Q z*N>6hrzVbLB=yoO@Ca$Wj8{5BBEL1U#W50n>7#XwLVk8-c!ZRHbFz{nWCWSme7ts_ zBnIdRIYG&dz!5S9>12HLOu;&qvs}&*R@|VDlrhM-ljCI!WvN<^kTc{R?~j)^Y|7kx zj9SBr$MJD$4{5yG5poNe(((vdLm>?42pPk+Ea=B*JX(oSIcD~tcV8T({V11v=NOqm z>fC&+Mx-O8|G9vuV>B3vgagM*{mb*ZA20deQv2=*8Ny0Zx+CNXm8u~gDX&O#PaLV% z@afwGHmD5c6~B&N607Ax^BnF1GW~CI$r0d zB&P6q^@QGK) zk32r2q=fD9J3bx%)wd{Vca&Ec(qrg|H474K&om9?r{hhMk@37^w>p)j-akSHQml%5 z#P=7)TJrxD*u^n>$@=E8UUwsxJ5C>{^>Tdw6>cjEW`mm}<``Ld$K%yA%BznbZv=L+ zr1vozqO$s)#~5{&R);)Br6MapaD>d_jcZeh?<@O`htBZ!2w6q4gzKR{kB^Wcl$QX? z7Lreh`K3Y?L;R;WpaS30C&h_eI6u$7cs32&_*Z8REN{l+sXv0DC(8z)Y5L(=d#b}z z!Sio0N2?v95j&o8_P4q1Vm*0(EE0)&KIRP_hyIL(90ZD?3$%ZTZ+wsH5MpBCh&d4( zPHora-UNFZ%{hyG7QnCEi&+plclL0JZBQ(6yf|vHo9R4ivIS@M$PQvYqT#=X;a*Pd z7)G>bRLzSwRILasQOzS_NZ@Q1OqY=}bO3oUw{#TSUi^f98Y7|r9AN#}P5D9= zJmk#rMh;D2?nF~c*p<#*Q^ihYZCGT_0+$2KFEG(~#CmF>$1_uvkbyUmGn=~r#?g}| z?MSkg9K`Tz%;uXP0*{ygfYJXWvxh(qamUX9E4(3R=gd<7*kl3w(Rna~2-e31 zF@li@m9R0MX-5Qs($?O>w~^%iDz3-C#o1!vVCL_EBMZg7_YX@_vc*c}pn z0X!!53#Sf{Sxg-Ogd7_|mBC~l;Vl2vbDdkxrgM-`l5Glsc*&V##SNG63j||;J-~lv z$F#vt6wNt28O>n?=nr5qBm&7}+kvKz0M&vrKNA!s(qZSY>Zu(@Q$ztcMq+%#b07B} zPa`0k3oFD;77!Ct{B=H>;CDn`L!>ia?!hRYkCu#T3`@e$gzZGu4|A{qnlVrhz;q4B z#Bj=ow=m?3B`y{QOb!eVfuuGqmGjX2@!Z9AUn2B&8F;Da>(AJO;O; zNPZ=TWy^U~`!U>SVu+@Ci>O#^`dOSyY6Rb?K!=Z*F+)U1un=_WhTul{Pi z8NI%mw0@29O#spTFg|aL_}HGiaRZPBlZx`N*qKPT-g@~jSA2f`(x6TNmO++vJv)g> zj(4s3QT(-wQ>o*kMIpq*7%bB>#WHx|MUl=|5f&e ziLThiz3;Wycm9Ho>C&G!*);nAl!oxaAKA;Vvn=>K=lP?M4|Y!!rv~JOO^~d^$R2QC zciED&1PmKsn)NZ~6de!y7y~W=kG&;Yh=;dHf04riU9^`Gbhl{I+;8DstHz|O0{4e$ zlljvWkP#3}BIQ$Vhs5Js1n~vDqer6rd$FL2FDTSOVEUfB!13oWg&ES9&@YaS0s$Pd z2&oVp6mb>@J{(wh9F`ElSq+J$jnXw@-un!w&9skrJ|4qyoCjG?F>zb?Suh-DCU>aS zI>cUF&2X9chlHyL<*caifz3TQzJM(;5-+heAk@ZFCraPo%0ek%;EeIW6Os22&6p7YttI`v1R7@ppJ1{wwK-~C3y3=LVd>jVj&Q{p z;hH0(6S8J{kYbKv9w(k57>A4U4#5+rh$sQLAruaxID#G>jF%KpufP+4r^FJQo4Cmm zv60qAc$PrHcU@cs$Tgn$OJol*JPHmC^NWOh5`v;a=99<+C#xxlGf`$_%Q$180f$D; z7SEMT$aQq3?kV7CWNioetm`PE# z2<1c*pZLVQhEv~nrBF)}?Sk{$gA%NQSUPh+Nk@=~dDyrMr~=?DfbmNGLA=sAgGs~5 z59hPlQmo>mh%_WW3rS{`1qj;fp^R%Em<8mRF35v(LN&n;irTLYB4Hf6fEvj=F+YDE z9Y!RPK)3^C=wK3uL?<+Z-D2fSlpoR&53wvXjWhw_{85X&&#}8?wIN=7!x@i&is99V zH{l26{0X5Uai6RQ2UzrX+#V4#IOIyE$siwynSU=0J)!t3HD95NW+HUt<4qZzAKK^z z6{%wrVA8-ZQPju3s4RjM3XEU`x^RoV1c8)T|4_V1G#%Lq?$cpeyx?JzIgg^0);2S$ zI}vtJTwJoEkPqpaB9#Vapv97mJi`-VOpDHOZI2X z7qTC^z1?|q3IpTSt3&M$W)WDBNO&pjJ#w=E&8K7|i_4L3F+~=P_ANXWjyJVk)WH%_ zjLw5I14`#{d2Q2-;JwCV?LC4>;kVM z4ZUbG%dua%l_)p-nKq2JtFT%r?npa8ba*ZPm7h_O6to$(B4@x znPGCb1Ptsjy2wsRgdVM%i z_AXPHxy$d!4F&OCh>qw5M%@N;wDn+broLG;5OU8LB#71_cIGml=5Qt`@r^EkLs;S6>S7SQjuI;HzT^hqI)DJPso z($yd{f8Pno@(#)Fg&;+fI@Ce1-l-L69#b}$4rNy3MkfJZa6{C}#8*RkZH5x@~lp|Z?)f}o#Z%GeF zFAy;^aS+mr4nfBJF|->CUoh-;ELsThF(;1G2xHi9YCCr44&J_l5-u1wv>Ao-S#%JD ze@pi6ugUMSDQ*Js5)d!&X6Ww*;w2hR53tqOiDuQzDOm^~yi^g_glLW*^AI;2FY24( zx?Z9LfY!BVGdqFqCdPu7ltR-W`yQTeu$7w4Ln3$WtrNnsd$vpiLwJ3&?WmoRyxu1U~L^7EA#+BTlC!z4^=z zEcJp-xaVRb+L3&kc!0Zd0?8>BB^4UhoueEF-QJL$qnX${Mw|!oPFbjqwC$M@xS4A% zJcig12zvOz8HqF}9b&I?tQvjPQqL%x7&|d*K-H!^)+i5l$WoFz04ZUKuukt04t6h^-64Qh=JP0(BhYc-z*h()InrH0Db2^*gam;-J`XWO z11(?fQfR-N&zk8wBKN=ra%2eYPQqv14-psu_Cr8WdQ2?@1clWOqc9NzgbOoWfz9~g zXo};zNMKP_OW|n$faxNXbQ6wU38ch~3GnxvSHhs&&xhJiZAh`o+!uXCTQF6k$ z%gK3;Y&t}>BV-B-bSsD70TwpwVF0{0ibJ3z*c$KOUjFdz@>TLFH48ePuoe~XA`pfJ z2QGAy!zt2oaP8?^F(5h7Vc&Hvfz|CHLYB&V`=H-F^a(_edl5g3p&uy6JdDsioL23K z1q)-9*q2cFWCjgLpm_-1wSnZhh~`(uxlMHafjY+HOa; zIgc3WfsXIwY^0I0J- z-#|S2xB!|Vx($;LLo+YVk;g3H4n~hT!?`Q{yoL1G%)OU{$R8idCY|~%rPkHVrWLIj z@q{~9Vsg!Y5?500GH>=d$WEXIuHev0f!Ju&LGd*2QyvH@F>@p*YMi?l&zy6}!v%%K z2u=wIxgiFgkQz|%<~hI~ycMP^X~9avTynl(;(jRgbd~NnD3;{j3U*HygtR`}%alU8 zGHi;A!-K9RHN?4PYEnj(6AGFL&LGGv$V8lY`M^qfLcoU@8%73nk54Gh4JCcSJxL-W z&Y5GH9$hvch|Nub^Da=dA@SyN)y9Maf1O;HCyw*PmIip)Ls5DBj*q@PgiubNc>*hV zj^(2PErlFir(}lXARsSbU?T(4acp+yU^0+-fL`Jlwg(^SCO5+-1$HBnA}2K&`ZE{} zV-ZK#d>-Xh5j6avY*moJ4k>b`TB~qaghfnBUnkZz*7ncj7K${ zHAxtRwPb3<@JWKUeB2R{>u%u!4k?WgehYVH&6TS&>5oL0Bi=-Ty_p?kU#>D!LG#dd zgV!QIZ8mO=acj+C^+;a)Loj6O;w#uis1qs26P9buOjK) z(K@&*#qzJAb(*+kw zLCIY`?iWH$-f2t2+nDS;{#c~lAYtz?*XeoBvgW2>S|PR#D7I1xQBzD#LUI5p_Q`2W z->$`}&FZt9I)<~9c#bmkl_1ITlkNJJuqqLnQ1 znjmA>!a;gzqZKTFd6@~52e*c8JrLwZmRv*?oTo%kKOXR=yk-}dMZy)HGOFa(lY{}G z8!1xIgZYksR}z?)cUw#1H`sT!1Hua4(=^i^Aq^OK%hZn#bS;$y9EM+MnNT$vlFpuBoc|Fr>bzE zoN*JW3b`1;iSp(LvbnE1l{G)ejmB#!eu}e@i;AAvj+X^RhjdLOm}V-W799S70FImp zwv4h8A@NR>;~4L26O5G`Sh&f+`1XT<^nK>XjzZNXh8mJ#FO4@mged)kyh2+I%9Tr( zUaYy>L87+K3}n4KqSo+cU=Q473MEforvPa~L2}{4IVt)Mpp0@bUY9uj=S23)Re>F~ z;DLiAj69Q)(AlpK!T|*_$`6^%2J`YE)gZ!v_tu#VM@owj+lY3c?B-*LLzLRbK86kK zTVEu^n9ysBT}I-LCyPJ6Ol?l`Oo>d6B0)rW2pC@qF-oSqLwO6pCfpfe68OT?V7VsU zVBn=voa8=$Nt64pkj8ST4z!%|E#O@gAh10`Sz8_q@_rS8DEDzn7%=}=VvmQjApMF( z%rJS8%|NZE)N*vqdU$ycvFI>&8E7MjMIt8q&UZ#4n;5Yj(8*;AM?{UNT)9qIT@YgG zkC;UPYK4g-`Fo5C6&^G;N`hm|D=5P44xb4qcRuAUGGaph-t|zS=<9}!Qb%?{VF4h~ zVPm;AQc_7-?2t}0Grs$@~0y_ZR4tVFG49*^=YQ)=2 z+leW&m$<@dW|uj)=;xTE-oSXvFLwpJiYQTbuYpgr{8{ z^isLNHnuDh1llAi9h2ZHs(hWkuq`MhfF0bNVY%! z{Ck5%QFz|KeEJBo^M>o&qXzSXILm44*)uLDEnbQ_b;WmbF_@D%c^FfU!vHpNH)ROI z`6%QbjP?B4$hk}M@B=C;AlhJ9xXbV94`$wHxkUT}8-C@Oz8L?&)6UEem&IL z*mE+{gud$z>`t1Mc!A!iSaY(IMT5Q1MB(^>^H2!E=w@(V;H>Cs5)85+oz$Hz|-q*

Q#ie;259%MLlY!8S{ zGcrli&X)QvcF$*#&Ft9I?|3L-{P%wnmqABAN@i=A)oPI;A%fItnOpNacR_B;eWf<+ z${M@~(3Xawhjc@%uThqfVOXsk3ED#h*xgj{Ey>fRvuY@=4@es#2MT3fWOr24$m{~?P!C%X!@hJh=0jCm>a2XR83~nO%J~V=&o!Ktn>1d?@TCn823976TcqeRV>NH z<+lc+T7w&ZG4kT7W9@|K6c5r+t`tB*u?9oSy-|=Sj|0_g^OISHz#dJFtV`@C321Wqg4J|-j$2p(#kJ`VL|A%Ox%WzV;$J#ND z(~;a9=(F{8I4&fI{xyen% z=1$e8{oq|X(cUr*MQcE*QI34Ky52xCgu9(Jn**dHRh{qhCe!d@XtP$Z)P(GC0@BT) zyDqx~3n|iT&JM|IfPKQAez2!W*I(New-GVA!Se+Ka+n&&Dp`a1^=61G zAEm{0o2_^~w%HN;yP<~66Z~*)CriUT`+dA=-804I>3iMp1Dh#|_aq|W>zOP7%95jP z#wtV|sx>>)5JvNsp}U$4{oyO>HQrLs|GX%(Uo_AOM8{)GW}Tpkpxr?>)ZY^GVb$=C zLgOtG?^Y1SmX2p)?~dQc#FrWc#YrA~vtbvl0@NEL`pros!k_lk6#Y?d+}^xG)1Vke z*BC(Nh1Y)PscQYBavFDS$qr!`ex_^kPM(~wBg`lT=>tH+HCW^v0|le))Lx+NeQR^83?uYVLTHI!XauW-6S{$Ba#%CRVoc#Iha z`h|JqWSs2bKEBg%s@F^{8j96KdZweU{>(E}T&1#X^c}v0t+Rm z`cp^A!0p?Fkc#Q)5An8??l9BeQtABrjOEKLNY%1lS|+S5$oHAB<`vWdu9%;+C{!h7 zt|=S6gk_@bqKoyY@#UOAEzt(o-!n{VS~B_4FY7PLc%vN9ZqTDr#cM3d$UAhq{&cR~ zSn)|g=ZGhN7WT`VtUxzojod?Gb%dTb#K$z7CChqvMoW;UTETYH^4n%i=wf4?Py<8> z!JwGGJE#lGNS|UdasP~5KMHoMw&TK?oJ}y>XlAI?6+7Cb^2`c+s--Hr8WOGS^MOs(x4fE1ynV#<(xUWU5|n(Y*An z#j`J`WG!M>O6$0IN`FnpLoHg=5bn;;yV+-B$tHwTiis@}ox*cPb_1C^*NTOpMak04 zUl-GIVQg)fk$I_dV$a5pnF8reWl@635UmE{YX3xg}yKL&$JY)g=j=AAYrvhJp*Dh3HrvB?*?)mRf}SHdrJ$m)FmG z(6g{vXCvAP%#p`bz-?s> zWr38n^1K*H=>!gABJ0`aKG~d1StYR@qh|VDsT~^Tu@{Fy3Sd-S?>UPP2eAbbzPaPi zP~3Fa06A2Qgd|kOYlFZxlGumH+j3?up4`(-nBznO$Gr)H(XvHnd9oQ*DfRP;#p_@D z6Xvg(ny`&xWcoq>p8$?FamjO-28g02B_R;v-RBb4?yJu0V7-=`f)61xAv%CqI=qxm zSV9TD_q=x=?2dG_XBy#jO_pv`t&a!TWU+;tQr#uJ^Qr%M&o$Z<+sk?u0@BJdXpnu0 zE#LLqvk>H{Bp8d6mJ#}`TYKkcq3>?d!*%=^30;jHzC)MOBH;|P&w&Y z0nFM2M0T+dP88}Ax#CIYbBW3{!FT3l4d+h^g`AJK*eYlu6N-knK`63=UrnZKXBV@yqe}Tm^D(JYhkCwqS{iD%I-~ap7f=G(|NrBYK3GMO35&CMkxpemkuRo zTT;P0jB{BUz@-5634jGw>je-Sf-rk-&u0Qj@vYjL51NH#nMcS1d|)0?yOpZ1-IPIj zXOn%^U2~l22rhB?8vcEqzt3JPBFr3dnpozrc(oD)c~)BcRHOUY6K4%iD<8LXR}^dw z3kpdm6ku*)GHeN<3}s}Hi7UA?I_xKzM$L*l0C0}QuE2mc&SNb}4JFY?JO|E)XX*nz z%15BUu9ioMJB@MLZ|!D6A`Fs2mrIt0*g!sf!NWSVn@5cjZT(yFWxbm zEdi38hBQb&My0DTBa0-^prM;~O9%BY>h0ceLLCrKU$^nG(qgL8lG7zZ$MR zLx80LE{sQV%iw$d|F`#LOKv1bg75tmL1S(FWq?yNgoH1Klv)K>Wx3%4ewRzR2^p~uAL}ns!1p)jJcZ%@ZBUulFDCCo5Q>H1}xVb((3svWup>FanKb#yDf z(k&NnSe-*nT^DA%)Yoyff%V0mhF7)a>V08*+p6b)>PBkW<)fIPO?44w!e~pIbCkqXHstU&3bOfT2oC%}Tac|$3}h9CH``n+pp<%T81dd}IxXyVxn_6?Y72UM3t#MEhl8*HW@}Y&;aAi4jsB15l~l9!CN`LQ(QuY z(^1reZ*kFupF?9Dd!a7lB(=FN_{cRtaUh(VtJZP3Gj#Bbp!wn}Q|1V$2sylBi0y!9eH5Y1{%EAVR)b>eNxe$-0vZ6&zbNe`FW zp|0LuX#2j#;P$8qc0<;t1m9n}ci%X&gxZ=C4%GLW9`03)Yp?_}uW=~1<=mTa?rd)o z*4?wHm{=>OsY9~MHPNAg&2>5bJ(FZs4t06!72Rob(!@z-hHasV@TZ=w^O_hT+vD78 zmFhj0TvErh>KsSTb~TrPi9L-ire=dWC~S-BlAJGzvzVF((uPFFM9q4rpd=UdpRTqVd&aL#Q_ zp)EYf5$e`%-(-`L(xBz-Td0{g)5o{sat2(uUR?_V{l2s-T!RAEzI#tH3g08;9BTlW zZdnRyYfjYqHV<_1gC-?;D0Thzc-E~cXrs8FOh=er)(qQ%u6K`uDejA1k9qh^P~Q4j z_NT2i9ci8!k{z8REx;7fx-=+(Z{SH2q3^a>-j^qBU4XDvjH02*8EYQeuEVuk*bRiO zN;2!2nUFfAKXSW z=f3TR>m@vz=R(KpVzZ+KAUZ%}F?kjk?;YV9UGuU+0WZ#1}vQxIaylX&;$$>23PR@lk2{>P$ z^F)A+Y~U>c&Ds?$UT=ignRY#$WTR-ZocMWOdRGBZ1aC%Emo>-VoP ziL929jS8tXl)e_X`#UM^o8ou9#x2ml#WeUesCvm+C~o(F^pC9O&SFvexqJv_IA^ZV z9?R1TZ0?!VC+GL!TCTc2fjOgL1r_d!!I*9*WtT2jT|#nzV>QN-DT?M zS6P-8gKR?fbhN{)e%c0{Q{1#%zeD9=#^23ZfoT{r(F*dt3GxPCGqXSwW(OL#Ckq5{ zrp{zGkldcboEw+k_-k~L-ISsCl+Yc6LUqAK!e&gPfiolX%B5l0uGY7Eo@dKIU=7Ye zm{VV;l{1qh{w(yo*6i8`o@MDQPEW05ViqmvdFzt}`(8XGYIu+a?6$Y5q_o^`3)jcVE zxm|rF;n>1o&)uG9yMbnXW+rE|gy)BCeWg9tSK2Y`cLUDB^zwyHCo?#lg|+Zl8$Y&F z%*mV-cX!8YNNv|$8%u<>n5aQUcj|@WzNlsQNo2dWXDG74W@>C9t|)g1)S|Vc4RVDt z$K8|)wk<)vivoTYoepCof9NK3V-a#!<0+%T<|W$qL(XZdfZt9M;J~W8+YQs0piZZZ zF@u<2&-dEO(|amoiZXiwNQg|33kz(pKAhWinwel%aIr}dJnqvCWwTnLy@T7^JdOy9 zpoL6#Hr`)fSR6Mue3OF2ENIWh-?D?O%p^*;%OuJ%y@f^*hb2KP#<_tW&b%=A5iQuu z1a!-}WJ?0@w#K`wRn?qexAmL(LNMO$T;GPT)U$@uG%`Cw+u*(xAqp-gInXTTMu)R} z&+*&Ee9xx#jf!dkz8j}?RurnXb!s^^$ccdmxkX!4>vo*sR)o1zcE_{JZLSsH2Ia`P z=d^B>y1%F>fSJo@OJC@OZ<^;L`GiGq4gE>xnHm{MrNaE*40-J9st35k( zTwQHjM;DYpJtO1_%35K4I~5RZ&$RLmpxV`>1!lsxL#kbvP0nWdK5Nf(?C;o}XL#%m zbZDPyf(3m6I=DtkJ4QRxzSo|N-{V<3ESqKQBXgT)Zk7=Ywz8{ZUT&gQ2jV_>N$z(v z==S-5H=v#iTe!M4ni?hZXS6M~vrD?rcoKC-(;|LfjHCR+d-|U31+^jdcI0Z#J<-M> ztX8z+$&Ksc86`H^fM1_Cvq7%FZy1{3mvh#x%C}Qp$J@iw>^rvWhr4mHJc0F5xpn-l5S+ ztVW{2)^NUoZFQAI*$A_>k233mf#tj>^it;8exTW(ay*l)a-7n3m9FsnTZyZkOy@1} zt1V)=KF-&Wdj;oyYdGI-X%%;8RC~T}yV{**16&6V2n5~+f8KVr!yO9X0Tnbtx|gkL zW>c!EpcDq&$YIhJP}(P#mm%I3-5tS{2e!vG8kys4-pcTq?|Wg(=eLscVq12my7QF* zhV+NJy^X-c_wDdD`Ni(ZFP5}$vvhMCaLgQx0rqq{xWVQmb#9t)uGPVp!-NcvUvb-C zUt|H(%(+ar6q_jDp%=EHf43CJzjY#hK)61aw=IMwEX-jh%9j)PZ3>gUZBA}e#c%_9 zLAUmG8EU9eb<)6V>ll8Tl}b0DKL~B^>s)09^^lw(+@yH7l}Zf|-;d*Q5yv08)fekR zRJq7ZFwN|cuRoQ;@p!!EP?i&#L>)9t?KUuQzOyd@Yi|H+dpMWlnxR{hM$#8{;An~@ zg`q(fNk+Nb8q~LZ2Fu+8wyEnhdH-hU7Bt}8ZP9%@o0q)4hH?(4n4vdWI zkkgi_ZzbCtQ#;g@S(NUMZ$^DvQ+7CXH;Zt}h3kp?ltc2q<(@N5eL!tpaYNuspRPxB z!;`Ap)M{iRYzK*R; zoGhN_i#C7kZB;VfHpY|%P2+hvW3U|voj1iZ3PJQ}tDfCfZWL;UWa?|G83#7=HGJ*4 z2Fo?`?vWP$iRVUHlTx7;=FWCm!7=N*zn;fEs@o*-4XWsYMlM7fMZVvZ3K*5NM=Z0@ zg9Ta>iRKG~cyi7h-o_{GFK?ZGSHuMCSSL@HF-KP;6Ggkx8IiEv4cX6b2-sgvXo6Oa z63)QB)hT7J*sPUtP{JLzwXSa`)m3#I#01gY4ycZOU5&^VRG1V*^xGoU`>ku6MrpQP zU0a6f>6k7#G+Up^U)Vl3F|`bfwNVx3Zr(QTg*M1>OAGlVAm4{3;RsAl^-okF`~FC7E;;_gtGY6ZZ;hy(|0VZ(2qfQ;V9U9;Tq3 zX^x5np6#tKdyILfjgnS|Z5gK9DOGQsMa5*RI`!x^@wAy40xwvP@W6zpsU!S0gU)^p z@Fh*n1lO*%c0G^+4f%RpyP-izbC2GKBpjc#{GMdz@njI^UEBe)hQnwSb-%(-!1tRT zzy<8^rxx{|TQAdMl23aR6Wj}J-D_Cd*C!jF%bTZ5R?D2$d0IyzmRKY^NOO7vdCs(j z*F0+bfkr=X?44>5O#{bk<@tUC%7;;Y9$U<%CKW&^)fQm$m>qglgJq-G!+^HiUpvN- zuVCghOSfZaYEz9n6$44vBieC76L$83Rz2H^ozIdyzRdCuMOQA@!Wu;|#H9DZO3EeL zI*p1-576Q}usL6U%#(X|xj(G;QyMCeO zHf1J=4*a(C-u{kk*PCJH;l-}@AdnD10c+Q3)^w-qg2%brx>NSxPAUKGkfO*>Ivvs* zCUH7B=?t@|-yzv}Qk-;3M5ocHf^SJ%Pk!P&qXIffvk9I(Dq?arjIZQ})yszD=e;bQL`e!p#c9<00HSA_ z_Hki}N!&kIAMj!_it_9E7U$bJkH!{LPoK`cM8;Q^AmeL`lb`de;qfR*t${@y^-I2= zXP4?%0;E=0J%A7lM_&1MT1=9&YwI+gT*h&#E@EF-ReR7fgMq<{n5F0S*kDpt0juR1 znZ#G{b<`C^UQ}l+;d>sJbFC+b@FuV09PW*xv=@`pYw`@(>paV^A9Ym+(9t+b=eikX zX{LJNR|T1lfmeD6KG5xYxXWc-0t%ZQ!sI52?2$T;9#|H!tjFEz%j^;vW&Lp}9gP{=K%SqMgZV z5<*0uPA8KrC6nthj7$9h+Ru{+qrZwG?w@p!Av#sx!}vS~5MiIB7V8IoIWKl-`>W=`E&1i_-t|F^Zl-vw`)867Uw+= z_~14s{>%|Bf1$^Cuwv#w2jT5vJV_G({geD0TDXOY`;I3#UxD#Pzbod=ZO^^MdH;Pf zO32^yTbZ!;)DQf{36~wNm?^XU;5IS*`Yk4dA#bwbt<3gC@GM-CWQ|T&&e-*CamK&8 z1Z{xZ0b%~ryA$5cURTapICmz&TyjvIOh3FkDZYLTE8*dUw-1jOSg(h;cP`M&epk$! z+QA*jb534h=sP)}tjpZPJD2MV{jY|@O8MSY6XAA*y7e)R$bzf#mz5O5DD6M~?9qUe zq>n=`3b+W`HqPcN_(hU}ynpjW z;ZytqdhKKoO-R)5CzB*gqai5`LQv$>e%vRI;?ZL;nI3iFFPUVZ`wqvYLVo+hRh-*%)VqcEpHeXJo0{{&!g#+c1r;1OdRS3v-S0 zVghi)4wLjO1~~LUyG};2RZh6`c>a0?2-ST*i;MJ!KmCshDPo|(3J^G|k_i+{2=qbX z3lNTZ+@JR1LOwEX*ita$l5AjMCvF(H1c{u)S1_a^xu{_*TPDNoGS2N1XkAH#C5zN< z!(<37Z$&`X*F^$E_6xWHTG$J9gFGp3)Qt0+iZLWpEP)2wecw2Lp#g^MNP=OJ5vS+p zK$1rUbEw1@2oC0cUc;esa@hpNGKgT#5jl%5f$Aqn5Wt@R4G;(s5=UVx4$Mjp{EY%2 za!Dq4M`QGkfZ^V+AqQ4@mQTp*FJFLxj=mRikw6ddUO}F`etAUX$Y<1Uks?ERQ9g{s z+|PPbq)wo95Iv*lJTB}R>3%uQrJ#Wa^LQAeJw61~!NlWebVQyHLG1TnrbWroFsx-3 zzLMu6X!dD#U@olJ`^4TTWQ3mwXZ;EV@^qM_{U84HKPA{L zFs`BJ4{;Of$Qm$?KFOweIVFJ2(j4B_BOAaJVQ^6nQ&%z}y+Ef0%<%WqXqdpzkS;3< z1HOVD;0~awyag;U223q8-sJ#GT$kXh!Rd^vnaXXI$kO!AkoqZPymoj zHk=}{)tn}&ZIE~IlSzJ4r-ShG{a2G4N3mSw0CuwguMZkk(s^+G^UD7M` zhI21RL2&<|mbfKs5L(~G5|5)8mW)0u)92T64giGda8ew9_9z*hlOpc{1HYGRX}6f3 zj*_Cl**q0rbN~JH{d<<2f0-BdWqdjY6=XO$>Ab0*s^(C8c_{Vh%>UzuPKO{70Z3tC zj=sC@0ML7x`W+S5X%969WYOkYPNoR7|4OIh{P(}<0Q}>##$7?^DEN9UC{S=rNV6o`@9V_NO&w+<(8ln{iMUx2#IKXjrr-&yg+_GHKA7B%o zI?IMY%6K84p5{sKcj+6D89jIsL?Mcg3*aOIqs|jxUWW)a-K zjPhb&myLb@?r0Mp(W_yC#xK73vYJA1{KXf*rD=?8_r(|GEabGh^3NAvs5>vpF&?W6 z&zA67NLal_3OFLKsrtj@9~7al5rja}Zz@a_Qo>BfQl!PGYvE(_NC3%@Q{{;hoV&t|XwS-pDIeD$A~UIpyo zo{<`Q&0ezxFB4E#MkkI$>*$0Y5wM{FpQ;;}bJ;5Nbiljf@aixB2Nn$4Wwz|5;#>tf zQh<>;?1LsX{~DWrEs>^mw33xUK}Q9y(pXBB7Y7Jd*XWpt2i*hb!C(F#lmBD;OmqeO z@s4mABitr$GIBs+s0W_vl`W4#J-axeAO7RNmvH1z+6zy%=y<9pWd0Kj=3BWf+D_9~>~49DXvm4xoQPA2<)}xgp~?F7sqE0qKSJ>4S!-Yb;+N zkfm5#lF4d#(#NHIb@F~X9brr!aJ`Jkz}Dnz5m?Pdf{rgojYzH{p!?v>d0A3&g0kPR zYcOok8;tj)pakTsla+|z!xbpg3CPt;j$CLIk{UQFN1WhJJw{G|aR4pGVr1Qv`5d^jS;+}1{GpYH=3eC$I7`5sSL8n<4Xzf9 z&8ruJ=PB54{i-=-OC2qkWwiu{hQ{OJwUj$gDk?$u@d(X7m>3SeS_RP|2|Oryph?sf z0Or-_lCfY&Pgbuk&2DD!YY<1-%D<>qK8&sS8m2#|jQ-7RNqe_47V9D@ zl2i0{=)12b?`9&%nSks1F`Z3%Q%BHjfL^pXL(07Ihth zjBKG5)R1!t?kW^>v5{H&vb;!U+ezB(^YtK-3(9?mkz5NJ0D{#dOZzBC2S``PjYlg1 zD+y~jA3@ns(K;`ytjj-@;#q2#D&cRkc}P7$#}li^eA%Igz=`-+~5JU$0`Eragp$`U4tyd;vCt)z2=$ zXan>N;QwNf1KV0o6C3iY3@w>M$SMftpu$aJiv~_fIfJC%N2Om9)8j|u2uy0+C-z84 zJp5P2=6CTrsumaLk7tyPG>!YC>uIhhV>=tPKZ>rbffaf)1>;&N7>fQ*afUshc-L83k?Izg@`=z;QHm|pi zH;T;R`Roji5@OB8W5J@FXbjTRb1RSg7Iz#1wDjS){O!SSXoX=bbU`t$4)>HYbV;C6 z()kob5VhU?2q7>jqZFMK(GX-vErul5A*{j*cJ&pQ-v-&jsO@t?11qqoh!|l6XX;qo z76EaFRzRqDQeP$j0Xm@G>_LL>TYM&$s+M0-M#2lR`Udp}wnG<%SuI@jGiokgO&7o_ zO=iI@@bVv?{o~hPKY#Xi4QcDV4T~Oz@nf|!+-Pd;XX(S)9IZNj{13Ii0KE@$%l8LZ zF0K}qba~AJi82_Wc*ig6$(@E_(0v7u$d6ZFN$=7K!>)P-ybC-9pT+bPK9k3^{|MYVC^62`7g>cC{h(Z`M*aL(h=Lm7jPf7|vDw!v+byPJfagNf7w8QDIa&T?D{tqkt}aiOx>U$n$!E*E zlWcs9Y0qK456khQ>Ly{pakvi`X5qNAHp>3SE1}D5e;o{q&YR_MdPoB|S?(5mlk;U{|2K`x6i-pxeN= z^yUJEwF2d%2o&y6dVd8$G&_dG$K^TE32QO-&Dm&E8c9N1lUt1Hsn=)2a_fK@nY-Aq2Dce;SR(56M@_ zX^xs4292a~wfO^NOM2>XKJL??G4d6|O(+P50uCRo+j;z|>J4F&qKF;m3r>hla~m{dhV8*yFKoo=^&&UB-o%lS@9Yvi;5att_0U< zEMbMK93H7qJM~h8FxCgz=;h17|xJ`H0_2ytqK zvFKT6ju(4~DgJ^hG^BkT9QKk(N9Ebx_jOEF?ESE8uiMk+zHKHD#Rv;B=u9<7x z#}1_~QWJ4az{3oF4PaJ9>7%l~%}xsoc+@(J`*?E5!q|Q597K~9Sw$L+i$OL~$LZo3 zR{#Z_shA05F4m>6iq2;I8yLVUe(>L^2}jjz)np5~9gVY`OS;RRs+83<<;zH=mF z5vtbZ$-?Ru2Cu9`dLi4HJu68-$^(?XqiJuTj_vg`YXpi9IuQUKc=pOFdV?$*S`|(# zIZUe@WhJfAd-q{2nJmqMJcX_+aF9pUnL_xQMi)x|l0kvPxPN+m(kZv1ty9qa2B0r| z0P%!pRXgj$6{thT6N{opv|R8uHd5pk2EcV5fA+O{l*}IeOfHR!!4}KiqC=w@o-m&$ zhFuN2FfdqdL2F%^B<8ahh;`%!M9ybwYj`@;gk?a7&10exsYUk!G=6J5J%xUdxF=a| zjp9*O&s82=Lbp=GDZ6Ss7_&)U%XY{c*$#OH?I_e?zj#Ej;lStu{t%(n1wxdMqyF2t zh1z(MHgBmvXIX|WI>~zgiBXJa592uPNHi8$iAz0F&qi&Gbkr-#Wzz7uk5REGCiAaC zKMl`LxwEp;3pw}TsN7^f&y#+&AA^qaOJEZmqjfcu_Ygk?ke~ERI_L&PlElUqNC=*@ zP+zmCAo4ppMlH4i8Q93|X{Y`vUl0r9l!Q`_ucnu#>vhyWUUGAnayd}LpIGdsh(fkPUHefQHf8oK&E{{f!ZQ)g1;*&vhPkv;Wxo?Yns`UN(SqIJ27 z@INjEUQCB6ST?805M3?(r05N^0@FEw5d$jyNIx^s0H--Rg!Tf-%05A>%LXZadOi5P zv`eI@i;@nRV8RN}3f{}GR6K$$cUCc(2Jm>Ap%#+oalSzmN1&PI4PsmIY|>K#eXHGI z+Sm=IpW1G4l;`MI8!N}^DyKFnh8c_;r;h3}kVJak*!z9u6MsOR3+k-cr1c4v|M^Q1 zC#4T^QP7jhG}u*161~QvrOBf%Vq@d==9-L!Yi{i3E91|=ZhMxEuO+0(U;Y=t z%r2x9~-EfVMuw{Uj}2PoNv6*A<_4$M9a!{TUw0_%*Q5NrblCp2q)`?D%E+nMMf#iO|cF z@#OKr!Qsi{KU_pP8J-+uM-%)tj3*>Hk!dI7U3moKKz@?!;PCL^@cX0q1f&1Zjb+P2 z8z(27?DShO^g2H)uN+@wU@$sI#mOJg&jM@wF+7D?e1E3y9LIKfn`RaK&&$>#8ecYQAinJ2gbTpIl|ZT|r#Fn9F*W5p6b z6qLE48?qqWu7AFve?e)W@87inXA~X_gdc>i+r$p6`{Vd^oWFwgZqUro&qBdm;X5Mm z-4KCRzA!D~;zbe<`;svi&W6uTYz@b(D$BktyHsfb?Lg&s)00RLaL3p3i|g8rt8)VZN~jii&?16ISqU&eCqhFf7iUiY|r;x9)vt_efWFVyuyJHxNuws z#1CWeKQ6C)8OP%{ar|L7ZmC)!zZQ7Lfj9x>8#u-Tfu-jdW{18oz)quco0i!asZ*6= zUrVTrhWaJWqc(FtE#C2f5`cMpk8{uSxf_Z*<{nSjVc^j);2uc)Fl2YkJ)UQKt`o4( z5xyT%{*hOjS8R066|+7Q|ni+z~=yj3M_F4F8QK~?ji-R5+!aD4P^sXt@vmMtHp$Hj6MS$P6Cg9tk0#P>z7_2Lx z5Pe)qU0pkEYjTxI0_+W{YY>Ja2(!k2n(ywLdoTz!qQKQZe5Ub6jfO2 zyMi+odUs5?z{`~R0n$z2g~7HCliev-o}BV~xep^MX1k)f8X<{~%QtHf?gDc$78*M@v<3p-hxGP`XEU0oX6+Jc;oii)#1 z>N4n39wd?Z98~PNJl0ZeW*rgbr9UE-`|bz>gQ6pqym4F;LyA=t$}$v19ep6ftFg|) zz+z6dtKXPP2I?4wj?{TC@F;(I(W0`>BSv48twpD@LIP$d#4}tDv??1c+pSYl)XDYc zVlSXM{9o=k$_@7?#Tdr^HFjJDW2`#l*N@K%45v=x2#ccj5!*88Siv1*Ijr3O+OcY& zb;uQkK0D-E{kdZ57yO~JY-lzn+1XivJK4_X7{ZZ_RZ5nV&QHF4;`^>wUMxWYkJ%Wt ziWNb#@j?qQC&M;o4Rq(HV>CTQ2s_Qqc@ci;2D{B|O&hz-ts?H7W+%S%p1yd#+w9h~ zvD@sBVRyqDXf)61j(A($hGu5n8Ru1l$5Bfj##t~|ji0%OQB)uh)TxK~>12sRGNlm~kj&VJSn zZE`=>3{9cgL1B)Vy$j>>+y1rieqGnkXbs|@Hd__J9$ojq=aE1|K)MdUCx+P>CI}{=W%A}zy9PYM&-BtuTjshk1 zZ6SitaUDv1>W87eJ>Cw4iO|iw}X(o7;!^AF5KWFO36iq^rI{%-1daa znL|Uh`j*VhU7KM{l8>y+93KA2;k0fw32h)FkGnp1-B399i%&lVz^+?Nkp4L3VF+rU z!(7n<_NI#ogF*2y02X(Fo2l!yM7k|3OGpNoNS8XH2r+Tl7$ZM+Fs)fRTxMgew#R(e6#zeaubA>v0PLETL-@8w{eU_w06kCm zEnsiFatNSNP@XAA*NY!A)`D_d^*V$Vqrv{H1}qPp!0}zj0SXO#_hYKp3trw&NJw(e z4ntUE1gyARP%gatrMi3qkX<7sVe$3?*YRAil0AoW-U9Z<`gLHtAs(1>1&B)bqq?Z~ z2TPSKPhSveDaF`wisAViDvWxT@aZ!m|2a6DrYbA%!6ErWd6AI%>REF2%cvM&E~#Sf z2|~JE@-nr?vMxZ{8(^tLRTYn5E@|?^pZ=YsSf^gqp_iGR2bc~a%Sl8ANioUt>-pwn z?dKy>WaKG8mqf!$tX(jSql;K&$;ZxYa_}~LMlh{bLHeo`brFqXa*;&y&B2`nXzUVm zt*8BAoZE-wt<0@e7DUKn7$;2fpN>_^t$)J6c7jMKA1<`X;U^oB|A zL+5CQ^MiQ#T|zK7n8e?SY6#%v}-3Xz?|RWl|ew#T{Lz57yS z+d5d$Cn*POE8>5dPBKj8cJK#s8V`UBKs1m}F#`4*bjbHB8u)(U%HlK4-?L}u8FThT z4X|ZID{+E6WDT>Zk0@_6Qnw*oFV+cqOvwKS_g94@%Z>>EE=x>I literal 0 HcmV?d00001 diff --git a/tests/test_vals_ai_adapter.py b/tests/test_vals_ai_adapter.py index 4d9851e4a..007a0eae6 100644 --- a/tests/test_vals_ai_adapter.py +++ b/tests/test_vals_ai_adapter.py @@ -1,5 +1,6 @@ from __future__ import annotations +import gzip import json from pathlib import Path @@ -608,7 +609,7 @@ def test_statement_slug_replayed_without_methodology_fails_the_run(): convert_pages(benchmark) -def test_mmmu_methodology_fills_temperature_but_not_caps(): +def test_mmmu_methodology_fills_missing_temperature_and_cap(): benchmark = page_benchmark('mmmu_page.html', 'mmmu') text = benchmark['methodology_text'] assert 'All models were ran with a temperature of 0.' in text @@ -628,8 +629,80 @@ def test_mmmu_methodology_fills_temperature_but_not_caps(): o1 = logs['openai/o1-2024-12-17'].evaluation_results[0] assert o1.generation_config.generation_args.temperature == 0.0 - assert o1.generation_config.generation_args.max_tokens is None - assert 'max_tokens_source' not in o1.generation_config.additional_details + assert o1.generation_config.generation_args.max_tokens == 8192 + assert ( + o1.generation_config.additional_details['max_tokens_source'] + == 'page_methodology' + ) + + +@pytest.mark.parametrize('slug', sorted(adapter.PAGE_STATEMENTS)) +def test_page_statements_are_present_in_saved_source_pages(slug): + page_html = gzip.decompress( + (DATA_DIR / 'saved_pages' / f'{slug}.html.gz').read_bytes() + ).decode('utf-8') + text = adapter.extract_methodology_text(page_html) + assert text is not None + for statement in adapter.PAGE_STATEMENTS[slug]: + assert statement.sentence in text + + +def test_mmmu_missing_cap_sentence_fails_even_with_temperature_statement(): + benchmark = page_benchmark('mmmu_page.html', 'mmmu') + benchmark['methodology_text'] = 'All models were ran with a temperature of 0.' + with pytest.raises(adapter.ValsMethodologyStatementMissing, match='8192'): + convert_pages(benchmark) + + +def test_loader_fetches_full_rows_instead_of_graph_projections(monkeypatch): + page_html = ''' + + + + + ''' + view = { + 'metadata': {'slug': 'example'}, + 'tasks': { + 'overall': {'model': {'accuracy': 50, 'temperature': 0.7}}, + 'subtask': {'model': {'accuracy': 60, 'temperature': 0.7}}, + }, + } + responses = { + 'https://example.test/benchmarks': 'x', + 'https://example.test/benchmarks/example': page_html, + 'https://example.test/_astro/full-table.json': json.dumps(view), + } + calls = [] + + def fetch(url): + calls.append(url) + return responses[url] + + monkeypatch.setattr(adapter, 'fetch_text', fetch) + payload = adapter.extract_collection(base_url='https://example.test') + assert not payload['source_failures'] + assert payload['benchmarks'][0]['tasks'] == view['tasks'] + assert calls == list(responses) + + +def test_failed_loader_is_reported_without_using_partial_graph_data(monkeypatch): + def fetch(url): + if url.endswith('/benchmarks'): + return 'x' + if url.endswith('/example'): + return '''''' + raise RuntimeError('table unavailable') + + monkeypatch.setattr(adapter, 'fetch_text', fetch) + payload = adapter.extract_collection(base_url='https://example.test') + assert payload['benchmarks'] == [] + assert len(payload['source_failures']) == 1 + assert 'table unavailable' in payload['source_failures'][0]['reason'] def test_reasoning_is_typed_only_from_a_real_bool(): From 202e25eed533e1f777191b3566faaeb9fc7f0400 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 15:37:31 +0100 Subject: [PATCH 11/16] Keep Inspect sample counts per scorer and bound plan fills --- every_eval_ever/converters/inspect/adapter.py | 45 ++++++++++++++---- tests/test_inspect_adapter.py | 47 +++++++++++++++++-- 2 files changed, 80 insertions(+), 12 deletions(-) diff --git a/every_eval_ever/converters/inspect/adapter.py b/every_eval_ever/converters/inspect/adapter.py index 802b41257..a32687c7c 100644 --- a/every_eval_ever/converters/inspect/adapter.py +++ b/every_eval_ever/converters/inspect/adapter.py @@ -187,7 +187,11 @@ def _extract_uncertainty( stddev_value: float | None, num_samples: int | None, ) -> Uncertainty | None: - if stderr_value is None and stddev_value is None and not num_samples: + if ( + stderr_value is None + and stddev_value is None + and num_samples is None + ): return None return Uncertainty( # A standard error of exactly 0.0 is what a task every sample @@ -318,9 +322,21 @@ def _extract_evaluation_results( # score, and states how many those were. The run-wide count includes # the samples this scorer returned no value for, and is what a log # from before Inspect reported this per scorer leaves us with. - scored_samples = ( - getattr(scorer, 'scored_samples', None) or num_samples - ) + scored_samples = getattr(scorer, 'scored_samples', None) + if scored_samples is None: + scored_samples = num_samples + scorer_source_data = source_data.model_copy(deep=True) + if isinstance(scorer_source_data, SourceDataHf): + scorer_source_data.samples_number = scored_samples + elif scored_samples is not None: + scorer_source_data.additional_details = { + **(scorer_source_data.additional_details or {}), + 'samples_number': str(scored_samples), + } + if (scorer_source_data.additional_details or {}).get( + 'dataset_size' + ) == str(scored_samples): + scorer_source_data.additional_details.pop('dataset_size') # Inspect reports dispersion as a metric of the scorer, and the # schema carries it on the score it describes rather than beside it. @@ -346,7 +362,7 @@ def _extract_evaluation_results( scorer_name=scorer_name, metric_info=metric_info, llm_grader=llm_grader, - source_data=source_data, + source_data=scorer_source_data, evaluation_timestamp=timestamp, generation_config=generation_config, stderr_value=None if describes_itself else stderr_value, @@ -401,7 +417,7 @@ def _extract_source_data( ) dataset_size = ( str(dataset.samples) - if dataset.samples is not None and dataset.samples != samples_number + if dataset.samples is not None else None ) split = (task_args or {}).get('split') @@ -557,12 +573,24 @@ def _overridden_generation_fields( def _extract_generation_config( self, spec: EvalSpec, inspect_plan: InspectEvalPlan ) -> GenerationConfig: - eval_config = spec.model_generate_config.merge(inspect_plan.config) + unverified_packages = sorted( + set(spec.packages) - {'inspect_ai', 'inspect_evals'} + ) + # Other scaffolds can override the plan's settings inside their calls. + eval_config = ( + spec.model_generate_config + if unverified_packages + else spec.model_generate_config.merge(inspect_plan.config) + ) eval_generation_config = { gen_config: json.dumps(value) for gen_config, value in vars(eval_config).items() if value is not None } + if unverified_packages: + eval_generation_config['plan_config_unverified_packages'] = ','.join( + unverified_packages + ) eval_sandbox = spec.task_args.get('sandbox', None) if eval_sandbox and not isinstance(eval_sandbox, list): eval_sandbox = [eval_sandbox] @@ -618,7 +646,8 @@ def _extract_generation_config( ) if filled: from_plan = { - getattr(inspect_plan.config, field) is not None + not unverified_packages + and getattr(inspect_plan.config, field) is not None for field in filled } eval_generation_config['generation_args_source'] = ( diff --git a/tests/test_inspect_adapter.py b/tests/test_inspect_adapter.py index ddaa74120..76529cc97 100644 --- a/tests/test_inspect_adapter.py +++ b/tests/test_inspect_adapter.py @@ -788,7 +788,8 @@ def test_extract_evaluation_results_one_scorer_with_two_metrics(): assert result_ids_by_scorer == {'choice': ['choice:accuracy', 'choice:f1']} -def test_each_scorer_counts_the_samples_it_could_score(): +@pytest.mark.parametrize('private', [False, True]) +def test_each_scorer_counts_the_samples_it_could_score(private): """Inspect computes a scorer's metrics over the samples that scorer scored, and says how many those were; the run-wide count includes the ones it skipped.""" adapter = InspectAIAdapter() @@ -801,12 +802,21 @@ def test_each_scorer_counts_the_samples_it_could_score(): _make_scorer('lenient', {'accuracy': _make_metric('accuracy', 0.9)}), ] + source = ( + SourceDataPrivate( + dataset_name='synthetic_ds', + source_type='other', + additional_details={'samples_number': '10'}, + ) + if private + else SourceDataHf( + dataset_name='synthetic_ds', source_type='hf_dataset', samples_number=10 + ) + ) results, _ = adapter._extract_evaluation_results( evaluation_task_name='synthetic/task', scores=scores, - source_data=SourceDataHf( - dataset_name='synthetic_ds', source_type='hf_dataset' - ), + source_data=source, generation_config=GenerationConfig(), num_samples=10, timestamp='1234567890', @@ -819,6 +829,35 @@ def test_each_scorer_counts_the_samples_it_could_score(): assert by_id['strict:accuracy'].num_samples == 7 # No count of its own, so the run's stands in. assert by_id['lenient:accuracy'].num_samples == 10 + counts = [ + int(result.source_data.additional_details['samples_number']) + if private + else result.source_data.samples_number + for result in results + ] + assert counts == [7, 10] + assert ( + int(source.additional_details['samples_number']) + if private + else source.samples_number + ) == 10 + + +def test_unverified_scaffold_cannot_supply_plan_generation_defaults(tmp_path): + def rewrite(log): + log['eval']['packages']['private_scaffold'] = '1.2.3' + log['eval']['model_generate_config'] = {'top_p': 0.9} + + converted = _load_rewritten_eval(tmp_path, CYSE2_LOG, rewrite) + config = converted.evaluation_results[0].generation_config + assert config.generation_args.temperature is None + assert config.generation_args.max_tokens is None + assert config.generation_args.top_p == 0.9 + assert config.generation_args.eval_plan.config['max_tokens'] == '2048' + assert config.additional_details['generation_args_source'] == 'model_config' + assert config.additional_details['plan_config_unverified_packages'] == ( + 'private_scaffold' + ) def test_standard_error_says_how_inspect_computed_it(): From 1571a1939cf202d68bbfbfa78cf2a2874883dc17 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 15:37:31 +0100 Subject: [PATCH 12/16] Expose published HF and ARC sample counts in uncertainty --- every_eval_ever/adapters/arc_agi/adapter.py | 6 ++++++ every_eval_ever/adapters/hfopenllm_v2/adapter.py | 4 ++++ tests/test_arc_agi_adapter.py | 2 ++ tests/test_hfopenllm_v2_adapter.py | 4 ++++ 4 files changed, 16 insertions(+) diff --git a/every_eval_ever/adapters/arc_agi/adapter.py b/every_eval_ever/adapters/arc_agi/adapter.py index 91955a041..0d90a76fe 100644 --- a/every_eval_ever/adapters/arc_agi/adapter.py +++ b/every_eval_ever/adapters/arc_agi/adapter.py @@ -76,6 +76,7 @@ SourceDataUrl, SourceMetadata, SourceType, + Uncertainty, ) from every_eval_ever.helpers import ( SCHEMA_VERSION, @@ -431,6 +432,11 @@ def make_results( ), score_details=ScoreDetails( score=float(row['score']), + uncertainty=Uncertainty( + num_samples=int(run_source_details['scored_tasks']) + ) + if 'scored_tasks' in run_source_details + else None, details={ **stringify_details( row, exclude_keys={'score', 'modelId'} diff --git a/every_eval_ever/adapters/hfopenllm_v2/adapter.py b/every_eval_ever/adapters/hfopenllm_v2/adapter.py index a1422fdaa..44cfd47fc 100644 --- a/every_eval_ever/adapters/hfopenllm_v2/adapter.py +++ b/every_eval_ever/adapters/hfopenllm_v2/adapter.py @@ -52,6 +52,7 @@ ScoreDetails, ScoreType, SourceDataHf, + Uncertainty, ) from every_eval_ever.helpers import ( SCHEMA_VERSION, @@ -267,6 +268,9 @@ def convert_model( ), score_details=ScoreDetails( score=round(float(eval_data['value']), 4), + uncertainty=Uncertainty(num_samples=group.samples_number) + if group is not None and group.samples_number is not None + else None, ), generation_config=_generation_config(group), ) diff --git a/tests/test_arc_agi_adapter.py b/tests/test_arc_agi_adapter.py index 5e2f85afc..8626b7227 100644 --- a/tests/test_arc_agi_adapter.py +++ b/tests/test_arc_agi_adapter.py @@ -396,6 +396,7 @@ def test_accepted_run_fills_the_score_result(runs_dir): 'request_log': f'{V2.repo_id}@{V2.revision}/{RUN_ID}', } assert score.source_data.additional_details['scored_tasks'] == '3' + assert score.score_details.uncertainty.num_samples == 3 assert [(o.dataset_id, o.rejection) for o in logs.outcomes] == [ ('v2_Public_Eval', None) ] @@ -413,6 +414,7 @@ def test_cost_result_and_other_sets_are_untouched(runs_dir): result = result_by_id(log, result_id) assert result.generation_config is None, result_id assert 'scored_tasks' not in result.source_data.additional_details + assert result.score_details.uncertainty is None def test_v2_run_never_fills_v1_even_when_v1_has_no_directory(runs_dir): diff --git a/tests/test_hfopenllm_v2_adapter.py b/tests/test_hfopenllm_v2_adapter.py index 6c999bdfd..31530de79 100644 --- a/tests/test_hfopenllm_v2_adapter.py +++ b/tests/test_hfopenllm_v2_adapter.py @@ -228,6 +228,10 @@ def test_generative_groups_get_decoding_and_multiple_choice_do_not(): def test_dataset_split_and_counts_come_from_the_dumps(): (log,) = _convert([_row(COGITO, 'bfloat16')]) results = _results(log) + for result in results.values(): + assert result.score_details.uncertainty.num_samples == ( + result.source_data.samples_number + ) ifeval = results['IFEval'].source_data assert ifeval.hf_repo == 'google/IFEval' From 6fc6a397b6c9517c85cbafdd801a767c1c3a0d10 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 15:45:12 +0100 Subject: [PATCH 13/16] Reject Vals loader payloads without table fields --- every_eval_ever/adapters/vals_ai/adapter.py | 2 ++ tests/test_vals_ai_adapter.py | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/every_eval_ever/adapters/vals_ai/adapter.py b/every_eval_ever/adapters/vals_ai/adapter.py index ae43ab515..1fdd81c2a 100644 --- a/every_eval_ever/adapters/vals_ai/adapter.py +++ b/every_eval_ever/adapters/vals_ai/adapter.py @@ -388,6 +388,8 @@ def normalize_benchmark_page( else extract_benchmark_view(page_html) ) view = view.get('default') or view + if not isinstance(view, dict) or not {'metadata', 'tasks'}.issubset(view): + raise ValueError('BenchmarkView payload lacks metadata/tasks') metadata = view.get('metadata') or {} tasks = view.get('tasks') or {} if not isinstance(metadata, dict) or not isinstance(tasks, dict): diff --git a/tests/test_vals_ai_adapter.py b/tests/test_vals_ai_adapter.py index 007a0eae6..9433b665c 100644 --- a/tests/test_vals_ai_adapter.py +++ b/tests/test_vals_ai_adapter.py @@ -705,6 +705,15 @@ def fetch(url): assert 'table unavailable' in payload['source_failures'][0]['reason'] +def test_loader_payload_without_table_fields_is_not_an_empty_benchmark(): + with pytest.raises(ValueError, match='lacks metadata/tasks'): + adapter.normalize_benchmark_page( + '', + 'https://example.test/benchmarks/example', + benchmark_view={'error': 'unavailable'}, + ) + + def test_reasoning_is_typed_only_from_a_real_bool(): def reasoning_of(row: dict): config = adapter.make_generation_config(row) From 3c570b1b5488137d328a7603fdf4863c2435e7f6 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 18:50:47 +0100 Subject: [PATCH 14/16] Fetch pinned HF results in one Git pack --- .../adapters/hfopenllm_v2/adapter.py | 123 +++++++-- .../adapters/hfopenllm_v2/dump_index.py | 10 +- tests/test_hfopenllm_v2_adapter.py | 252 ++++++++++++++++++ 3 files changed, 358 insertions(+), 27 deletions(-) diff --git a/every_eval_ever/adapters/hfopenllm_v2/adapter.py b/every_eval_ever/adapters/hfopenllm_v2/adapter.py index 44cfd47fc..493a9fe95 100644 --- a/every_eval_ever/adapters/hfopenllm_v2/adapter.py +++ b/every_eval_ever/adapters/hfopenllm_v2/adapter.py @@ -10,6 +10,9 @@ ``--source-api`` reads the table from the leaderboard Space API instead, which is unpinned and no longer served. +The default results fetch requires Git and transfers the pinned revision in one +shallow fetch. Only result JSON files are checked out, into a temporary directory. + Usage: uv run python -m every_eval_ever.adapters.hfopenllm_v2.adapter uv run python -m every_eval_ever.adapters.hfopenllm_v2.adapter \ @@ -20,9 +23,13 @@ import argparse import json +import os import re +import subprocess +import tempfile import time from collections import Counter +from contextlib import nullcontext from pathlib import Path from typing import Any, Dict, List @@ -71,6 +78,7 @@ # Source URL SOURCE_URL = 'https://open-llm-leaderboard-open-llm-leaderboard.hf.space/api/leaderboard/formatted' OUTPUT_DIR = 'data/hfopenllm_v2' +RESULTS_URL = f'https://huggingface.co/datasets/{RESULTS_REPO}' # Evaluation name mapping from API keys to display names EVALUATION_MAPPING = { @@ -517,9 +525,10 @@ def parse_args(argv: List[str] | None = None) -> argparse.Namespace: type=Path, default=None, help=( - f'Local Hub snapshot of {RESULTS_REPO} (a directory named after ' - f'its commit sha, which must be {RESULTS_REVISION}). Default: ' - 'download its results_*.json files once from the Hub.' + f'Local Git checkout or Hub snapshot of {RESULTS_REPO}, at ' + f'{RESULTS_REVISION}. Git checkouts use HEAD; Hub snapshots use ' + 'the commit sha in the directory name. Default: one shallow ' + 'Git fetch into a temporary directory.' ), ) parser.add_argument( @@ -593,13 +602,22 @@ def results_dir_revision( ) -> tuple[str, bool]: """The commit a local results snapshot is at, and whether it is unpinned. - The commit is the directory's own name, as in the Hub cache layout - ``snapshots/``. A directory at another commit, or without one, is - an error unless ``allow_unpinned``; it is then reported as that sha, or - ``'local'``. + Git checkouts use their actual HEAD. For existing Hub snapshots the commit + is the directory's own name, as in ``snapshots/``. Other revisions + require ``allow_unpinned`` and are reported without claiming the pin. """ + root = Path(results_dir).resolve() + if (root / '.git').exists(): + revision = _git('rev-parse', 'HEAD^{commit}', cwd=root) + if revision != RESULTS_REVISION and not allow_unpinned: + raise ValueError( + f'--results-dir is at {revision}, but this adapter is pinned ' + f'to {RESULTS_REPO}@{RESULTS_REVISION}. Pass a checkout at ' + 'that revision, or --allow-unpinned-results-dir.' + ) + return revision, revision != RESULTS_REVISION return _snapshot_revision( - Path(results_dir).resolve(), + root, option='--results-dir', repo=RESULTS_REPO, pinned=RESULTS_REVISION, @@ -626,18 +644,58 @@ def contents_parquet_revision( ) -def download_results() -> Path: - """Fetch the pinned results snapshot's dumps in one Hub download.""" - from huggingface_hub import snapshot_download - - return Path( - snapshot_download( - repo_id=RESULTS_REPO, - repo_type='dataset', - revision=RESULTS_REVISION, - allow_patterns=RESULTS_PATTERN, +def _git(*args: str, cwd: Path | None = None) -> str: + try: + result = subprocess.run( + ['git', *args], + cwd=cwd, + env={ + **os.environ, + 'GIT_LFS_SKIP_SMUDGE': '1', + 'GIT_TERMINAL_PROMPT': '0', + }, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise RuntimeError( + 'Git is required to fetch HF leaderboard results' + ) from exc + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f'git {args[0]} failed: {exc.stderr.strip()}' + ) from exc + return result.stdout.strip() + + +def download_results(destination: str | Path) -> Path: + """Fetch the pinned Git revision once into an empty, caller-owned directory.""" + root = Path(destination).resolve() + if root.exists() and any(root.iterdir()): + raise ValueError(f'Results destination must be empty: {root}') + root.mkdir(parents=True, exist_ok=True) + _git('init', cwd=root) + _git('fetch', '--depth', '1', RESULTS_URL, RESULTS_REVISION, cwd=root) + fetched = _git('rev-parse', 'FETCH_HEAD^{commit}', cwd=root) + if fetched != RESULTS_REVISION: + raise RuntimeError( + f'Fetched results commit {fetched}, expected {RESULTS_REVISION}' + ) + _git('sparse-checkout', 'set', '--no-cone', RESULTS_PATTERN, cwd=root) + _git('checkout', '--detach', RESULTS_REVISION, cwd=root) + checked_out = _git('rev-parse', 'HEAD^{commit}', cwd=root) + if checked_out != RESULTS_REVISION: + raise RuntimeError( + f'Checked out results commit {checked_out}, expected {RESULTS_REVISION}' ) + raw_capture.record_git_checkout( + RESULTS_URL, + root, + ref=RESULTS_REVISION, + label='Open LLM Leaderboard v2 harness result dumps', ) + return root def main(argv: List[str] | None = None) -> int: @@ -677,14 +735,29 @@ def main(argv: List[str] | None = None) -> int: 'unpinned': contents_unpinned, } - raw_capture.record_hf_dataset( - RESULTS_REPO, - revision=revision, - label='Open LLM Leaderboard v2 harness result dumps', + if args.results_dir is not None: + if (args.results_dir / '.git').exists(): + raw_capture.record_git_checkout( + RESULTS_URL, + args.results_dir, + ref=revision, + label='Open LLM Leaderboard v2 harness result dumps', + ) + else: + raw_capture.record_hf_dataset( + RESULTS_REPO, + revision=revision, + label='Open LLM Leaderboard v2 harness result dumps', + ) + workspace = ( + nullcontext(args.results_dir) + if args.results_dir is not None + else tempfile.TemporaryDirectory(prefix='eee-hfopenllm-v2-') ) - results_dir = args.results_dir or download_results() - print(f'Indexing harness dumps in {results_dir} ({revision})...') - dump_index = build_dump_index(results_dir) + with workspace as root: + results_dir = args.results_dir or download_results(root) + print(f'Indexing harness dumps in {results_dir} ({revision})...') + dump_index = build_dump_index(results_dir) dump_index.revision, dump_index.unpinned = revision, unpinned print( f'Read {dump_index.files_read} dumps for {len(dump_index.entries)} ' diff --git a/every_eval_ever/adapters/hfopenllm_v2/dump_index.py b/every_eval_ever/adapters/hfopenllm_v2/dump_index.py index 8abc76692..309f2da38 100644 --- a/every_eval_ever/adapters/hfopenllm_v2/dump_index.py +++ b/every_eval_ever/adapters/hfopenllm_v2/dump_index.py @@ -17,7 +17,7 @@ RESULTS_REPO = 'open-llm-leaderboard/results' RESULTS_REVISION = 'aa81ecc38fdc5708254b833923368970efdf5ef5' -RESULTS_PATTERN = '*/results_*.json' +RESULTS_PATTERN = '*/**/results_*.json' DTYPE_PRECISION = { 'torch.bfloat16': 'bfloat16', @@ -292,7 +292,13 @@ def build_dump_index(results_dir: str | Path) -> DumpIndex: index.files_read += 1 rel = path.relative_to(root).as_posix() try: - dump = json.loads(path.read_text(encoding='utf-8')) + content = path.read_text(encoding='utf-8') + if content.startswith('version https://git-lfs.github.com/spec/v1'): + raise RuntimeError( + f'Result dump {rel} is a Git LFS pointer, not JSON. ' + 'Bulk results fetching requires Git-tracked JSON blobs.' + ) + dump = json.loads(content) except ValueError as exc: index.rejected.append( {'path': rel, 'reason': 'unreadable', 'detail': str(exc)} diff --git a/tests/test_hfopenllm_v2_adapter.py b/tests/test_hfopenllm_v2_adapter.py index 31530de79..ac0f51993 100644 --- a/tests/test_hfopenllm_v2_adapter.py +++ b/tests/test_hfopenllm_v2_adapter.py @@ -12,6 +12,7 @@ contents_parquet_revision, convert_model, convert_models, + download_results, process_models, results_dir_revision, ) @@ -478,6 +479,257 @@ def test_unpinned_results_dir_is_recorded_and_reported(monkeypatch, tmp_path): ) +@pytest.fixture +def results_git_remote(tmp_path, monkeypatch, request): + if shutil.which('git') is None: + pytest.skip('Git is required for the local bulk-fetch fixture') + + def git(*args, cwd=None): + return subprocess.run( + ['git', *args], cwd=cwd, capture_output=True, text=True, check=True + ).stdout.strip() + + work = tmp_path / 'source repository' + work.mkdir() + git('init', '--initial-branch=fixture', cwd=work) + git('config', 'user.name', 'Fixture', cwd=work) + git('config', 'user.email', 'fixture@example.invalid', cwd=work) + (work / 'README.md').write_text('First commit, without result dumps.\n') + git('add', '.', cwd=work) + git('commit', '-m', 'Initial fixture', cwd=work) + + cached = tmp_path / 'cached snapshot' + shutil.copytree(RESULTS, cached) + spaced = cached / 'extra organization' / 'model with spaces' + spaced.mkdir(parents=True) + shutil.copy( + next((RESULTS / DBRX).glob('results_*.json')), + spaced / 'results_extra.json', + ) + for relative in ( + 'single/results_shallow.json', + 'one/two/three/results_deep.json', + ): + target = cached / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy(spaced / 'results_extra.json', target) + if getattr(request, 'param', None) == 'lfs_pointer': + (spaced / 'results_future.json').write_text( + 'version https://git-lfs.github.com/spec/v1\n' + f'oid sha256:{"a" * 64}\n' + 'size 1234\n' + ) + shutil.copytree(cached, work, dirs_exist_ok=True) + shutil.copy(spaced / 'results_extra.json', work / 'results_root.json') + git('add', '.', cwd=work) + git('commit', '-m', 'Pinned fixture results', cwd=work) + pinned = git('rev-parse', 'HEAD', cwd=work) + + dump = next((work / DBRX).glob('results_*.json')) + dump.unlink() + (work / 'README.md').write_text('Later commit, with a result removed.\n') + git('add', '.', cwd=work) + git('commit', '-m', 'Later fixture', cwd=work) + latest = git('rev-parse', 'HEAD', cwd=work) + remote = tmp_path / 'remote with spaces.git' + git('clone', '--bare', str(work), str(remote)) + + monkeypatch.setattr(adapter, 'RESULTS_URL', remote.as_uri()) + monkeypatch.setattr(adapter, 'RESULTS_REVISION', pinned) + return cached, pinned, latest + + +def test_results_bulk_fetch_matches_cached_index_and_conversion( + results_git_remote, tmp_path, monkeypatch +): + cached, pinned, latest = results_git_remote + commands = [] + actual_run = subprocess.run + + def record_command(args, **kwargs): + commands.append(args) + return actual_run(args, **kwargs) + + monkeypatch.setattr(adapter.subprocess, 'run', record_command) + monkeypatch.setattr( + adapter.raw_capture, + 'record_hf_dataset', + lambda *args, **kwargs: pytest.fail('unexpected per-file Hub path'), + ) + captured = [] + monkeypatch.setattr( + adapter.raw_capture, + 'record_git_checkout', + lambda url, checkout, **kwargs: captured.append( + (url, checkout, kwargs['ref']) + ), + ) + + root = download_results(tmp_path / 'download with spaces') + + assert results_dir_revision(root) == (pinned, False) + assert pinned != latest + assert (root / '.git' / 'shallow').read_text().splitlines() == [pinned] + assert not (root / 'README.md').exists() + assert not (root / 'results_root.json').exists() + assert not list((root / '.git' / 'objects' / 'pack').glob('*.promisor')) + fetches = [command for command in commands if command[1] == 'fetch'] + assert fetches == [ + ['git', 'fetch', '--depth', '1', adapter.RESULTS_URL, pinned] + ] + assert all( + '--filter' not in argument + for command in commands + for argument in command + ) + assert captured == [(adapter.RESULTS_URL, root, pinned)] + assert build_dump_index(root) == build_dump_index(cached) + rows = [_row(COGITO, 'bfloat16'), _row(DBRX, 'float16')] + assert _convert(rows, root) == _convert(rows, cached) + for expected in cached.rglob('results_*.json'): + assert ( + root / expected.relative_to(cached) + ).read_bytes() == expected.read_bytes() + + +@pytest.mark.parametrize('results_git_remote', ['lfs_pointer'], indirect=True) +def test_results_lfs_pointer_fails_loudly_after_bulk_fetch( + results_git_remote, tmp_path, monkeypatch +): + cached, _, _ = results_git_remote + root = download_results(tmp_path / 'results') + monkeypatch.setattr( + adapter.subprocess, + 'run', + lambda *args, **kwargs: pytest.fail('unexpected extra Git transfer'), + ) + for results_dir in (cached, root): + with pytest.raises( + RuntimeError, match='results_future.json is a Git LFS pointer' + ): + build_dump_index(results_dir) + + +def test_results_git_checkout_does_not_trust_directory_name( + results_git_remote, tmp_path, monkeypatch +): + _, pinned, latest = results_git_remote + root = download_results(tmp_path / latest) + assert results_dir_revision(root) == (pinned, False) + + monkeypatch.setattr(adapter, 'RESULTS_REVISION', latest) + with pytest.raises(ValueError, match=pinned): + results_dir_revision(root) + assert results_dir_revision(root, allow_unpinned=True) == (pinned, True) + + +def test_results_fetch_requires_git(tmp_path, monkeypatch): + def missing_git(*args, **kwargs): + raise FileNotFoundError('git') + + monkeypatch.setattr(adapter.subprocess, 'run', missing_git) + with pytest.raises(RuntimeError, match='Git is required'): + download_results(tmp_path / 'results') + + +def test_results_fetch_does_not_modify_nonempty_destination(tmp_path): + existing = tmp_path / 'keep.txt' + existing.write_text('caller data') + with pytest.raises(ValueError, match='must be empty'): + download_results(tmp_path) + assert existing.read_text() == 'caller data' + assert not (tmp_path / '.git').exists() + + +def test_results_fetch_capture_records_verified_checkout( + results_git_remote, tmp_path, monkeypatch +): + _, pinned, _ = results_git_remote + sink = adapter.raw_capture.RawSink(tmp_path / 'capture') + monkeypatch.setattr(adapter.raw_capture, 'active_sink', lambda: sink) + monkeypatch.setattr( + adapter.raw_capture, + 'record_hf_dataset', + lambda *args, **kwargs: pytest.fail('unexpected Hub lookup'), + ) + + download_results(tmp_path / 'results') + + (entry,) = sink.entries() + assert entry['kind'] == 'pointer' + assert entry['pointer_kind'] == 'git' + assert entry['reference'] == adapter.RESULTS_URL + assert entry['revision'] == pinned + assert not sink.degraded + + +@pytest.mark.parametrize('wrong_step', ['fetch', 'checkout']) +def test_results_fetch_rejects_wrong_commit( + results_git_remote, tmp_path, monkeypatch, wrong_step +): + _, _, latest = results_git_remote + actual_git = adapter._git + + def wrong_revision(*args, **kwargs): + result = actual_git(*args, **kwargs) + target = ( + 'FETCH_HEAD^{commit}' if wrong_step == 'fetch' else 'HEAD^{commit}' + ) + return latest if args == ('rev-parse', target) else result + + monkeypatch.setattr(adapter, '_git', wrong_revision) + with pytest.raises(RuntimeError, match=f'commit {latest}, expected'): + download_results(tmp_path / 'results') + + +def test_results_fetch_failure_has_no_hub_fallback(tmp_path, monkeypatch): + commands = [] + + def failed_fetch(args, **kwargs): + commands.append(args) + if args[1] == 'fetch': + raise subprocess.CalledProcessError( + 128, args, stderr='Pinned commit is unavailable' + ) + return subprocess.CompletedProcess(args, 0, stdout='') + + monkeypatch.setattr(adapter.subprocess, 'run', failed_fetch) + with pytest.raises(RuntimeError, match='Pinned commit is unavailable'): + download_results(tmp_path / 'results') + assert [command[1] for command in commands] == ['init', 'fetch'] + + +@pytest.mark.parametrize('fail_indexing', [False, True]) +def test_main_cleans_temporary_results_checkout( + results_git_remote, tmp_path, monkeypatch, fail_indexing +): + workspaces = [] + actual_download = download_results + actual_index = build_dump_index + + def downloaded(destination): + workspaces.append(Path(destination)) + return actual_download(destination) + + def index_results(root): + if fail_indexing: + raise ValueError('Cannot index results') + return actual_index(root) + + monkeypatch.setattr(adapter, 'download_results', downloaded) + monkeypatch.setattr(adapter, 'build_dump_index', index_results) + monkeypatch.setattr(adapter, 'fetch_json', lambda url: []) + monkeypatch.setattr(adapter, 'process_models', lambda *args: 0) + argv = ['--source-api', '--output-dir', str(tmp_path / 'outputs')] + if fail_indexing: + with pytest.raises(ValueError, match='Cannot index results'): + adapter.main(argv) + else: + assert adapter.main(argv) == 0 + assert len(workspaces) == 1 + assert not workspaces[0].exists() + + def test_stated_model_details_are_kept_with_the_library_version(): source = _row('example/model', 'float16') source['model']['architecture'] = 'LlamaForCausalLM' From 0c9064574e8866bd9cd51b33e8a154e5b5302593 Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 18:55:22 +0100 Subject: [PATCH 15/16] Read published flat aggregates from a pinned Git pack --- every_eval_ever/cron/flat_snapshot.py | 256 +++++++++++++++++++ tests/test_flat_snapshot.py | 343 ++++++++++++++++++++++++++ 2 files changed, 599 insertions(+) create mode 100644 every_eval_ever/cron/flat_snapshot.py create mode 100644 tests/test_flat_snapshot.py diff --git a/every_eval_ever/cron/flat_snapshot.py b/every_eval_ever/cron/flat_snapshot.py new file mode 100644 index 000000000..3b48ff2c4 --- /dev/null +++ b/every_eval_ever/cron/flat_snapshot.py @@ -0,0 +1,256 @@ +"""Read published aggregate objects from one pinned, shallow Git fetch.""" + +from __future__ import annotations + +import hashlib +import os +import re +import subprocess +import tempfile +from pathlib import Path, PurePosixPath +from typing import Iterable +from urllib.parse import urlsplit + +from huggingface_hub import HfApi +from huggingface_hub.utils import build_hf_headers + +_COMMIT = re.compile(r'[0-9a-fA-F]{40}(?:[0-9a-fA-F]{24})?\Z') +_LFS_VERSION = b'version https://git-lfs.github.com/spec/' +_RETIRED_PATH = 'flat/retired.jsonl' + + +class FlatSnapshotError(RuntimeError): + """The pinned published objects could not be read safely.""" + + +def _pointer(data: bytes, path: str) -> tuple[str, int] | None: + if not data.startswith(_LFS_VERSION): + return None + try: + lines = data.decode('ascii').splitlines() + except UnicodeDecodeError as exc: + raise FlatSnapshotError(f'Malformed Git LFS pointer: {path}') from exc + oid = [ + line.removeprefix('oid sha256:') + for line in lines + if line.startswith('oid sha256:') + ] + size = [ + line.removeprefix('size ') for line in lines if line.startswith('size ') + ] + if ( + lines[0] != 'version https://git-lfs.github.com/spec/v1' + or len(oid) != 1 + or re.fullmatch(r'[0-9a-f]{64}', oid[0]) is None + or len(size) != 1 + or re.fullmatch(r'[0-9]+', size[0]) is None + or any( + not line.startswith(('version ', 'oid sha256:', 'size ', 'ext-')) + for line in lines + ) + ): + raise FlatSnapshotError(f'Malformed Git LFS pointer: {path}') + return oid[0], int(size[0]) + + +class FlatSnapshotReader: + """A temporary checkout of aggregate objects and the retirement list. + + Resolve the dataset's current commit once, or supply an immutable ``revision``. + Git transfers plain blobs in one pack with LFS smudging disabled. Only a read + of an LFS-backed path downloads its payload, at the same pinned commit. + Sample sidecars are outside this reader's scope. + """ + + def __init__( + self, + api: HfApi, + repo_id: str, + *, + revision: str | None = None, + remote_url: str | None = None, + ) -> None: + self.api = api + self.repo_id = repo_id + self.revision = revision + endpoint = getattr(api, 'endpoint', 'https://huggingface.co').rstrip( + '/' + ) + self.remote_url = remote_url or f'{endpoint}/datasets/{repo_id}' + parsed = urlsplit(self.remote_url) + if parsed.username is not None or parsed.password is not None: + raise ValueError('Git remote URLs must not contain credentials') + self.pack_count = 0 + self.lfs_download_paths: list[str] = [] + self._payloads: dict[str, Path] = {} + self._temporary: tempfile.TemporaryDirectory | None = None + self._directory: Path | None = None + self._env = os.environ.copy() + self._env.update(GIT_LFS_SKIP_SMUDGE='1', GIT_TERMINAL_PROMPT='0') + config = { + 'filter.lfs.smudge': '', + 'filter.lfs.process': '', + 'filter.lfs.required': 'false', + } + self._authorization = None + if self.remote_url.startswith(endpoint + '/'): + self._authorization = build_hf_headers( + token=getattr(api, 'token', None) + ).get('authorization') + if self._authorization: + config[f'http.{endpoint}/.extraHeader'] = ( + f'Authorization: {self._authorization}' + ) + count = int(self._env.get('GIT_CONFIG_COUNT', '0')) + for key, value in config.items(): + self._env[f'GIT_CONFIG_KEY_{count}'] = key + self._env[f'GIT_CONFIG_VALUE_{count}'] = value + count += 1 + self._env['GIT_CONFIG_COUNT'] = str(count) + + @property + def lfs_download_count(self) -> int: + """Number of named LFS payload downloads requested by this reader.""" + return len(self.lfs_download_paths) + + def _git(self, *args: str) -> str: + try: + result = subprocess.run( + ['git', *args], + cwd=self._directory, + env=self._env, + check=True, + capture_output=True, + text=True, + ) + except FileNotFoundError as exc: + raise FlatSnapshotError( + 'Git is required to read published flat objects in bulk' + ) from exc + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or '').strip() + if self._authorization: + detail = detail.replace(self._authorization, '[redacted]') + raise FlatSnapshotError( + f'Git {args[0]} failed while reading flat objects: {detail}' + ) from exc + return result.stdout.strip() + + def __enter__(self) -> FlatSnapshotReader: + if self._temporary is not None or self.pack_count: + raise FlatSnapshotError('A flat snapshot reader cannot be reused') + if self.revision is None: + self.revision = self.api.repo_info( + repo_id=self.repo_id, repo_type='dataset' + ).sha + if not isinstance(self.revision, str) or not _COMMIT.fullmatch( + self.revision + ): + raise FlatSnapshotError( + 'The datastore revision must be a commit SHA' + ) + self.revision = self.revision.lower() + self._temporary = tempfile.TemporaryDirectory( + prefix='eee-flat-objects-' + ) + self._directory = Path(self._temporary.name).resolve() + try: + self._git('init', '--quiet') + self._git('fetch', '--depth', '1', self.remote_url, self.revision) + self.pack_count += 1 + fetched = self._git('rev-parse', 'FETCH_HEAD^{commit}') + if fetched != self.revision: + raise FlatSnapshotError( + f'Fetched datastore commit {fetched} differs from pin ' + f'{self.revision}' + ) + self._git('sparse-checkout', 'init', '--no-cone') + self._git( + 'sparse-checkout', + 'set', + '--no-cone', + '/flat/objects/**/*.json', + f'/{_RETIRED_PATH}', + ) + self._git('checkout', '--detach', self.revision) + if self._git('rev-parse', 'HEAD') != self.revision: + raise FlatSnapshotError( + 'Checked-out datastore commit differs from pin' + ) + except BaseException: + self.close() + raise + return self + + def close(self) -> None: + """Remove the temporary checkout; downloaded Hub cache entries remain.""" + if self._temporary is not None: + self._temporary.cleanup() + self._temporary = None + self._directory = None + self._payloads.clear() + + def __exit__(self, *_exc: object) -> None: + self.close() + + def _path(self, path: str) -> Path: + parts = PurePosixPath(path).parts + allowed = path == _RETIRED_PATH or ( + len(parts) >= 3 + and parts[:2] == ('flat', 'objects') + and parts[-1].endswith('.json') + ) + if ( + not allowed + or '..' in parts + or '\\' in path + or PurePosixPath(path).as_posix() != path + ): + raise ValueError( + f'Not a published aggregate or retirement list: {path}' + ) + if self._directory is None: + raise FlatSnapshotError( + 'Open the flat snapshot reader before reading' + ) + local = self._directory / path + if local.is_symlink() or self._directory not in local.resolve().parents: + raise FlatSnapshotError( + f'Published object must be a regular file: {path}' + ) + return local + + def exists(self, path: str) -> bool: + """Return whether a permitted path exists in the pinned checkout.""" + return self._path(path).is_file() + + def read(self, path: str) -> bytes: + """Read a published aggregate or retirement list, resolving LFS explicitly.""" + data = self._path(path).read_bytes() + pointer = _pointer(data, path) + if pointer is None: + return data + if path not in self._payloads: + self._payloads[path] = Path( + self.api.hf_hub_download( + repo_id=self.repo_id, + filename=path, + repo_type='dataset', + revision=self.revision, + ) + ) + self.lfs_download_paths.append(path) + payload = self._payloads[path].read_bytes() + digest, size = pointer + if ( + len(payload) != size + or hashlib.sha256(payload).hexdigest() != digest + ): + raise FlatSnapshotError( + f'Git LFS payload failed SHA256/size check: {path}' + ) + return payload + + def read_many(self, paths: Iterable[str]) -> dict[str, bytes]: + """Read each requested aggregate without retaining an extra byte cache.""" + return {path: self.read(path) for path in paths} diff --git a/tests/test_flat_snapshot.py b/tests/test_flat_snapshot.py new file mode 100644 index 000000000..db0ce99a6 --- /dev/null +++ b/tests/test_flat_snapshot.py @@ -0,0 +1,343 @@ +"""Offline Git and LFS fixtures for published aggregate identity reads.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +from types import SimpleNamespace + +import pytest + +from every_eval_ever.cron import flat_snapshot as fs + +PLAIN = 'flat/objects/aa/plain.json' +LFS = 'flat/objects/bb/large.json' +UNTOUCHED_LFS = 'flat/objects/cc/untouched.json' +RETIRED = 'flat/retired.jsonl' + + +def git(directory, *args): + env = os.environ.copy() + env.update( + GIT_CONFIG_NOSYSTEM='1', + GIT_CONFIG_GLOBAL=os.devnull, + GIT_LFS_SKIP_SMUDGE='1', + GIT_ALLOW_PROTOCOL='file', + GIT_CONFIG_COUNT='3', + GIT_CONFIG_KEY_0='filter.lfs.process', + GIT_CONFIG_VALUE_0='', + GIT_CONFIG_KEY_1='filter.lfs.smudge', + GIT_CONFIG_VALUE_1='', + GIT_CONFIG_KEY_2='filter.lfs.required', + GIT_CONFIG_VALUE_2='false', + ) + return subprocess.run( + ['git', *args], + cwd=directory, + env=env, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def pointer(payload): + return ( + 'version https://git-lfs.github.com/spec/v1\n' + f'oid sha256:{hashlib.sha256(payload).hexdigest()}\n' + f'size {len(payload)}\n' + ).encode() + + +@pytest.fixture +def repository(tmp_path): + source = tmp_path / 'source' + source.mkdir() + git(source, 'init', '--quiet') + git(source, 'config', 'user.email', 'fixture@example.invalid') + git(source, 'config', 'user.name', 'Offline fixture') + plain = b'{"evaluation_results": [{"name": "fixture metric"}]}\n' + payload = b'{"evaluation_results": [{"name": "larger fixture metric"}]}\n' + files = { + PLAIN: plain, + 'flat/objects/direct.json': b'{}\n', + LFS: pointer(payload), + UNTOUCHED_LFS: pointer(b'{"untouched": true}\n'), + RETIRED: b'{"fixture": "retired"}\n', + 'data/unrelated/record.json': b'{"not_checked_out": true}\n', + } + for relative, data in files.items(): + path = source / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + git(source, 'add', '.') + git(source, 'commit', '--quiet', '-m', 'Pinned fixture') + pin = git(source, 'rev-parse', 'HEAD') + (source / PLAIN).write_bytes(b'{"later": true}\n') + git(source, 'add', '.') + git(source, 'commit', '--quiet', '-m', 'Later fixture') + remote = tmp_path / 'remote.git' + git(tmp_path, 'clone', '--bare', str(source), str(remote)) + cache = tmp_path / 'payload.json' + cache.write_bytes(payload) + return SimpleNamespace( + source=source, + remote=remote, + pin=pin, + tip=git(source, 'rev-parse', 'HEAD'), + plain=plain, + payload=payload, + cache=cache, + ) + + +class FakeApi: + endpoint = 'https://example.invalid' + token = False + + def __init__(self, repository): + self.repository = repository + self.info_calls = [] + self.downloads = [] + + def repo_info(self, **kwargs): + self.info_calls.append(kwargs) + return SimpleNamespace(sha=self.repository.pin) + + def hf_hub_download(self, **kwargs): + self.downloads.append(kwargs) + assert kwargs == { + 'repo_id': 'fixture/datastore', + 'repo_type': 'dataset', + 'filename': LFS, + 'revision': self.repository.pin, + } + return str(self.repository.cache) + + +def reader(repository, api=None, **kwargs): + return fs.FlatSnapshotReader( + api or FakeApi(repository), + 'fixture/datastore', + remote_url=str(repository.remote), + **kwargs, + ) + + +def test_one_shallow_pack_at_resolved_pin_and_sparse_checkout( + repository, monkeypatch +): + api = FakeApi(repository) + calls = [] + run = subprocess.run + + def recording_run(command, **kwargs): + calls.append((command, kwargs)) + return run(command, **kwargs) + + monkeypatch.setattr(fs.subprocess, 'run', recording_run) + snapshot = reader(repository, api) + with snapshot: + assert snapshot.revision == repository.pin != repository.tip + assert snapshot.pack_count == 1 + assert snapshot.read_many([PLAIN, RETIRED]) == { + PLAIN: repository.plain, + RETIRED: b'{"fixture": "retired"}\n', + } + assert snapshot.read('flat/objects/direct.json') == b'{}\n' + assert snapshot.exists(LFS) + assert not snapshot.exists('flat/objects/missing.json') + checkout = snapshot._directory + assert checkout is not None + assert not (checkout / 'data').exists() + assert (checkout / '.git/shallow').read_text().strip() == repository.pin + assert api.downloads == [] + assert snapshot.lfs_download_count == 0 + assert not checkout.exists() + assert api.info_calls == [ + {'repo_id': 'fixture/datastore', 'repo_type': 'dataset'} + ] + fetches = [command for command, _ in calls if command[1] == 'fetch'] + assert fetches == [ + ['git', 'fetch', '--depth', '1', str(repository.remote), repository.pin] + ] + assert all('--filter' not in ' '.join(command) for command, _ in calls) + assert all( + kwargs['env']['GIT_LFS_SKIP_SMUDGE'] == '1' for _, kwargs in calls + ) + assert all( + kwargs['env']['GIT_TERMINAL_PROMPT'] == '0' for _, kwargs in calls + ) + + +def test_only_touched_lfs_payload_downloaded_once_at_pin(repository): + api = FakeApi(repository) + with reader(repository, api) as snapshot: + assert snapshot.read(PLAIN) == repository.plain + assert api.downloads == [] + assert snapshot.read(LFS) == repository.payload + assert snapshot.read(LFS) == repository.payload + assert len(api.downloads) == snapshot.lfs_download_count == 1 + assert snapshot.lfs_download_paths == [LFS] + assert UNTOUCHED_LFS not in snapshot.lfs_download_paths + + +@pytest.mark.parametrize('corruption', ['wrong size', 'wrong hash']) +def test_lfs_payload_hash_and_size_verified(repository, corruption): + repository.cache.write_bytes( + b'bad' if corruption == 'wrong size' else b'x' * len(repository.payload) + ) + with reader(repository) as snapshot: + with pytest.raises(fs.FlatSnapshotError, match='SHA256/size'): + snapshot.read(LFS) + + +def test_supplied_pin_skips_revision_resolution(repository): + api = FakeApi(repository) + with reader(repository, api, revision=repository.pin) as snapshot: + assert snapshot.read(PLAIN) == repository.plain + assert api.info_calls == [] + + +@pytest.mark.parametrize( + 'path', + [ + 'flat/objects/aa/private_samples.jsonl', + 'data/source/private_samples.jsonl', + 'flat/objects/../retired.json', + '/flat/objects/x.json', + 'flat/objects/./x.json', + 'flat/objects\\x.json', + 'flat/manifests/example/entries.jsonl', + ], +) +def test_identity_reader_rejects_samples_and_other_paths(repository, path): + api = FakeApi(repository) + with reader(repository, api) as snapshot: + with pytest.raises(ValueError, match='Not a published aggregate'): + snapshot.read(path) + assert api.downloads == [] + + +def test_cleanup_after_consumer_error(repository): + snapshot = reader(repository) + with pytest.raises(RuntimeError, match='consumer error'): + with snapshot: + checkout = snapshot._directory + raise RuntimeError('consumer error') + assert not checkout.exists() + + +def test_missing_git_fails_loudly_and_cleans_up(repository, monkeypatch): + def missing(*_args, **_kwargs): + raise FileNotFoundError('git') + + monkeypatch.setattr(fs.subprocess, 'run', missing) + snapshot = reader(repository) + with pytest.raises(fs.FlatSnapshotError, match='Git is required'): + with snapshot: + pass + assert snapshot._directory is None + assert snapshot._temporary is None + + +def test_fetch_failure_cleans_up(repository, monkeypatch): + run = subprocess.run + + def failing(command, **kwargs): + if command[1] == 'fetch': + raise subprocess.CalledProcessError( + 1, command, stderr='fixture failure' + ) + return run(command, **kwargs) + + monkeypatch.setattr(fs.subprocess, 'run', failing) + snapshot = reader(repository) + with pytest.raises( + fs.FlatSnapshotError, match='Git fetch failed.*fixture failure' + ): + with snapshot: + pass + assert snapshot._directory is None + + +@pytest.mark.parametrize('checked_ref', ['FETCH_HEAD^{commit}', 'HEAD']) +def test_fetched_and_checked_out_commit_must_match_pin( + repository, monkeypatch, checked_ref +): + original = fs.FlatSnapshotReader._git + + def wrong_pin(self, *args): + if args == ('rev-parse', checked_ref): + return repository.tip + return original(self, *args) + + monkeypatch.setattr(fs.FlatSnapshotReader, '_git', wrong_pin) + snapshot = reader(repository) + with pytest.raises(fs.FlatSnapshotError, match='differs from pin'): + with snapshot: + pass + assert snapshot._directory is None + + +def test_non_commit_revision_rejected_before_git(repository, monkeypatch): + def unexpected(*_args, **_kwargs): + pytest.fail('Git must not run for a mutable revision') + + monkeypatch.setattr(fs.subprocess, 'run', unexpected) + with pytest.raises(fs.FlatSnapshotError, match='must be a commit SHA'): + with reader(repository, revision='main'): + pass + + +def test_lfs_filters_disabled_without_needing_git_lfs(repository, monkeypatch): + # An inherited filter that fails must never run during the sparse checkout. + monkeypatch.setenv('GIT_CONFIG_COUNT', '2') + monkeypatch.setenv('GIT_CONFIG_KEY_0', 'filter.lfs.process') + monkeypatch.setenv('GIT_CONFIG_VALUE_0', 'a-nonexistent-lfs-process') + monkeypatch.setenv('GIT_CONFIG_KEY_1', 'filter.lfs.required') + monkeypatch.setenv('GIT_CONFIG_VALUE_1', 'true') + (repository.source / '.gitattributes').write_text('*.json filter=lfs\n') + # Stage attributes only: existing pointer blobs remain ordinary Git blobs. + git(repository.source, 'add', '.gitattributes') + git(repository.source, 'commit', '--quiet', '-m', 'LFS attributes') + pin = git(repository.source, 'rev-parse', 'HEAD') + git(repository.remote, 'fetch', str(repository.source), pin) + repository.pin = pin + with reader(repository, revision=pin) as snapshot: + assert snapshot.read(LFS) == repository.payload + + +def test_auth_header_uses_environment_and_does_not_enter_git_arguments( + repository, +): + api = FakeApi(repository) + api.token = 'hf_fixture_secret' + snapshot = fs.FlatSnapshotReader(api, 'fixture/datastore') + assert ( + snapshot.remote_url + == 'https://example.invalid/datasets/fixture/datastore' + ) + config = { + snapshot._env[f'GIT_CONFIG_KEY_{i}']: snapshot._env[ + f'GIT_CONFIG_VALUE_{i}' + ] + for i in range(int(snapshot._env['GIT_CONFIG_COUNT'])) + } + assert config['http.https://example.invalid/.extraHeader'] == ( + 'Authorization: Bearer hf_fixture_secret' + ) + + +@pytest.mark.parametrize( + 'data', + [ + b'version https://git-lfs.github.com/spec/v2\n', + b'version https://git-lfs.github.com/spec/v1\noid sha256:bad\nsize 2\n', + b'version https://git-lfs.github.com/spec/v1\n\xff', + ], +) +def test_malformed_lfs_pointer_fails_explicitly(data): + with pytest.raises(fs.FlatSnapshotError, match='Malformed Git LFS pointer'): + fs._pointer(data, LFS) From 1a133696ff79fe87ba387bba18883d6c6646f4ba Mon Sep 17 00:00:00 2001 From: Jenny Chim Date: Wed, 21 Oct 2026 18:59:49 +0100 Subject: [PATCH 16/16] Backfill all inherited identities from bulk snapshots --- every_eval_ever/cron/README.md | 16 +- every_eval_ever/cron/flat_rebuild.py | 165 +++++++++------- tests/test_flat_rebuild.py | 271 ++++++++++++++++++++++----- 3 files changed, 334 insertions(+), 118 deletions(-) diff --git a/every_eval_ever/cron/README.md b/every_eval_ever/cron/README.md index 3a98e41d9..8735a4b1b 100644 --- a/every_eval_ever/cron/README.md +++ b/every_eval_ever/cron/README.md @@ -469,11 +469,17 @@ manifest core hash through the entries hash, so a supersession change publishes a new snapshot while an unchanged rebuild stays a no-op. Rows from snapshots older than either record ids or `result_identities` -serialize without identity fields until filled. The first fill is a manual run with -`--bootstrap-ids`: one bulk download of the aggregate objects -(`flat/objects/**/*.json`, size depends on the datastore) into a temporary directory, read -locally. Afterwards `--backfill-limit` (default 500, `0` disables) fetches -any stragglers one by one per run. +serialize without identity fields until filled. Every run fills all missing +identity fields. One shallow Git fetch reads the published aggregate objects at +a single resolved datastore commit; semantic comparisons reuse that checkout. +Git is required. LFS smudging is disabled, and only LFS-backed aggregate objects +actually needed by the run are downloaded individually at that same commit and +verified against their pointer hashes. Sample sidecars are never read for +identity backfill. The summary reports the pin, Git pack count, and required LFS +payload paths. Temporary checkouts are removed even if the rebuild fails. +`--bootstrap-ids` remains a compatibility alias for the normal behavior; +`--backfill-limit` has been removed. Existing daily diff, deep verification, +retention, and recovery reads keep their existing fetch paths. ### Reviewer comments and changes diff --git a/every_eval_ever/cron/flat_rebuild.py b/every_eval_ever/cron/flat_rebuild.py index d1d1057e2..7f767aa50 100644 --- a/every_eval_ever/cron/flat_rebuild.py +++ b/every_eval_ever/cron/flat_rebuild.py @@ -1,4 +1,4 @@ -"""Rebuild the datastore's flat view against the Hub, without a clone. +"""Rebuild the datastore's flat view using the Hub and pinned bulk reads. ``tools/build_flat_datastore.py`` in the EEE_datastore repository defines the flat layout and rebuilds it from a local checkout. That checkout no longer @@ -41,11 +41,11 @@ import os import re import sys -import tempfile import time import warnings from collections import Counter, defaultdict from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from decimal import Decimal @@ -63,6 +63,10 @@ # Imported as a module, not by name: the retry helpers are replaced in tests # and the commit path has to see the replacement. from every_eval_ever.cron import store +from every_eval_ever.cron.flat_snapshot import ( + FlatSnapshotError, + FlatSnapshotReader, +) from every_eval_ever.cron.submit import ( DEFAULT_BATCH_SIZE, DEFAULT_DATASTORE_REPO, @@ -86,11 +90,7 @@ #: decide between trim and pin; unscanned ones are conservatively kept. DEFAULT_SCAN_BUDGET = 20 DOWNLOAD_WORKERS = 8 -#: Inherited rows whose ids are fetched one by one per run by default; a -#: first fill uses ``--bootstrap-ids`` instead. -DEFAULT_BACKFILL_LIMIT = 500 BACKFILL_CHUNK = 500 -OBJECT_PATTERNS = ('flat/objects/**/*.json',) SUPERSEDED_REASONS = frozenset({'newer_capture', 'retired'}) _NUMERIC = re.compile(r'\d+(?:\.\d+)?') #: Excluded paths a maintainer has acknowledged; see ``exit_code_for``. @@ -356,6 +356,10 @@ class RebuildReport: verified_reserialized: int = 0 ids_backfilled: int = 0 ids_pending: int = 0 + bulk_revision: str | None = None + bulk_pack_count: int = 0 + bulk_lfs_download_count: int = 0 + bulk_lfs_download_paths: tuple[str, ...] = () superseded_captures: int = 0 superseded_retired: int = 0 ambiguous_series: tuple[str, ...] = () @@ -1086,28 +1090,29 @@ def backfill_ids( rows: Sequence[Row], fetch: Callable[[Sequence[str]], dict[str, bytes]], *, - limit: int | None, chunk: int = BACKFILL_CHUNK, ) -> tuple[list[Row], int, int]: """Fill record ids of inherited rows from their published objects. - At most ``limit`` rows (``None``: all), in ``legacy_path`` order, are - fetched, ``chunk`` objects at a time. Returns ``(rows, filled, - pending)``. + All missing rows are read in ``legacy_path`` order, ``chunk`` objects at + a time. Returns ``(rows, filled, pending)``. """ missing = sorted( (row for row in rows if not row.ids_recorded), key=lambda row: row.legacy_path, ) - todo = missing if limit is None else missing[: max(limit, 0)] filled: dict[str, Row] = {} - for start in range(0, len(todo), chunk): - batch = todo[start : start + chunk] + for start in range(0, len(missing), chunk): + batch = missing[start : start + chunk] contents = fetch([row.object_path for row in batch]) for row in batch: - filled[row.object_uuid] = record_ids( - contents[row.object_path] - ).apply(row) + data = contents[row.object_path] + if len(data) != row.size_bytes or sha256_bytes(data) != row.sha256: + raise FlatRebuildError( + f'{row.object_path}: published bytes differ from the ' + 'inherited snapshot; identity backfill cannot proceed' + ) + filled[row.object_uuid] = record_ids(data).apply(row) return ( [filled.get(row.object_uuid, row) for row in rows], len(filled), @@ -1115,25 +1120,6 @@ def backfill_ids( ) -def local_object_reader( - root: Path, -) -> Callable[[Sequence[str]], dict[str, bytes]]: - """Read published objects from a local ``snapshot_download`` tree.""" - - def fetch(paths: Sequence[str]) -> dict[str, bytes]: - contents: dict[str, bytes] = {} - for path in paths: - local = root / path - if not local.is_file(): - raise FlatRebuildError( - f'{path}: missing from the bulk object download' - ) - contents[path] = local.read_bytes() - return contents - - return fetch - - def parse_retired(data: bytes) -> list[RetiredEntry]: """Parse ``flat/retired.jsonl``; malformed lines raise.""" entries: list[RetiredEntry] = [] @@ -1674,11 +1660,64 @@ def orchestrate( allow_bootstrap: bool = False, dry_run: bool = False, verify: bool = False, - backfill_limit: int = DEFAULT_BACKFILL_LIMIT, bootstrap_ids: bool = False, now: datetime | None = None, ) -> RebuildReport: """Plan and, unless dry, publish one flat rebuild. Returns the report.""" + bulk: FlatSnapshotReader | None = None + try: + with ExitStack() as stack: + + def read_objects(paths: Sequence[str]) -> dict[str, bytes]: + nonlocal bulk + if bulk is None: + bulk = stack.enter_context(FlatSnapshotReader(api, repo_id)) + return bulk.read_many(paths) + + def read_retired() -> bytes: + if bulk is not None: + return bulk.read(RETIRED_LIST_PATH) + return download_bytes(api, repo_id, [RETIRED_LIST_PATH])[ + RETIRED_LIST_PATH + ] + + report = _orchestrate( + api, + repo_id, + read_objects=read_objects, + read_retired=read_retired, + retain_days=retain_days, + keep_newest=keep_newest, + scan_budget=scan_budget, + allow_bootstrap=allow_bootstrap, + dry_run=dry_run, + verify=verify, + now=now, + ) + if bulk is not None: + report.bulk_revision = bulk.revision + report.bulk_pack_count = bulk.pack_count + report.bulk_lfs_download_count = bulk.lfs_download_count + report.bulk_lfs_download_paths = tuple(bulk.lfs_download_paths) + return report + except FlatSnapshotError as exc: + raise FlatRebuildError(str(exc)) from exc + + +def _orchestrate( + api: HfApi, + repo_id: str, + *, + read_objects: Callable[[Sequence[str]], dict[str, bytes]], + read_retired: Callable[[], bytes], + retain_days: int, + keep_newest: int, + scan_budget: int, + allow_bootstrap: bool, + dry_run: bool, + verify: bool, + now: datetime | None, +) -> RebuildReport: if not repo_id.strip(): raise FlatRebuildError('repo_id must not be empty') started = time.monotonic() @@ -1729,23 +1768,9 @@ def orchestrate( inherited = [ row for row in old_rows if row.object_uuid not in removed_uuids ] - if bootstrap_ids and any(not row.ids_recorded for row in inherited): - with tempfile.TemporaryDirectory() as bulk: - api.snapshot_download( - repo_id=repo_id, - repo_type='dataset', - allow_patterns=list(OBJECT_PATTERNS), - local_dir=bulk, - ) - kept, report.ids_backfilled, report.ids_pending = backfill_ids( - inherited, local_object_reader(Path(bulk)), limit=None - ) - else: - kept, report.ids_backfilled, report.ids_pending = backfill_ids( - inherited, - lambda paths: download_bytes(api, repo_id, paths), - limit=backfill_limit, - ) + kept, report.ids_backfilled, report.ids_pending = backfill_ids( + inherited, read_objects + ) backfilled = {row.object_uuid: row for row in kept} build_input = [backfilled.get(row.object_uuid, row) for row in old_rows] object_cache: dict[str, bytes] = {} @@ -1755,7 +1780,12 @@ def existing_object(object_path: str) -> bytes | None: if object_path not in listing: return None if object_path not in object_cache: - object_cache.update(download_bytes(api, repo_id, [object_path])) + if object_path.startswith('flat/objects/') and object_path.endswith( + '.json' + ): + object_cache.update(read_objects([object_path])) + else: + object_cache.update(download_bytes(api, repo_id, [object_path])) return object_cache[object_path] build = build_rows(build_input, diff, contents, existing_object, listing) @@ -1768,9 +1798,7 @@ def existing_object(object_path: str) -> bytes | None: report.excluded_paths = build.excluded_paths retired: list[RetiredEntry] = [] if RETIRED_LIST_PATH in listing: - retired = parse_retired( - download_bytes(api, repo_id, [RETIRED_LIST_PATH])[RETIRED_LIST_PATH] - ) + retired = parse_retired(read_retired()) supersession = apply_supersession(build.rows, retired) rows = supersession.rows report.ambiguous_series = supersession.ambiguous_series @@ -1991,6 +2019,12 @@ def summary_lines( f'- evaluation_id backfill: {report.ids_backfilled} filled, ' f'{report.ids_pending} pending' ) + if report.bulk_revision is not None: + lines.append( + f'- published aggregate snapshot: `{report.bulk_revision}`; ' + f'{report.bulk_pack_count} Git pack transfer(s), ' + f'{report.bulk_lfs_download_count} required LFS payload(s)' + ) if report.pending_collections: lines.append( f'- automatic supersession skipped in ' @@ -2056,6 +2090,7 @@ def summary_lines( ) for label, paths in ( ('Excluded records', report.conflicts), + ('Required LFS payloads', report.bulk_lfs_download_paths), ('Stale acknowledgements', stale), ('Ambiguous series', report.ambiguous_series), ('Regressed series', report.regressed_series), @@ -2133,23 +2168,12 @@ def main(argv: list[str] | None = None) -> int: 'loudly. Slow; intended for a periodic sweep, not every run.' ), ) - parser.add_argument( - '--backfill-limit', - type=int, - default=DEFAULT_BACKFILL_LIMIT, - help=( - 'Max inherited rows per run whose ids are fetched one by one ' - 'from their published object (default: %(default)s; 0 ' - 'disables).' - ), - ) parser.add_argument( '--bootstrap-ids', action='store_true', help=( - 'Fill the ids of every inherited row from one bulk download of ' - 'the aggregate objects (about 1.4 GB) instead of per-file ' - 'fetches.' + 'Compatibility alias: every run fills all missing inherited ' + 'ids from a pinned bulk read of published aggregate objects.' ), ) parser.add_argument( @@ -2174,7 +2198,6 @@ def main(argv: list[str] | None = None) -> int: allow_bootstrap=args.allow_bootstrap, dry_run=args.dry_run, verify=args.verify, - backfill_limit=args.backfill_limit, bootstrap_ids=args.bootstrap_ids, ) except FlatRebuildError as exc: diff --git a/tests/test_flat_rebuild.py b/tests/test_flat_rebuild.py index 06fc7b2b8..263fed459 100644 --- a/tests/test_flat_rebuild.py +++ b/tests/test_flat_rebuild.py @@ -3,9 +3,12 @@ from __future__ import annotations import json +import shutil +import subprocess from dataclasses import dataclass, replace from datetime import UTC, datetime, timedelta from pathlib import Path, PurePosixPath +from types import SimpleNamespace from typing import Any from uuid import uuid4 @@ -14,7 +17,7 @@ from huggingface_hub.errors import EntryNotFoundError from every_eval_ever.cron import flat_rebuild as fr -from every_eval_ever.cron import store +from every_eval_ever.cron import flat_snapshot, store NOW = datetime(2026, 9, 6, 12, 0, 0, tzinfo=UTC) OLD = NOW - timedelta(days=100) @@ -98,6 +101,54 @@ def create_commit( self.files.pop(operation.path_in_repo, None) +class FakeFlatSnapshotReader: + """Explicit offline substitute for the published aggregate transport.""" + + def __init__(self, api, repo_id): + self.files = dict(api.files) + self.revision = 'a' * 40 + self.pack_count = 0 + self.lfs_download_count = 0 + self.lfs_download_paths = [] + self.reads = [] + self.closed = False + + def __enter__(self): + self.pack_count += 1 + return self + + def __exit__(self, *exc): + self.closed = True + + def read(self, path): + assert path == fr.RETIRED_LIST_PATH or ( + path.startswith('flat/objects/') and path.endswith('.json') + ) + self.reads.append(path) + if path not in self.files: + raise fr.FlatSnapshotError(f'Missing published object: {path}') + return self.files[path] + + def read_many(self, paths): + return {path: self.read(path) for path in paths} + + def exists(self, path): + return path in self.files + + +@pytest.fixture(autouse=True) +def bulk_readers(monkeypatch): + readers = [] + + def create(api, repo_id): + reader = FakeFlatSnapshotReader(api, repo_id) + readers.append(reader) + return reader + + monkeypatch.setattr(fr, 'FlatSnapshotReader', create) + return readers + + def committed_paths(api: FakeApi) -> set[str]: return { operation.path_in_repo @@ -1615,24 +1666,43 @@ def test_unreadable_result_identity_cannot_establish_coverage(results) -> None: def test_count_only_index_rows_are_backfilled_before_supersession() -> None: old = capture('a', '100').to_dict() del old['result_identities'] - row = fr.Row.from_dict(old) + data = captured_record('a', 'hle/m/100', '100') + row = replace( + fr.Row.from_dict(old), + sha256=fr.sha256_bytes(data), + size_bytes=len(data), + ) assert not row.ids_recorded calls = [] def fetch(paths): calls.extend(paths) - return {row.object_path: captured_record('a', 'hle/m/100', '100')} + return {row.object_path: data} result = fr.apply_supersession([row, capture('b', '200')]) assert result.pending_collections == ('hle',) assert all(row.superseded_by is None for row in result.rows) - filled, count, pending = fr.backfill_ids([row], fetch, limit=1) + filled, count, pending = fr.backfill_ids([row], fetch) assert calls == [row.object_path] assert (count, pending) == (1, 0) assert filled[0].ids_recorded assert filled[0].result_identities is not None +def test_identity_backfill_rejects_published_object_drift(): + data = captured_record('a', 'hle/m/100', '100') + row = replace( + capture('a', '100'), + ids_recorded=False, + sha256=fr.sha256_bytes(data), + size_bytes=len(data), + ) + changed = data.replace(b'100', b'200') + assert len(data) == len(changed) + with pytest.raises(fr.FlatRebuildError, match='published bytes differ'): + fr.backfill_ids([row], lambda paths: {row.object_path: changed}) + + def test_retired_fuller_row_does_not_block_series() -> None: rows = supersede( [ @@ -1907,60 +1977,177 @@ def seed_legacy_snapshot(files: dict[str, bytes], uuids) -> None: seed_pointer(files, manifest) -def test_orchestrate_backfills_ids_in_bounded_passes(tmp_path: Path) -> None: +def test_orchestrate_backfills_all_ids(tmp_path: Path, bulk_readers) -> None: files: dict[str, bytes] = {} uuids = seed_captures(files, 'hle', ['100', '200', '300']) seed_legacy_snapshot(files, uuids) api = FakeApi(files, tmp_path) - report = fr.orchestrate(api, 'org/ds', now=NOW, backfill_limit=2) - assert (report.ids_backfilled, report.ids_pending) == (2, 1) - assert report.pending_collections == ('hle',) - assert report.superseded_captures == 0 - report = fr.orchestrate(api, 'org/ds', now=NOW, backfill_limit=2) - assert (report.ids_backfilled, report.ids_pending) == (1, 0) + report = fr.orchestrate(api, 'org/ds', now=NOW) + assert (report.ids_backfilled, report.ids_pending) == (3, 0) + assert report.pending_collections == () assert report.superseded_captures == 2 + assert len(bulk_readers) == 1 + assert bulk_readers[0].closed + assert report.bulk_pack_count == 1 index = published_rows(api, f'{fr.INDEXES_PREFIX}/hle.jsonl') assert {row['superseded_by'] for row in index.values()} == { uuids[2], None, } - report = fr.orchestrate(api, 'org/ds', now=NOW, backfill_limit=2) + report = fr.orchestrate(api, 'org/ds', now=NOW) assert report.noop assert report.ids_backfilled == 0 + assert len(bulk_readers) == 1 + assert report.bulk_pack_count == 0 -def test_bootstrap_ids_reads_one_bulk_download(tmp_path: Path) -> None: +def test_bootstrap_ids_is_compatibility_alias( + tmp_path: Path, bulk_readers +) -> None: files: dict[str, bytes] = {} uuids = seed_captures(files, 'hle', ['100', '200']) seed_legacy_snapshot(files, uuids) - api = FakeApi(files, tmp_path) - calls: list[dict[str, Any]] = [] - - def snapshot_download(**kwargs: Any) -> str: - calls.append(kwargs) - root = Path(kwargs['local_dir']) - for path, data in api.files.items(): - if path.startswith('flat/objects/') and path.endswith('.json'): - (root / path).parent.mkdir(parents=True, exist_ok=True) - (root / path).write_bytes(data) - return str(root) - - api.snapshot_download = snapshot_download - fetched: list[str] = [] - original = api.hf_hub_download - - def tracking(repo_id: str, repo_type: str, filename: str) -> str: - fetched.append(filename) - return original(repo_id, repo_type, filename) - - api.hf_hub_download = tracking - report = fr.orchestrate( - api, 'org/ds', now=NOW, bootstrap_ids=True, backfill_limit=0 + normal = FakeApi(files, tmp_path / 'normal') + alias = FakeApi(files, tmp_path / 'alias') + normal_report = fr.orchestrate(normal, 'org/ds', now=NOW) + alias_report = fr.orchestrate(alias, 'org/ds', now=NOW, bootstrap_ids=True) + assert normal.files == alias.files + assert replace(normal_report, duration_seconds=0) == replace( + alias_report, duration_seconds=0 ) - assert len(calls) == 1 - assert calls[0]['allow_patterns'] == ['flat/objects/**/*.json'] - assert calls[0]['repo_type'] == 'dataset' - assert not any(path.startswith('flat/objects/') for path in fetched) - assert (report.ids_backfilled, report.ids_pending) == (2, 0) - assert report.superseded_captures == 1 + assert len(bulk_readers) == 2 + assert all( + reader.closed and reader.pack_count == 1 for reader in bulk_readers + ) + + +def test_bulk_reader_closes_and_translates_errors( + tmp_path, monkeypatch, bulk_readers +): + files = {} + uuids = seed_captures(files, 'hle', ['100']) + seed_legacy_snapshot(files, uuids) + + def unreadable(self, paths): + raise fr.FlatSnapshotError('Cannot read pinned object') + + monkeypatch.setattr(FakeFlatSnapshotReader, 'read_many', unreadable) + with pytest.raises(fr.FlatRebuildError, match='Cannot read pinned object'): + fr.orchestrate(FakeApi(files, tmp_path), 'org/ds', now=NOW) + assert len(bulk_readers) == 1 + assert bulk_readers[0].closed + + +def test_all_id_backfill_and_semantic_comparison_share_real_git_pack( + tmp_path, monkeypatch +): + if shutil.which('git') is None: + pytest.skip('Git is required for the local bulk-reader fixture') + count = fr.BACKFILL_CHUNK + 1 + files = {} + uuids = seed_captures(files, 'hle', [str(i) for i in range(count)]) + seed_legacy_snapshot(files, uuids) + touched = fr.object_path_for(uuids[0], suffix='.json') + unused = 'flat/objects/unused/unused.json' + sample = 'flat/objects/unused/sample.jsonl' + files[unused] = b'{"unused": true}' + files[sample] = b'{"sample": true}\n' + files[fr.RETIRED_LIST_PATH] = b'' + changed = f'data/hle/dev/model/{uuids[0]}.json' + files[changed] = json.dumps(json.loads(files[changed]), indent=2).encode() + + def pointer(payload): + return ( + 'version https://git-lfs.github.com/spec/v1\n' + f'oid sha256:{fr.sha256_bytes(payload)}\n' + f'size {len(payload)}\n' + ).encode() + + work = tmp_path / 'source repository' + work.mkdir() + + def git(*args, cwd=work): + return subprocess.run( + ['git', *args], cwd=cwd, capture_output=True, text=True, check=True + ).stdout.strip() + + git('init', '--initial-branch=fixture') + git('config', 'user.name', 'Fixture') + git('config', 'user.email', 'fixture@example.invalid') + for path, payload in files.items(): + target = work / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes( + pointer(payload) if path in (touched, unused, sample) else payload + ) + git('add', '.') + git('commit', '-m', 'Published aggregate fixture') + pinned = git('rev-parse', 'HEAD') + remote = tmp_path / 'remote repository.git' + git('clone', '--bare', str(work), str(remote)) + + class PinnedApi(FakeApi): + def __init__(self, root): + super().__init__(files, root) + self.info_calls = 0 + self.pinned_downloads = [] + + def repo_info(self, *, repo_id, repo_type): + assert (repo_id, repo_type) == ('org/ds', 'dataset') + self.info_calls += 1 + return SimpleNamespace(sha=pinned) + + def hf_hub_download(self, repo_id, repo_type, filename, revision=None): + if filename.startswith('flat/objects/'): + assert filename == touched + assert revision == pinned + self.pinned_downloads.append((filename, revision)) + else: + assert filename != fr.RETIRED_LIST_PATH + return super().hf_hub_download(repo_id, repo_type, filename) + + readers = [] + + class ObservedReader(flat_snapshot.FlatSnapshotReader): + def __enter__(self): + super().__enter__() + self.checkout_path = self._directory + assert not (self.checkout_path / sample).exists() + return self + + def create_reader(api, repo_id): + reader = ObservedReader(api, repo_id, remote_url=remote.as_uri()) + readers.append(reader) + return reader + + monkeypatch.setattr(fr, 'FlatSnapshotReader', create_reader) + normal = PinnedApi(tmp_path / 'normal') + alias = PinnedApi(tmp_path / 'alias') + reports = [ + fr.orchestrate(normal, 'org/ds', now=NOW), + fr.orchestrate(alias, 'org/ds', now=NOW, bootstrap_ids=True), + ] + assert normal.files == alias.files + for api, report in zip((normal, alias), reports, strict=True): + assert report.ids_backfilled == count + assert report.ids_pending == 0 + assert report.superseded_captures == count - 1 + assert report.bulk_revision == pinned + assert report.bulk_pack_count == 1 + assert report.bulk_lfs_download_count == 1 + assert report.bulk_lfs_download_paths == (touched,) + assert api.info_calls == 1 + assert api.pinned_downloads == [(touched, pinned)] + assert any(pinned in line for line in fr.summary_lines(report)) + assert any(touched in line for line in fr.summary_lines(report)) + assert len(readers) == 2 + assert all(not reader.checkout_path.exists() for reader in readers) + + # The unchanged manifest still reads the reserialized data's old object. + noop = fr.orchestrate(normal, 'org/ds', now=NOW) + assert noop.noop + assert noop.ids_backfilled == 0 + assert noop.bulk_pack_count == 1 + assert len(readers) == 3 + assert not readers[-1].checkout_path.exists()

t|!EI-lnv64wW#}w+MQa?2$yvYk0aZbp1)J*RuCozAJ6!fQt zt9#_fKUG@z7>WBOQApqJox1NlpzBAFpD{Ns3}clj5V+a`|jiNvy23 zOU<0B{n)ucs$|8WPI3R&s#Kk_DdA(6 z&^k@~!b)u~A$&J;a(M0PUgbBQ^~-yG7S|;xA6R|t1ZDR}D2sM-0{nq$wh2pcvtSlJ zP}R1RyWOu+POgNWeEfoPCutKb9D(ZP?oQS8vSJCWU5M`ly;2^#O5n+&Uj6cgr|TK8 zcWuQJ)3**)^ilY%RIU1Wx(0@V2=qZ}Ql8+&a=l`i0(Bo&E%UUmmmg5d^Tg%KO+`jO z>;0c{1=Nk!F{*)jQYgJzob}`+&{~yarP}?WO2;OS#QWEdJtPNGopNgaTS6_iiygnRa9Qk}rSTJ~i~U(h?yi>XJ~br&lO80me=F3$%m!!E^zk~-`s zg?ZHr#Gj^3tXlPYne^?yO!<0TPg1{Lye}(Ye_EH$Li^9B<5;Ofb9z3u`Q++Nk=S?S z=L$?mB8@${GA5UeRF8tx=gglm&V8bg#Qc#!>Xe}&nSo{9>SDc|nd8l&#c5$>Uar4P z6Szq#ffw>vY~(U4ikB#5mFUF1RiFi zq#)*I-gZOD4uKp5fOlATMXSE~Lkk-jY*UJPKtyH5FqD3fUs9 zihjA~{(!UjI*AipFZcml;yV0}kG?!P!=)@gB#U&WS}J;;qzFfD)|rPN%mp(`f}A|y zqI$9@bwiUx><-`~MPg+XLID^=(!NAZhW-pj6BTDg(bc>$8GUBbiaTk2oKv(fWmieE z99vQcih%a@sp4yv-}r{|XvNDwVi(|_?^5M#L}@2XYjL8^D_${Zhx1zz=v%2y=jSUP z(kPO!0b3Bvahj-`#M`v;+Q=tq*ojZn^eM|nZ?$4ZfN;a9sJ@3XYN|>AS(Og!i94#a zpN?!N)ftv-W_$L8W0*iIYl|ux)%GG6z|dbr%gh@l;fU+suqk%~az>mAlWoqVBulfMbgytKar(`vMPFMLBj-5 zK~xwx9w{nRq#5J9g%4tmbmRTK^kA_)DzOP7OmbsTEK7#io~zm}vpAE3%*oOD zG*XiARZ4t~nqTPLQHBDHJ+yyYo}7GqncC0sK8xroiLaWfU?mWg6I{yiC(7e4Fzl5G zwYXW0DNtS3xA#`6g%olJ$z!c>k2AmdLfTDzc~*6Nx16CM@_wI4U1UWiD4OT*?V34hSn;G5Dal8jK*5 zIgksmH&FnoXQzeGSJ}O1^Q139UyB^b&7Iar#2~tRz|)^q^H#F_E;8J+kZvFmU8S3! zsj9O{F}rYwO|f)pRu;TL5Sxgeg8ap3GLo@^Kq#dGULfGkxQV=YyMFRQi~lIOk9rBW+iA#V`KP$MS-Opmw-Z%MDx2m%Rxlz3n8aK>_@ zr-g{`vka+dJGsOfKjcP~7Wz3kJLnHGmSe3hoWvNJdMy{ILZYNI3qt>nL{WG%um|ok zMJ`V|frD^_)~tmOWVsk@iW>jvsXs|3-n=3Ba?K2kcX`kxvvjpU`WIT}HxXJ}b|6 znYwC7=CZ)7BTN*o4o?m>FNGmzgz2ID04+X%Y$BBuKnP!WT0W#nZr2z;5GRF?0iu-a zPuPQUlobS3Dz}Vb2S6|F5gJM*9j2UZBH#CYoU-pGx&iPcCux#IP*kp?q$)EWPPjK* zia4ngwm2bCI6>!Yydy^QB;bR6=R263fl?;JOzdGSs35WHXv6(wqWuIRmYI)P6yOC? zk_q%ZR*Mn={x%w;#v%XPT+|;q(QBg>mY&4U;nB_qbe&}R&GkF?(j2Kmd;~m-PPBH! z77n_tIIW-UPLzb2!odz*Z*1>YL=uX;XJ4LRs+{tAJPYokh=czt>4W2_HQu*R#LDWYaJyVDxXiDy^$gCvc%6?<({O(4m3L9_!N5(`R+XC zKrmOqTCk@VN=_9n=odN8n7s#h$C#rx$(m#r{JsLq$CaFFl*f$DTay(&QW0MXK4i zj)>KiQAH@$hwJ9a{NFDX1sHGWKLxB(6!?}tDNf|Vz|X&UHVxbOS7*-Xyn)3aX_&_I zm&w~_?Wqn=1kZocerDH>7u~`}zih#OzUw*@*G4y){Z~tV^g0I*Mkvitv7CX~bt@8w>7y!Ab z@(JfMm@Xp+a17)NkU!|Ab~XTMn{pBap*=*etEcQj{ER@J`P4Rw=Oc>Y9Mf$@fQahG zj>wbg*)I8WU!nkx$!e}pbf+xuTE{xmv?op0wT%D#kN?%}s4e!)X48<5&l}S?4x*>+ z_Q-cyemH4sYO8DLo%WU1QPh4%Ydqie1Z)g!P9Pwy|I99CJVcL2F$0*%MHKmXrbwa# znCKS3m=D4^9t_O{0R5PNB>K<)_&-7jBpeec&Mq#$W-sT$dB`5k9M1tv5Z1N9JREUU zPA;x4*ehp3)px#w!yHdb7P(j~TK3Eayox+pO9E~si)jPoqh6FL_0mqaD^ik)DKIg` zc9fx*lTw_MBIlGr?t>0n*bd`o_RgL~xInUJ!{C3fh-6cGuM zCAXLt{F?4mhNQhS#Lx>&@#S*Xx3;6GrW(C=WNKX#a8S1tRqORFrBjNm8=ft|$^4o9 z1-1fHj7^sUG%xHBy+-lv6`#hSs(g>%;{A#pI{5?tiUV~4I?eIv6nI!sKw0>qi_WAD z?933D_0j{VMRXU=)AN0iQ^gR?LO1fYR?> z01Lwd;tWd#pf6qve^ib_(ga5}o2RoG5K2gn@!!LcI{Q%QVw#Y?;*OwXQXPl_$bFQJj&k90uT zl+n;1tzhj!8PCy6sers@IG z9-Q4P_8k2H5Zz=eq0IpshB+$tZXOFesM!IEyOXC#)UdOaGXu{l2vS8e8NEjx50t_6 zJ;hG4r`7Z$H85A^S&Ezvax1At&PGqg$02674+(KQIW-geOHTpD%od;M{VGq5zV3%-Ax({4)S^99jY1^mkqXXM)CB#(a{@l#_`_uyrJvq zrcPl#3xalom`vpb=2oQX4at=@H4FU(!lDw4;4=~JgtQeVgswR7Sd{dZ_L?7I5!yjn zw`kpW#>!pkMoUx1(B*Xde6FbZ zBh6(9;ulCbk?(CE^kw?oBMnR^DVjO)j@8oXy2j@oX<$1DTzQG7gdCYb)V2ZdobB_t z2m;Vy=<&itiq!tJP7uWv?Eo8v9zD{z;qlZTVeOOUjuVnS_((%z*4OQ0evf%!A7?NU zns?A7c;t~roA`Iz$NCmK@F>4g3F3cT47{7WXbAw(&WV=_I*zzL-U9$f_a~t}+rF}E zJ_NNPek3GGRL)=hLDT{&K3Gow)&lw}NS*C8Sd30BAQk*FaBc6lfq5Cv8=j9PSUJXB z$;yG@4pI!713Jd0lP-DI847}aH(~cP*NepLr0s2CkV>qps@f(oMhY&PnF%iV9_=VE z{QL8U%2WpWybj-uL5RjVdmuoe=yVIxEC3PHcfUm0KlxTk`fOS0SE$Ji?EsWtgkIt6 zUwy|*zn4pK=R=JB7B7k&0@+mNfkGZ_4WX7#ozaLx!#JGdd|V%9j!Q>z*+F3+U=U!5 zI1se2x~x0Z)H@VgLD7eA3%_af|8Djhg64TnKUfv(rE2g{A=TVU#do#V@LN)V#FyAv zuvtHi+N7-w(YpnA2t!Uwb*3*~`oU6EHTi%47p*xRCdFJb-wU@wiUZ`~9Ury65T=vW zb*vtHbK7BGJxBL+cyL%0t{>;@eDQDW>}KKMmEV&uR>)b^dEpTP+`0v|(xWMo4CO78 zbKVZ1UDW=1%=zdY9HDa?b-72H{~H?k()(hI5CRl|u9=>nojo~!{`WhqlY4&VH)H&S zdCr}4J6d`}`qf4cvore3{Ie%d&Yt|<(fCUji`DF3@l=;>jENi?78?tK^|G zrx`tY8u9o>bcLUjZvXKQpkGgZ|LP2Nyw(NMl7IXo_u>5Szdw1>a-g?!_bWLB@CT|( z{0)W-oB>Z~;$1^rtl`6rCr`fm9f0r^fL8$VS^l1l%~59$yuWzAHSj)f>;QQTyA6zY z34kcVft!c0vF%@|Rr@t*=QlteN3e6ebv1Z9eD|I8ruA!-Z^Q3Vusm;!_}HGiF=7Xr zzIa&dZRn!exApR0uK4`=r9qtlqYD5}{w~UN3siJ?BtAaOfByaFX0>szea{2jt}wD?8J!&?_bd}e{y7Cj=>IC+p{m2un$3YQdgq@L136_WG8U-yPul0soPQSD0%iYh$$ZzNuSgS2cTGtB*jF4KOnE zwZq*J9n7o+iV?+yZ8$OC`u@b_4?8a3ax?%iqt#=tQR&}~BE6oWcDJ4v%WCy4t-`*@ zflYQ1PJD0IVd(<4Z5y`JZ|O$&(>g3#%P_QFS2Mes+O@hL`mE@!PPbzjy}r@2y72m; z&x!$vV3?|<1MW=$*l}VyK(Mx-7`@f+niX~@ zH+V}Erj@5!4{L~QE)OFL+bfX78#i98i2ST7h!*UFt)zA0-bZ0EYbztWxFiWZ*}$lr-i8|>RblMk$vF9YG3=% znA&NnR>$m{eO*UE!1&OTK(_@VkY@FII`9<`h(0c+UI+G&|J#GnHS}+p+1eqx_WFIZ z*V~G(y%y}vr-iRtt7m}L1zfn_x2)cWwvmA73@V2SLX_24E#rfuD^Q_cUsFw8>zbxf z+?&ey^^<glH5wRE#r_sl#TXN?x}$6m(*egmrCLG`Y* zHvr~wOmzA3=@ueJz zt_0=NIOX4Wx~{Ixt-a({K^#NZ zkXzk4*#PY zN4jvoz-YJlEe@PGW32l_0Q~qkLT_-7+u^sNJ))oyyucK2M_&#l?7mf%6le{0ufdoN zwiGY-y7&Yy8m=hz1I$|);Z3&sM|9!v1L5?2-uT0-i*DEI$S)-*;IS5Ztv+kxw<|rs znkx2@bg)b1tM{#p_*UHS>x_RkGD<1G`kB!Kbtlbm2zLa_UhAvl3?mdK$9S)L~JOS+aNGZQ3Q z_|DBk_v{ZF3GkB@=d`;1c6x1ZyWL=UtQn0!f3w5ThQp&XW8*|?OsG9s-vLQw>+XQy z5ud^TQ1DAOP#63O8~eNg55|ANt?F(7a@xnP+94R>a zG~*moX0D>aUra>;ss?BTddh5w`Id5*Li{7`AC#WCYGL^PYz-Ue(#tV8P8@uhFr~O7)oKsjUqx*Gz{B$@(g+MvKf6I`(uy2Y7xu@ z%E-W;mDta;qh0356y01tj-P7%R#!Dm!!Y|@%jopY?jOI3{ottqn2dK#9j(*rS(^UG zuSiyUs+qz4S0emNya&B^;&=*mrgXC}uoL{eLZ_#ShDU_{7AR{nt6VyW4W#X3fJxU| zosO#ZdkBG##1`SIX%XNPBsc&`j5%D|5JldlH;y#k{D$X`4q#?MsxAM_gBvrhuD-EOOA=;#rq zS$bFPTN1Krt8bcpz)Y=gnW|xZ25emtsHy^#TDN7Iy}oLxn%332{eJf|ENjBD-%2TD zx6?C`93n`%MyuC1EcCe1I)-WWJ|dM|6-Yk{J26@v%y_11eZhU@VK%Imjy_3UJj%Li z^iLd4o48581!&YUtgc~MeN#QIbNVR&b`vu}{4+G8-v_Zz)h+WgVE1JvI!wyw_uzv~VwQ3Z)Y(h{B&*jlJ9@9Db+mp5 zl#x>h(+19A=`Hlt?&w|1G68s znt`U5ZeQ0wgR-u8-3N}5JN|kKSfi)*)UKtffT6vv^)bck6)W#2L?n$)tKSDEV*x z36|OzPBED?^zEaMvga<@fBwh+W**ipVv7)PfO3ORF&%;*QoZpAt9BH-1EdI-RLSTC zmIJp%2{V`5cbp2kGaRhN&Tjmd4AW{wY$S?=NA`@fJI5||2fsLgV+)T;JG(q=Jz+PL zTPvdCO6bYGgvqjhe=1UL{X06gQ%!jE`!04ymp-1T7cma)2_-qiG*48*%aavrlOH@; z?7{b1xo$@OaE?FczGAHPw>b}&*W4v{k_*>8`)~G7m`3b>o;Uu``1z^l$7ikBAHv{w z=!0_mj+@T@U(o#AbxneF)*UvJA8 z<_4qvI~N@Rye{H6bOvzJ&l))4#*?qsS~+8O_9Zp@$3JBA^K%9Zdjimp=b`si=|EyX zV5kISMJSzNPa8&)P5<;K_N92+f`@Z6E!)V(ld=^v##^#z_$!fZ>#Sr2I5Ti<9wPVfvVu5XWU m%X6KieNav_dY>jXgLeaew8YnGJaeBj_WuEWX?0CTnGgUMBi`!( literal 0 HcmV?d00001 diff --git a/tests/data/vals_ai/saved_pages/mmmu.html.gz b/tests/data/vals_ai/saved_pages/mmmu.html.gz new file mode 100644 index 0000000000000000000000000000000000000000..1c805653ec3664ddc84b560c179f3a84ac5646ad GIT binary patch literal 24346 zcmV(^K-Iq=iwFP!00002|Lwi&cH2g_F!(=DQDG7v@`M0!2Pj!i%d(w#+Dd#ZC*A!y z{?s5@A`yWA1Aw|%(OTyr=FhA(^8oW2^C;&@X78#3h&v(4j?|sxB9cH|cI|uJ_I~{N zvmXxL{^Rv?vPhQxlgId%_--)Y>(QV`W^Uq2Ct)jO z3n`vPlm_I)rx)yxG!1=n5(bGByFpAQ?(}pXh3jA@P28CFo{+~2>du}#Ued%R(}f$w z6dL~V?F-52iKl_PqD1rWn)OG74M8j=jZ40JPhYP^fO<&fjg&>JPntlIs6{|5`W{( z_P&*#KK=Vo+V`)Xeyc)bBin@xgE^1SD4c{zoWmvnOfQBfq3?(1MbytIuMa@y%0ItY`C&xKPp%)6{g?j{ z6E~VJyfZo@hlhthlGjm4e4s6jCZQY6WO5KLS1wdouKmPYhBMbE@RveuK!&#j+)FO- zMM5tU^6Q$$*k3#ZOePBw(HLsMuW1-3&{HrY0N^5=g?>1{lv8+DQMjT}a=F(F=R1j) z_%sJ_2Gy$#RIH`LO-tmrbUd|9GAMIR=K1Ih>dCG*3g@E<4W^5w8=c0Z<#M@}SHZlw zxgEy2xkgn@QCf;GC<bVq0!D1 z#+fS{ub0sH<*^S!NRK(mYwLX+1}nDtuS2vV+;@@;-wRHmTOX9r6vvIkgYVenC_bBi zeX#_V09D*OanE>(MMO{bdL!{$nn8;)lq|I6u2w#kl5jm;NW50;eV|~ebWOQ{|Fc%g z1uIawJ`6DcjhljoHN2^dSQ|xg%mHNVu-6kbB2NKZSr-%d5o1vqi{{&2!QarGZH5FG zj7LWyT?8FCC0^jZ1_PgX zzXYQt|1GV=11X~OHORn6@AxT=PN`2X55piV{E*( ztCp-f1?*4VAPj(#ehOnZeIq=IDl_=AuYZYw)tivZQaFO(OvR32if-YS-jKEI! z{=#(BP7h2uxIzg$!6CqdI+2FAdDXDXWofNXV8MCXqKdmB}Iis z&)mp!p~@k3<8_4Z-}RuOb>vPj;q?KGmAnkdi2Sfd>b!!c0(kTOdf1(~PhqUHX|!HW zUU@OL0#;~=U!VIF9X#lBg9Uv465d0xRS5QqQCiK=@ifDvXlEA2UkEBAe$FIqi5iTpFlH|Imx3D z`AZT@-aG)Qf#1B`%cJ9G&wqk~tOEMlpJ$DceljF9b|7kgSFz+GWqEe0+TL z6S(&57a@J&^}}#MIOgQhBT{?<>YW)z0O|6zl{j^r40SJLmvy}@qoy+-!-NGSU>n1l4gF#3J$FnJfgQ{Dq8 z00jID->L5h8C(Lo~WFfq#gzRte&`zr)iW_pGSLF@N8v-Ac78T)_|Jrv;4FYcL}zRHiKzml1e z`GFZ3ERMF~av84UNq1+-QJjFzoQ_E&B@{yhlb1|2}rfsB?RNWKSq z0L&GG7yXpNOrQ1(y{(F*j`BZqgh1w6BXeyYL0cr>)>-Jyh(bOGE_2B@q|b`KbKi4L zCp)C?BErScdnFzNw~>b6vO$sTE1O=IizVSv5_!SAz>i6lArWGl?{K$NAfm}S3P`{7 zm~3i@v7o3>@-;Li_sG{EkwcOoY~ms5$H{s!fB_L&YOBoI^+T>7QguWE;0+4;D)0_K zCD0|{XzepBF)oajDV~K#zKqBz5ZNQ=5qTbwsk?&iD0$<-SAPmV9EjZ+Fw|>4CG%)Z zUPb`a983b;d^rZ`r<-5ECC4$EaTwD&$VzX%7D~K`2r%{w5OL`Gh40QwKVQ3-sB1y9 zi-*qxH1;#Xozh&%n5o}|B`zWS(PaCeUuz*y!xK`PGL}JD-4d-(HXsrUolOO zzd6J&XV{2%iC_1Dd}s8~O~7vp=IIL%L~z>pFEaB$LA#gY7kI00@^4GO_n8**W&o~t za!EjcyrvO&qd6+z+S>}9taw!XyiAH(Qi$}vrjJ%$kTQFNSypCP=X&D-mOe}yjG9#Oe(aXskI z!YT7Oh9&2X+Za|VwaKUt`@Pj-dVKcXt4ljMd3`c7r@t&Fj{n@YmoER4P|q{7r{=>&P4oS-rR6EdgX$f zxW^Wze|+;QEhnR=pJCAt`5am1vv%}3k_DF7(9Am_{Zap$o&WvY&!fSa`txW>e;xrd z28nbEsuAA&Jo@d+!AMS^^*+`g3{q1UtbMHI4fU63G~s(B%EDBiJ8l= zDVIjE>M}c{c;xK_JVlAWFTMg1SkH&sfP)Ib<|IVC7-;**aMipFjtU zn_wPQe6hTYfAx=7<*rtdcLoZu(z1A71tSShX^?yLez$}ej)c1K9gw{z1ucecg1hx9 zJg3ot8`J&()Wg(Y&uGjmG`P$8&ci7n`iM?JF(;Sbua^_Hmo)l$^7HIUzm~p%4~>5q z*FR4L!(zwv?STt~itGT_q4zk-yb$pUJzLfj}q7vsvM6YBT!z*=P>)nckr6Q?oe%$R0P`%mj$9kM~52=>$|FsxDrB z{dLvZ6>V`?L=_4CJEX?!pY9Uu#`2JuQodb9d#-FTX zAOyTeg+y^moO;jsHa4@E{uNI7aGrWx|$i&^{P@ zH0|J)(bAXtYu06m;r1)>}yq_;L$}>l1E(-|x&I?cl z_A6#Di7rc8tYLLX5b!7oBNz*=!r^LozZv}lJ{WeY=Cj7Np=;nEFyZwuRCLO9t(YP= z=Y6IE*863LeM5prk8-u@FxKBV&P83HPO9Yei&&9lCuD&CxyC{w7!swt`pzxZYY6!lTbzg z+SJtvjUpPo4t;OR1ThGu^kJ#pY$$m6N_JN(8q5wBoncRAeE+D96W$>!S+w zjuzDSk1m60YA-j9cCCr)*HOXQ%0PE#Gq%Qv+f2CW+wZx!&$}~WE7wwd?yh>cQGsPD zLdrN?MWx~RsgnCcI(5G%tl>dC&GMJs{@`(C$3QXGX zy<@rc*TA8BB0E#dh`DT&;G z@1({O<;&T*a}Eq4U8BZ=*&(;b3`dj?g5EzbVD!m}JX7&e<-PcL82`q`18 zYJEJ(`U=m|p<4+f$=D$OX0OUe@Y=0$d+J4W<5NC^$|Wr+Xn^Bjz_Crb+(UtM zRAC#o<=jf+iU+~FXvYQEtB-*QM$8H(>Q&$GNsC%1A;(@QVS^G6*W>&1JZtAilp{pd z_)iyZFh_Q8-WIJw({z&Hw%*33Z`#rQ?LwcT)@W5VMTJrIch28%l}+<*PP5+TWD?bG zrU=qX-5Bse%W-um4YFJ-g*QP&yMM89Ej9-jJpVm^44$V%FI?QBiT9Ap-e0W%_1N|#Op_R!b=|-?cEkWwYjv^9Z(i{4>~2%Mh(IKElxz7o zX0@JCKXFA>)wtT+cG`*PigKv{DWYx!u)8x4*AGI}?S)1ZqRJ*9hWYFK zSQdXmpxeRli>W;|rYiKfas%p%zBKD<5=Jn+*ioTi9Qxi24>k2A;Y#zD(kk-sX2Sru z4}kYR;G&{<1y7(&ynvO6Hg$cx8%FC_?ZL^u`4*y1X#R-)iLm0VgdMH)-j%&k;I&62e3b-pjVP}N`u{12-98wk}VR%J3$!G6c`|= zfW@>@epwZU63r)lJ{)G6m-dNJ&5ml!17s_kl)XkEp|*AFP+`OTxe4INT^s<UqBD1_2N zUcb;CYhqY=-SmawIAe3Hx}w|fyxEKfY3HVTb>_w1#Phx6a%aKbA?vQOixdAcD`#i? zhdOZ=|46`nD>M=k7s;6GA%fOkC|RPOMET!!d<#tVL9`$-p|&`+3co>xN~u;X$m)Iy zp=DmB8a}4}$qtzM{6L;=gi*D&3)4m&DXCw8CJtJ3FTAWFJGBSe#(iLxu&9dj)KJH| zZcd?{h3lWB?KpZ>4Jt(?m}h`yBYNhY0bO!jjlEs`3viX!qwst zdN5fn^Qwrh)DHu-TpcRZ9`HuK%lUDds&;E+tDnYu;ZP62ThmIu8afwa^wes>`4-Rn zWiXAeLX=SuEZMlmYuao|RJ>ib3}lk1U=5*ELv)_06PD;?hUhq>5lYh4}O&fe$vZ zg3k}w6j?wwDU{`y13ZucUmBl+G`J_TB>5#i1*ov9=pMe}sbALjf;Ys54-%0!%w{bj z;(>Glzb_;_?MF6C?UD<{X6uM=?f|cm%*2t^g=DnBf*2C8q@W85tkPA5`5Xi{G(AJT+oD0r-70$z$Z^B~>Yz2|Hev)08?GYZ) zp=W75!sT9wtS*6?*;Is+xI%B0cmXr}Y+Q}}W-5dN$4hwr1i?aMhnG_;n5i;Uu@F|C zg$arkAYaXklov4TfEqCj=A;D)c-vCqh|Kn?M$pNV$Eyq&^lWhIzg`mN5njY21NTX& z=uKEI9P;!OwKUG-drvkuBOi}eX{*2k;#(&|^2@LzFtE_Y#qlA09)f@$>aq%GcH>29 zXuSL%iMH_5V|Q#DFwvF30jNLMzY&2CSr@k_?v!1A_V8+U5%$W9KQtxt*!YChJjlAa zJw@{;z6xWgylAFqf)PHz#vD(4H#qI_we27bSGdFThah+!S)Y*L%)ql_K8)aC`N_r7 z4`Lo~L0tJh*YWQB2xbY&i1`M>8_p3B4eDE->J)!AYQme@JeG6QuP@V%{gv zL6=RS1pbwlK2RiwnwY{IcLvC*$27Y_0~9&5iM>#jGZlmSjPPmUoyz{RH}4Qtw-ljV z;0l=Jp-99d7s_}gCV_J4`#+Ur2_2tMX!cht@`m?wdo?)^&r0CWan=^F*b{#evw)3H zNXu`tZf;M?fS#A<7`aD(Xex3%d_wwt4;|c|axD5&d1l1IL;XoQj)zZ3I*h}0d(thS zEwHndc|-a6ADV_aWPCzGewX%gduqB9z*gCu=brts{ThylPwCNM4|UWdq^DA8| zUWOCTr!`B(Kxskqtqa3H5$BS*GNhs{{;9~Hiazuc%oyk_rnVKdvjGpyrdT^U<1lxWBSO*>?!W1Mxh_)WhT91 zg59)%d;gPj6^DMi>G{H4PS(+!Ml6sg5XS@-HEXBgjXGIZgiiJXscafbxI!?hT{E?> zc5nYe-Rl9GE;K+PXn{-kX~BoaKAJjzJi>0;VbJabgQ{^L@bfNE=y!v{5l}cCpkRw3 z@WG{MV?j`TK4+4GcV@WKOiv(WdaY<;isDiuscC)O1zbo<;~)(sXX1-Sp}en`?Wb~U z*3r-`6HvOvhEWkKLcL2#`XydZ5})40d7bUgzgc#NlYQoS;Zsx7u(X>N~x|pTmP;EeDM2LTPb*}U8;X4Qa!khsjVdY z6x&e$P89uf=jq>m=V_~qY}jSYen2SnN3;>mne=FClDdxiAR)WWDx_QdC3L1MUHZArXYU19B#oCwC_Qr}! z>%A7haQrR^Nz~R{VMDACoQB0hjh~B+jZG4la=hdO`^pvTV~nTtcO~LiDdZg?pH~T1 z3Vii>1=r;QMdUSC&oJ^?a{$)55Ezv}9Qgqjt5-%-6L3N-Rbxybaq$R#lp=AowS!?V z$|)g)9xI1<7wJHi>Wj2zRjGN&A!Ivh!_QjMW7MFw9G;`VU`+LhOOoXgH^ydR*RkYu zQ659!2>a4dpT$@1SgF7{Wzccur$c0s4kUkw)T;8-Ef}**@|49t1=Vjz?~asCA+8~I zLGJUKFs}geHOh(#gMjTPN#udnhb{^`#eigy5@yqMF|Sg4YsCaan;q&B3*_MbhV%~n z5ZE`rd%Z&{JK7CF53ein)57_j!k~Qdmo*P@$;srAw1A)%MbF_F3AtF)X@`#Y9 z7}pkq2|bUeE~XWI^e8vHvLY^n!mL*#FLMlkBcA|M#r?td(5=;~HL}KOcU=@Y)h{m;xAKAR85Zgq;#cXksiUm)R zE^vFBovHdDC83Klgy&NT&*UNDK`X)%ZnKh;x%93S-N2pV+-LBfWZFHyUH&e+6gw2} z-*2|hwcPf3FPmMIf>hRFun=MvsPkGNVoNaET*)l>Y*ByUwcwQf8^cZ(P@m&HbZ#+Y z&M8Y@R!mxg?^c_>mE9YMp@X^Y%8i6Jt&vbZ02Y%NM$xatKl_~egBnMaB@cRuECLwn z3-Ak-^I*%0lYd41)JAgJqKQ;{wIw!}qcL(DGSuJu*jzDD*#>N16@Uz49A2i2jQ zEMyEjl|B2K^i?8dWd`Iw0d#c;f5t{hU5m^Tc$2*baU6B{cEHNi4CbF=FkfD!-CYm< zS>B10kI@9mJ*Gu~BFG>0XAi?!+9|N|?ERFMT zzvX`HU0n|bGBW-i-4$9KyTwQwOi(a%CE?$Ms4yu3cbCz2cLT!y-Vc*s=oF>{^aQ3B zX;!ygd^g|gd&6+(LfH_=NW5?M*@G)^GL|&LBV{|ED?^^aVV(UxkGv%P&f?MRxYvUn z?_KzQ&t>h-Htm8lJ``Dt!q4}5@#S(7`n_+$uY0{6EU|mVRz>!%lpR3QvY!lwDt>eE zn}*+Tq;KdPoF06nr;nbw;IhL5BYkk_uKHMOFf`LgM>G+SEPV9Tn}fE%N_JWTlX}5# zC;JUd?y#h-_;{RutXkP~Rms25wap6#yE;<-Gr1mydq{Q|-aCIt-|r>6B9(#9QSH6+ z-}mWYho=m^9=cdOk(V%xa4=l5UmS4>ATDG??EUsz0uDP`*Kd*?LD-WhTs{v#{n5BD z9#Qc#8L;;H3<97Ae1_2Hd*Bl$*5s1F#ES;An0)v4@Rcz1Je@|gW(!(_t9T6@)0bsA z%UC6P1@A(cX}WL&lqvUGJp-ITn%AO27Tae|s@Z zh69R8S_VQvBcss0{$!68;8AG=!~=m7j?0}1%tm2a7N^1?#QI$vNACeD=#&nwXrtB1 zpv^%>hgX=>6IdS9CJrZoUeNO_ zpHzgoRu%&$G4WoM4U#pR@btHX-S~5ci${+TiAV4Vli#v3gGZ10!Mg}uy4)PPrqkZw zx8HgqHDB*N^yC34BIJ-^luqvlgX;oZ(~k`o%ck83$beygwVOnjS1XQK4ClY8ju|b% zLH%ag|38j?_#W9M=Dn4=5#mVl%mr9-JHK1~^3!p!Xm-s5#}8o*5LNpE=8 zUV~n+O_{-NvcO<1XE5ye>%D)%Y@EFyUtXPo_cTC#8;4-zX%qp5|M^aE;Cu2M9}pZ0 zrnh8}v(#Hz7tlm}{m)_ZFozNR$56N46{aXJeViN|O_^v9SHoa<1(REh9a%R(_NF(u z9`KGigX9W;*!9G_su4S2nFb4{ER{cCv5g8@2TRO$(i<}EQPvTl7dtnfwuH@JLyc+L z$brZn06MRIuFl(WPgpQIHq~Xli-rfot$uA&)tr*Kt^;eME(^TQD2dUC{aJ)-ZomDO zgWsK99)b5l!EW@J3&$e__ZkbOsUv#>)>YcU*h6H`VB`sZE52t&~+OHEf8~kAsgZmGyXGSFe=O^HT2qrFJ**mJ#WFf z!iznY9Q@#i!^7v_zdio`hiA_pJu22_rGG+42hqbRh!K`ReoF;Y-8f|W5sa)Ha`-qC zU$KE9LKunL_@oY3*Vo*eY(bA8Mvat;i8Plj^oW`M+NF;kaXsKIQ3ruxSOML< zNAXLv-mXwFKN!f?aJtu`E6@frPxsyQJs_3S1%oOxSGR(We{!UMc{K&?hLq03SYw&d zD)3>Yo%A=J2M!{r1CA%C2Ek;g8NOQ#uY~Q{SzUu~%Nhg+dg;aBJb>kXCy=(?PJj#W zjVK%pb`T5(%JuufaNW&lskMD*b~Ay%NmY*!*{|o0ws9TTj5eN2#WqA5}{OvPpV4AhL_M`F=bBv zJz29;K?grv9P}{B70jaddXd#5g+Xp28h*=WtZ(^*Rkr>%J~;o!@)?FUBxN?)%Xoz3 znR(L|mp}R7zYgj8=s;(Uu<`HpuC99oEcIRwr~Dqvox|poEa4x#e)Q`)Om@*Ljmkl1{NARTPVq5Ll$o?@eRr>!Xl5T*sHmkBz7#*lA>PLPH`=zblVq~aS$?6 zn>m0~69=T1D=Vo3U>Gqb=I)iQ2#Y&s>HH<;x6D!^vSwH$@V(wVV%K6t^*HgSrH1G@P%yQ{aG;x|9uaC&!B18RzL${L5WKQfidDWScDxy4^Fge0>7C&y zct2L$!e=XCSf?uD|4aRTt;b`0KZmgYVybd_m_jiL?^Z`A!|a zd#4P=3y?BUa?xe_QW5eDn$877%YVEV4>KP7exn$&8ZUSavX7b0$(bgr#mbnCqtc5z zIn$)H;Pw0AcEEf(cl{fH^$N?AH#DYhG+hve`t5+RUXCMC>n89`LyU2K0|0;WCX^h2 z)5&2YXKEMBok1;75i@!d*lteL=ND7z6L$X#&2`4O9igPliFBrZS^|IfYX7N(7i6SP zAT7Y`{I2v=@zhBNf=(w=zUZd`j=^>4&H1#TBNdU&q-9#^lM|Fl?2Y7fk}Lda@WwV% z-DO9pR=o752kR}3z=LpoiL;s^cVM(6t;1TDjGF)mQ@0zbkDstS@?1y6vG$o-lWvT> zcnU`FM$i?pCmq!_&;f(Rjz5tie(PQc%i0l@iFi9;lZ; zyaX>~J3Ldz+o3sCc?|NOe?-a}47^XS&8s7w>ANLD7stusEdkY~1v_?092g~2YgUomENic-x zAjo%NO@5O23%Q(0)f z#hy)WdAzvW!IHR1N@3F?pO+{NmieT1*P~n$w>$WfJ4>%uTyp12_fcU5zX7!T&E|BC zKaE?7>-0O3{zl#Dm10=(qp)tg3RmkD;MR+iNqEsQIwfoJJ8U%F|6tgL5o{ocp^4&a ziJ`y#6}y{Sd*%6Np?}m|*iZa^&v7k_!GAC(-I^~cLK_(N2Lk}r-PcY-vu=s}qp7l? z5mk}Ln&zrRB{)h&k`;9t1LBfQ#&nkYjW0&!;5ys-#CVE&|9u0`~;9cb_jLmpLP5+P$0*#kx#O!2hT zlYOCi(o!{r1usn(^eke*m~Y5Cxv){H@xXXT3@XOhQS2sFD`zZiqMC&LE16ZY!0R!- z!S@YBNMmuP5dvx>INV&Vr5l?cO|xkqU$bc+N3(5MH?@F&vhMw{F8|x@7(^vbMS6^a zpAI<0BM5~swCMqXZhQ3kV+5Df_rvpE-M;;#BqA+Q%6Z`WnpzmO!@|%gdj8x2&n62l zYvPHPixEX)8hIzRVMMYRBZYt>Bo{2T_vEvE@H9buXPeE_&oI_Vt{0A=cb3f*L`z3z zv7?Ch@F$*pOi=Ru8y(pwwrI1O@*SN?ZuSdPEI(?=1iHE3uP%2xz~ANk&sjzDnbUZdwmLQB;` z`9(UtlI{k1>MP~ohQFb|Nzu?qem)+?H+(qRU{bT@&TAgwY42pZK+ACHqcPrS8=k?O z@C?}p4RMVGPRgzF2nA)|k7S<xp|Z^T!N~k$Z{Kno7!#Y zZ=p6Jt)A(Pf>veS2O%q35`~f^)U*C)Kdxq|zsPu`g7McZpuJ?ZGg}&SpIwTdU|*Vt=Z$S!cYzxb}P&BY9<+MZMwbL3x4WXiJ5=EjE-8xMj_u z@$_ZM`)D`!Ow+BJ*x#RzY?2QTR?pzsRxAH1C2oA0JM)y9ZRWA~%!Wzvddaj-dT|yF z66-Qvj;toty&vOg>?67YIhlZns6&EmJS3H9=Wc-4hgRP77&=&k_jrPLF(ngE#fmO- zn(ly=Aum{Z0GQ=6eHXa_sO%FjiW9WwnG~}OEMrBsjIV|OC^}vsQj^OLK)_TY25+<( zf*+s&FxNjv5NH?LxMF6neE z*?|;-5pROfN|hT7HLJ^UtBkerZE9Pc%T`eU9uhqdVTw8OeRt^^pdLW`dDBzL69E$N z+Ra9tZjP#i;oPUAImOg8IN{}5Pf`@ELs>Ar9ZG7llC`RCEt^7sSEVqT67zb%(jaQ@ zRJoZZO|4`NfLPnNt96G?Z#d*Tj26iV4w$6q($)_3Vs~3huXw!YE@jhPNCoP3sC1Jq znyh+5IR?ts!dAyd?T+P~rgBkuI>JAaCSkm3%wDRgT1z=YQfy`1S^$s^zBO*?ND*~G z!_@LjwL-VTH*C9m%d(~bzXG#&E3LP_tup<6bX%wbHIjik;6#|@$jF#(| z9>=Sk&)@X^SUpxdl&P|V$T~LKWF4z10#?HHU2n8>=lr|a1d!06bt)e(!Q0+iTom>( z;Sv;9wGM?hkeD4xXK+-9iGg0BQ`5$_l3Z1GE*`i+*tIw-2y*I3iq)l{ zig0f!n1YMqFR4yjX;Z)n-P`TuayZytIER6)g;Pqv$W*<(q$WygP`JG$^W-cvX*En! zDw&Rq6<6(T#nZ8MD!!$2OY+x&yRA$d`rC>mG>RJA301*6S_{0PzG2x$P7IBbEL2OT zx^n4G;Tw%d@YAo;ok=*S$$~~BRALfyTt~0)FPK0i@Kfn0ZscC3KJ43dGzo9m&RDs7 zJBD;L4c7@E?(Fz5dQ;`%wGb@^n#Jk7P&!-CjpJ}3l!PA zcoCjIb!YS3$_hLBkge$O92A?kxJaEXxDaPFJHVG5gKQ1oYD8sS7S<>maEOgavQ6XP+I$cNZ zG`Bj1YtFirc8np&g8XaUmJK`Guc*hiBCD3AIcA2Ov-zneFEdQFPN`c5&Q`*W=7B4+qB&bfu2^G5H6Fys3u{i>hOR5kG`CdQFgnm& zvF>Hyt&@vw<5q05aonobjpT}9*xExFx!rx_jy!H5xU4z)_HnC*V&2QRo1BMjLw1a2 z`PM919d{tMY1(7+ArRZT39)rYYi4s**3~ZLcC3dmZu2JN)-0<{yfw|*K5m%9ndZH- zxq0^in=o_hV7wJW*VXYuAa%{k*VgeXPUHA(OCFCqSa3se?!}N**HX5PR&6m}iY1$B z2lh6{s(L@JTGNK#woxm3^RNwQ$?T%cOvh0l0FT!#Hn)%82LHU-irFxHtBdJ2w1?ob z+*H7A;OS-?Zh%B6T8GgC`N#LJ(W?tMcNx0{Cc-j1D0f@g_;$bN*lQPqw~bieDC|0F zYjy`}E2?c+_fGBV!e0B>E!nc{#<7nrS=WpXwoFwu^#M?Ov$Vf$)Mhimbyc?24lJ(f zmilnk9$@0pOSjE?_Smw`v26p^ z6}8!X8!Fyo+&*I6a5mcEe&t&0Hf&XE)@C+1ZTh&&3{<%vuccv8d)t^5wRzU9f%Q_m zEG{WyOM6J#%3*JT?Vh>RHeh#k!l#~J*~rRKWMfM0fyDQ)M^-V&UCk|ty`9L|0^-o<2l>L zuG%f$4S0!4hh=s}v+rrcw0dlC+jw(ZyH#d4F`a1#R~GzT?IDS}*>UVGTjQ!DJ8B17 z>$ ztjsF5ZQsvwbi)bpZDUr=mQym0teYLE4(`3KK1|);4GGG)S?FPwW14ms+n-wyx?k4s za+|0fC5mjDYU>`X)kP1gj_KT^9;`VIy=~Zr+Hx{v9klh^&wAbMvw3ID6}`?XhfGW}D30U6!s@C3gVj0m^(gOtkpBnkiddmiLtVa`bL~ zk#`$5+Rh1^maKP~^qY>RYY*V~MeVJaZ3xg>_BCK4>~uLvso2W>?`zawcWNKLBahYA zBeb^CY53~>oEvVxxU+5O#@KF|^XZ@pI#@qlH7x5PIfISi9NN**ei#X7z*dJF1CDh+ zyV%XQ#xXxq}Y;B|);f5W_|<&WB1)9oX+<5C4hG;?UNYL}A^{Plf&C=yEf??R^18^2=??i7yK-(UnhO4PvW{`?w7!QdH ziZO?_p`hiQfNIE!rgu5Snma9YUk0--a7X*tHQBaXC((w}RwrvKJXRXtzA;-9HgHRV+~xN28V4(KV<4QlyWjy- z{@84bzoT|x2hG&2`#ET_Ef8M&=q*`w%(e?*W+zQx*bjgSDgk8Mhi-xmY&}$O>)lQj z823&2#)ygA5TGeq+k{yeo1LQZ+|ON#P9Y}SF<{7MOZ*N)!vo`8&e|BJecz5iHR|KG zp({$u1z1})J6)a9O!a}Rmejhg8+CF+s9$VwKL&at%?QdESMf7uVT(^x}wba&Q#da45*mCawDorO!tFk?A zwsW>>$X1v0<2pFR_QPQHazx_x@dNCR+UDaK9WDneIN!Y&Nxy9r)mEgKj%=G++v2FH zbkhEgZr-mD)I@G?A311tv+aV7-pS=R4MlqxE`KS%#J0h!ExN$K(0rY?N!0rZI@%#B zcl-FQ{PFQR-LI&f*j~3Ez{*!`RP^@on;22D&FL}KZo6dG!%~GE0#mml#&Be%Sn<)y%CLORZT+)2@@hXcztt zd}Hal4gM3%zqG)F-s~4^CkoGVzgSLT_Opq|+i0Ukj(my}ZaS=l6i@#kG2LY!be(|q>^^nS#3egPsuSZTi-$%X^^Ei7d$nn5sK{Dy418X7&i_ICWQhJ%-G} z<6lBA$eyQzSx>zgCHzBLhG$edL;O!&zdjH6%1stzw%0pUHKLtiruxSte71SBbcpH@ z1OH`QRDEsX8+^Zci#SS?jPD;D{P<@7;2$Yna}Fn8Z|C{VSGBn=B*D7X5F>NsBP@KA;lb!b?lw7l-o; zso|JIji}n9menlca87FCrJ}lt)&W=_%-bC1)JCJYdr?LVDwftOX%?Ob zTS#LQ)gVw&-uSO!QO76dD7~QjcYUFT*1c)I5|0kE=~By zMbWb7x<2#HyjhdU-)ij}jk%8QLfTZtgcq#0ibWML-A%(|229{WWSLLZUya6FMc{>^ zVq;{jGjH({R3$p1d%b<`m2We5)pzl^n@kr38SxSu%3*Yhyu9;9iaKFp_2tMVZ_{P+V2LH{qpgzUfk6oc8<&F9a|%3wB~e)IwTu%f}I z8H>XHDUXHs_tA%O1XFGL_fJHW@l);TL3to)j~{&~Z^dE`ITW{!KgE~~+cd3lc}$hA zKlRs|V_i2ZB>Ir2PJ{OpMl%{g?-5Kzij!XEbMV*bYx3XV-kxypsL6Rqy)yoe z&v6g|MO^XOQgO7r3ZK{Xo*XU0b0h^W(Z$J-B;krYa+j-JasobR;=%lzX80P!UFjVp7nWeZBro=1 zcKC$wQt1bI%va=|^!tOoCs$`~MChIyUj|c>z;Cu>)n|W6(jN@^gX`Ts;$7ayMt}P) z_hawsdN7c~z`xw1yMQk6%H^2{r~rO>eg+eQen8I&|JLIWV&UFkuzL*+?*iI5D1;#g zXTcL1Oc&t9o=Wq`gP8zBShAo?pz%Bc=TtmeQF;orx(wGaRl&@P@V$z{Wr7|rbD9(W z5w9l+IwgPwm?(K-?!Dk;G7wtviui*~g=WR*NHdIt7mQCZ6vlmmKzLi{vqTabxHK>n?In60sj7o`xuC zAoIN^Z`|PY@d&0&l{bed75r7@-M&~4udj(yTkBl5#K5iL{-z2ZyeqJ7 zp)D6sh`A^0QE1*4M7si_-2fschujV#W{IqY(CP{y-#$Oi<#7byUcbd3dvYQOsIs)v0x5sDl?R?`E$}&w2@{F>VPh&7!83|wLH)) z)nGl)ooxtaVmKD*0DDjV3SY=y^%`JQGY^c;fGIK@rUKOJq3Lcd_zKl=E?Zyk$$!BQ zrUCFC58>(#-HRH?6up8bD-Re;)>jXgeoN98Z{2K2%f*58RUoNZB`GG=%fqU0vCs>u z1>-u_Lwj4ZQO|~q3Hz$r0I9i+`B)mfdO*!v32;7BT`J24>0d0o`Jw}a9G69&Wa;Ya zVeD)^@GQykCKnOzlE#!A;rAM#3?&ED_oMw1#?7nv~2b8jqm1*8j@4=Mh)57 z3TBNz%F?IRJIl%~eHE7BY~!uG#h*2x8TJ-gWhu_;LA#Bi&}1t$74WqNAYCb{xiKec z0b5I@p4~L|^T2~RXt z51eyb4^OivIS)^>pH@J_G*H_>W67ZEA-l76uyWKbN`^)H>EidG25iUPA|yO}Q~eT# za>rR33nEmVs*zXlxCU0!DWaG;_*!vFX*Y?qe)|=xO+KUyxku~~qIWBtHPaAnsn%;o zy|Z1x3ns@+S&IOr@P^JF+kfT>QM1yl?3 zVN5SnuZMKwHh9!Cyy-O%DOMdsCHB*nO%R!PWk0jbd}bYxH8Y|D&M-|+1MgXPRYEnH zbjsSuY|};>_+Z;iw%N4QrrWn)0;xtcE3AZUlv27NQUkg%E*W*su$8q;qkfvN-_kf` z>s`V;HLZ4mRV=w}DtFq74zJndQR=r^@XAI15yD;rn5lFSb<8VMzfOHiBUDNKl93!| z>A3Fj{9;uDt!`|GmL-9yhqkj75_wvVH+foM-jIE-0j%sBSzJ98=*tqd)B~n{Jn63y z`tQRaX_R|_UD@V^jGGoo4E^@>Z_j^HUZm z6>^yLo`iZ0NQPFi4o+dxF>YG?F*=(so?GNN%PoRjdl-_R>Sh~w-KeCL3^A1~z{N6* z)Wdbt9hPbq5Wtl|P_oM4VeXI8qLj*~;YNuy?Rtx9*dLiSJ8)#1ps^Gl9heexdDIWc zxw{WtlW(KyRkyhbc6nd3#=&O!H|mElzU8`aDMiL+@wb6>w37JSVhN`RC#K1$-`l*^ za$Gr+g@CdiMyKKu2$#0Bi)hzRsLal$tekA(SR|Y1J$cE$)PPoU3OMeRlCRyclxg1C zmer(!C>u4kjYZ1~|0>`8EP~(U3Ys^aSrw8ufI3ra;4y08vC8|!MguS2W;Vt0Dx?#t z3P4tAx3hMy4pS=B17zM+v9@MCs8O@b`QK{wSuTX_UN+`LXc*Y-=U{DV3o_otgAPK; zwvM9ckJS)|r7@@O|l8t_@(fqL+rkEc6oR7R_L2I^?rw1&}m z*2KK6Wh=$=-&nTq;YAIQQEr+RHAhxXS5VK(S^$_cyLZxrh zliTmPsUZ>yqf6FV45dAY7m*Rii1UUMCPsFlOo8fL*`PVOzIAXiZ3JKxz7v9N|+umINS!m!#C3ykz02J0KXtRM*38 z-v#bmBf+^~682cuGZYP7Ig!@@w< zgE_u~?_Y>iT=xG#IY`+QpaM2pjDyEZX_H+1G|mXld;AhS}sAC)_5sh2F7 zQEzdb&Xz0XzoK6v{S}{kkay2C*$I z1E~T^t0eUe>uk>WmQ$Zn#M#Z%r*4YIg3WH9O(1{N-~LkWE?2E_VN>CPLV|fo9|~9a zf(3^yd5|@yiE%%);U3%Vk43h`am}Vj-(i_88@f3Au`9cMMW;+@TIn2XABPf+fmiD& z0fj50V9{6W^_^NgWZ^U|7>@PZo&hVwO3j@Gt2;)`;-uDj52ba@W$~^d{uOt?N9@x3 zCisC5{gR0Kdp($J@^GI5++!A*V3gg4KgOx%@p8Fb%TJ&BfpcMA$iKua;06Y?qRDj8 zBhx61<1q5(7_|(9H@IAecsLx;kc+)Q)_7lEJ)!f9e?XmZxe5asB+@$aHyuAsJN^m! zeHG$OeEI20^8Ysfs|FqC>(Pr<7$tkX89i~=e$s;h1yGzu#TI!~Q~BV(4(a;nKxd7> zoNu)WZaT^L`D5glBdT^rN-^ka;?6grW7CIO8{cmts&0)M%f|QCjZZ7h#f$L#EygH# z>XzfWZc0SSvyU8jcD0`P-t=wcVo4x8tqwHFz5EV%Yg3>aaF&H;*b;EIhA|og&hif! zK$^bbx>pkiM8v$v#gbrgG?1y>O}vW_peqSou9C|g9$YVtL6Qkmee4X3I|+l^#Lc*n z+;tLWQC^qs1$@4<3}tok9)r|bWHpgAgMf8ZE#l8HT%E#X2m^WgGz-m=buk0A==m{W zVRJD&WlZuh_IMA)aC;bQ&d5dMh-eH%2T^^)zp&sMpFLh-bc!&VG3ti|l<-EQVdTVk zXEZ|gmoi}FxtA=~6WI$#y(e51;M@oq8v3AaG-W4Thh(u{x&Z;%rBOoQ&q)};gDEAE z8_a2p5oMMHPaAxN(PUZI=V1g>Mo|_);4BZmbL~>c|pa6r9_MSZB4+-ka1O70h$y?|+C$8+P1jSEDCLE-X zC?5bqd>@DClGs~%z8ev+p^I=9`r-T%@#ZDSH*`iZW=a?hfmE-49FobJV}z-Km5*^I zLl21a>l!c%n1dGMQy8ZgOnvkgCLvfp1d;*lEnOg?OtP>o02W4`2DHTlYZn8x5F9{EFk}!@Ho8KD28y8fvON zx07r6EuE&#JP&~TDUH~g3wu9ez^89EKsi5gfjOt)NHAkOLfWxE$3Q?3 z<&zVSjrcoWp1gnxM3hE}mdRTtcWiLS$E)AVfg26;sA*5GWWG(y4XS2qz9^a0&GPw#IcAh2ENjK z4Z;V$k?(rKMm@%sO40E^;fWA(PyV*>04rb>i~NAL87(8W!zBd!$2GMC9d~Av1629= z>XD!9GO2$wPjVu z$TOL5u1PT{2td2L#7qFQHAOT4a4UG^`5;-HTioZ|j6;F}PbpJ3sXZbkzkp-L$Mt)b zf&h?n8LkD?0bJVeh(-m*IUJ{;{h48#AOjg~E z*N~u339dNh&O0wFFcpMQOT>ns$qd7y7DptIm*WtO^b&k?7Yz+|ue1&SwHq<;_h;@Z zslfnr0d9`4Pr?F#mW*6JDU9eGW@G^N0r zhO>&LccTmMj79Qa%}z#|p{kfy(|{@T{*39a=TQXA^c0YZ^mvZ*WspOX$I?CmgEk!s zyb`95upo}CJjV@NpBuE`eG|Htv)@6U-*W_b5oU2y8A|`>zyI%aMeu+A`~N98qoiLw zag}y6B%umylT2l#4Q2iZLs^YLfp%V^ml_h)8tRV0j3i9;m*@#66gd|c(PkT;xWHiO zx)2yXUy;D&$;~RtCZuY*JhE9}VJ=qw75_L1L(JV$6V?=+))$PiUW7*vO`|lKo^Aj1ru+Srzh`6PB}`N~Ro(SGshNrv(+a*oStG8ZwYg)XbN>+h4U~Tp*S%pWR^n%X4QMWpKwWP|7Dg@ z!_qbxH8d?F>H^?7XV~sB?&>_i+=y`NkUS3D=4zPW0dUa?NAQ~`oXFP8asY{yy*OFViB*#>p>*2~1 z%p|}oIG*@!aN6U0(?N(QMnJ}c5WSEnLT|2B6}#Y>MLA}xDeF!D-BN+MFt0hsJYW$5 zSeFQl#C`~nR71WDI-<(i7!)^RM7w2*x3I12CsHm_Fnikwv7YP~RKIgB_UZzvWHg z@3h(9egplrHBpHNt?l_86O~}$YNC>w(><%;ufYdAUqeqkP>+$_c+lqso1MSKrGKCt zl@BLxraygWzWP*$d^juSPO7NGcbnwYf>FzdW3qg3rQ2|*%_;2-Lx%dtXYQM0WZ1!wbuR}MbUJny?+exZ;uUC zGsoswbH+F8U*FQdp{h=M|E3HyQ=yn+*v{BA^V9*mzoM^cbcmP4a*5?-ZER?!VJn7X zn`1ATd1e;N8TfoQ9;GMI1+0>{fX(O?hmp9q1xDHpqT7 zubk$q2#XEe*4C{xmNngeSi+*px~@95YS^k`oA&Q5E1GQCmZ>{q-7#(W{JmvG2S(5h z#ngcL0S5biW#t*At0PKJ?woJv* z#)@IvV^zOrOM}T_?9-Z3}K}=YjF7BZGaiG)+?-a1Jc(_g41c zrE1F90Zp$eV}RrLR`wWk!m?G>7(1$=8qWQ)D^E6iN5|;(Tgf6giF3F5MB{c(-opD_ z`a@$_L(>dX)pQ5`TEDk<0{$9kHINm}vM}7G`TH}hu8qgwojGG$2gbc$hW&77Zsl1? zws5KqbC-5s1f#W%9~P$?IOo#Aj~s&?R>tb@&8e2Gn3gd%#+rtXfc|@X0=5h;kZL-P z26hD;qWk64H2w7**_B^H-{QJD zYhsiobYWH7G;~cHTfZ~of-I|=?I7Pc*4Vk}g2~-E7kp?P{1F=luj;lwzVA6#2)wWe z6LME&D>8y^(Y1!kec0A@isFjr{p(tc^Pqxp2LC|CmEqr8*Q)Xun99)%%{Dc|xv9+D zoo96!?W1FvU~gdR_etknS;MUdLd(PDCqpH6?}Tl#i!oPkCbNyNEjN=(J6Vmlk`d(V ztw&cYu=NPrf|RBU&!0uOr{Cj8OQI0(26@YQFPF_;9J+jo*Le7qOQaw>MxSkF4GCYR zACh$WZiokiTye5{l`%tr7ds>BQ#Z!jNMhp^9=M5jM#%!NMvGnj+)zSr$MERLOq_U@ zbVVWJp`|WF*|G3cj zFZfV#L`k^9yK*t^WGi3c5sq-hZ(Hs4zIe88+omNR3RFNb8BeX^f$(ruY5~U7v5idw zqvCKn4UaRzoyMlT7(X+eyN#`(jk}F4W$rtT&Uj`$eewKmqibm6Zlgniy&K)Yqj|30 zk#6;Ew2W-Dk-QXmgqq1PjzTjz{F>SE6cs207G^@CCnxZQ2|c?6hX;9h+1}>ZPvxk6 zs+ZoO*?!*>5Vrtm3IyZlg9OkP7_{#hgY)$ED!+-E>MwD$EQO4#XnuBX9r^t)8Wr%M zbU5v%=k2r`pSIhMmX8I|4CZfc6uBTg8qpg8y#YX@d6fb}%ifd%+(!I~{=1@In1P$3 zKQ&{265!7K&#l!>34oUNPvqYe`TUSDJFr=EAX%6bwrl{i9g2qyV~h$*Nj4#nmcFFI z-vs``R0C!NT4rpNUXyjn9}(r0oWt~iRHhkhRCLDg6K1EGtF;$+*S}-!d?UZdpfiR3 zM;dgdM*mQhPF2^x{Gjp5D~udN3!g8%gvPje1Ett?FLo|}83rLgmakvLBfW#nKQzEH zxqN7VWwP_&FqZQ77Qooz*)qf{SL-ESrEoV9_Bcbc+c0FS&DnNb!%+cU+pI0YDWi7#=Ry>wH(J(wXb%W zue75YtBYNJ{fmExdGFi{B-WT@ZysPH_`9U17Z^br^gI47Xx7H~P8HR5A867rLztqB z9Yl&{8;&E;q{ybJ>!yQSxt6Be`X3xorYYlzA6`)v;+Py7?z!Z>j<#&P2*f2KRv!n3!c&1Ltf8N0RBthNU@bUe`k2UR2RNmCR z9)rcmiNBU1R(BLfu}wt*4s~qvKHclmD(?eQlDZ|2$6#a(u(-OR>W1~N<+?lo%r<9}ky2Sl;I%O~DRsC`dKwnu;OD zsv(ApFZeO)O~9wm2>YjhvJQAu-u{4GiHC$_Pfxsy@7#ESv83Wc2|`9A@-mQC7!leH zri(C&3A7#J8RfvgB>(f@|0fA39=JY_fCpFv<^_gBzyRh1Q<8+yWwANTNpLwNaY&v5 zc8TksyO%Lx$xV1F0qjgB{kP!(M%6!!$&BaZh}|XT2yu(e!5a^F>^u;$#v^$^-m=(Q zBGpkuc`BBF?(mRX|HOms7#^CP`z|)dHhlti&RaT(+&K$!h+&>iDP0l%f@CiO_sp9! z?v-=R%);p!U*{eXBLBLk(d7~K*>RG6-|zp2`wqj1{qMcr7rpm8L74RAB%DI$XLttX zbnqX8Vgn~(gn2h|q-4bk#KM2@sqD^XERxkL7MGqz{od5~rl-B(hUR@*txE{T24nQw z;|&46_7lv>0FvJCA;7)CZl#uYLi(SxYQOy^s_*R)7}x-?pRA)`x3(ir59lgISrJQp zY^k9)QT5Ry@;U!1L*X2xb%g8<>PAcml`;1`6nMs?ZS`yVBm!8OGylsV2{Dvg|B6iL z0@wgl1L?&}5Whu_T-P*kU3yY3lmERbTNKPqi40*?LUVP3G9c6SsN^-8T+0E ZHiK_z;>1O=^q&y&{{yqVWoY~L0RSlYeoX)X literal 0 HcmV?d00001 diff --git a/tests/data/vals_ai/saved_pages/sage.html.gz b/tests/data/vals_ai/saved_pages/sage.html.gz new file mode 100644 index 0000000000000000000000000000000000000000..7085d9fc26e2cba005d9094b712c4ebd60d3ceae GIT binary patch literal 63815 zcmV(+K;6F|iwFP!00002|LnbMW7{^eF#Ns0g4OQ!kxosD;GK4zZmw<4CQVM0cK4h# zua}l+iM1qZNy?X|{_pP$cqc&+lI_}Uor^7zAUK#A%ngIVSD!z7^Ys1yzI`suljY*k zSNKy}c&pjrz+Vldsh4>2Nf=Fhc@fO!$>AW0HvT}GEW9{A94v#CJeT81WZ_@X zAMzwzNGIVckz;QaOUK^ibQXo1)l@$AVt?>R`fBcb(??$|{lt?db1#a0==g{CFJxzs zU0r!g|8Q^?_~+{|O5lCao4=Y*a7~hmhRS*q&xc+#+bm&_@dzKeg8#Jj;OBU-C$0Q*>6w@K!x7vqZUI_Of*IT% z)F6f?1q=XKOi{q%VNy3+pL<8Piz$Np+EUwYALEIocH zEyJn5h$S3b(D zgOWD|Ll%e}Nm={WFW$y-gOwI2Zk0-+|*eiDUoEXAASpZ!S^D>=;TD1>Io<>4Tl z?I%I9@S8xt0*>>e-4HUD9f4Qy_>)bz% zgT$}%c5)shi68AxfEg-n&siO7rvZ?o5@^G zpNfNDe9%_9re47RqL*gF7F6B>hPVQ$n}dc=1QQP_HICvq1(34C;UFWi^aMy=yy6(X z#Mo59rq#aJ@H6zL<&Xe_@%UZn&sQ*b`b$0r$dvQ&jCIr-3UVBjzth~3NM#7u*WoIxeUZQ4@r@OJ zocSw1@{%yB@Z#e*i9+ejP>7`vr-uEBw+dH4$wdyMn7qlXiZ(MWreFRX1FJpy@M-XO z3cy7&*e`mGCv$)4DPaV<-wpMfEM0^Ru3OdgM=zbCOA%vNFK!}qk^%Xdf5p8p69=>uqOe_nJZ4U;{| z&rNh-6)6Pkdrw;LNznnl<|{`>?|ua1o_>?0VP^NkP(kI-q%XdZ>Q|sWibH=osyx(u z>JVPKm*0d}1aRfU$Lb|?b8v74puwIL?@3+`EB+}RjoL8MKXfqoA|(dNl?(^UV02Kq zuSti8Qm8Ea)eJ-j3}ZON7bYLV59&t%1%QCB;Ro_@RKS%%mwfO))?ll@8UG}ymjeyK z+gbqw4If95cd3B+mW0SSN(|!ES3oE)N?&MCQb(g2l;9dwzF!rehQl=it57N9XVYvr zNSS3Y8Y%b=XfeR)JO0@_^YUBOu?*J=6>@K=ioxcnDK3}cCT=xMoy$Y%bG3N~98T%5 z1~ahrIyjmGv6aXNRXgzk{XxMY;D4eQC!<6L!CJpDlD@3IVlztLBdv)Myt52-z(4B!$4Ljt@5P-W=K;NZS9Y%ng2mI|JEApJFxPJzgQ zbRJ31BWdET;dQ?BeE=U96R_Z5+MNMIy-gjmz>lSu5dbvNpDjzY90if_OrM`(uN zm^Q1Tb@{Q@;zc9@W50kY4zGT(@Mev#Z@o*DwIJEEi_cf6>Zej_l2%&A(q@fT!gDWL zT&8wJ@kkUd7St@lNz}Se&8=`%>nmKn+eBynW&Datdi7O$HI($%vfYD85pGd3gl3pyVyk zE`^?s($E)JyWqZLsQ%ucgi&S#hwJvm#)WtID3B`SpBmZ{GRpffTpW9p*^cr6 z10`hI$=m$W(@1&`Q-SK70%s=yOtv?XbQwvnLFKMtsH>y&H3$e^2scUP%G`^O;Hp2Z zH!LsLnc$olS4Kh=rEh44CUQ^c9FQ9T&tDTguJJ{xZKz#3V7tO!vwoH9EjvyRHUS2gHhKoBe$5{mXO6`Q-Hf$jS3>Pv3!h zPsMd^2kc9Q>APua{1nisf8uQxNenH)e=%j%lr&i8}qxPtaBT-v7_WkCPXx{feH6(u-AflbX?M^e_);X!?imU*+u- zwDJoqF2dARP*R^AZH{D)ElQdNC(>{{{CfZY`}L>s=#2a{Uiv?cff-kcdiSUoa~vB=JvUKryJp1YYy?Uu)fd*3zZv z3ySODPdt_;8;~zcRC)_KGctoNmfivhzYS;q4?O1$v_Kn8Iu56o&|>OQY$^>D!pueg z=*&m!EJj1T>5y>Kun6UCy2bN&4bw7lTLxR>edfZ9_{<^ zBtc{B=9p-ZN<>3Sn{m!3&U`{T-+boaiYSQ?pkE~8o-bq3DGVkY69h4D`DF&;JUt^#A z4;{{C+0PYQ z>Ua^LZrFmHJ~ER2Mgt0fl&+Ub*(?9+P|^-MTHqVfR*xiilszCHN-q0>E9O-s&X*Z` zv7Wo?;tF`~ZxT`SVPtXyBaH2Cnw?y+owF=9)#-taYAlvL)DlrJbGsx*daEJ?YVZ15 z`uB&XW&Bb4>n9YcS5;-@Pybzea6c2D1<(~ZbIqUq&ZKrJ!iS6oARRJmqbelIhVdHh zT3%OW3))xuwcZ&;UiL&`2Bz4~ex-rAs=2eCm;8!MkOm8@JiMMOEfan?IzZz9u7mQI zHZc$a?xR5R(Iw8U=V~9N7c=~XE+6`-M+|!n*{u@ux+=lPi(tBr4g_MO7+PI|IrJEm zJ*svv%lzpN!9RS&?;yXs?^SQhn?N}ugM-f+T}&q72HgB$fFukY3D<|!M&l}T6zZ~ukZ*$( ziojt@?IqD=LyC1ohcW`bOTq{jI>E8RNcgZD{iAd+G_U5f&e+fua1f|)HViG9a>bmc z$jw!osfP7o(_k-2@WmIEr&t&pmWFd(*5^)@l7Gk|k~BvK_+Me7Mat39Hg7Vfq9z(2 z&rKe!VG}Wtb=6c(*?ZD(5xl1uLL;t>Yl%(sCVL8G(}Knovt>|rBM(idbg{dm1+Gj& zn+(vdSDpA##YVryP%!;U@z!g9HGMh{7SmzY+o*$w>MWEo%Fz+x z`dy29@8l~>+bFiV%wV(jwcpwCzE>XU>`+bw5WA$rQCDc(G<87IgL4Agw_ zMIF-S46EVj`hwv~@_<6?yj8k_8q0nf%C2NMP-KWadXmK|D$jZ=kUdG6)sXAiA$x@(bIZdpsS3pgPMhj= z=}!aie{dR<>5dwI{**;)`4vsNzYt1#r*j-tRtK3ba7 z86TkOz$lVIA1%W(e>gxAHq|2?L`lO@m7uw@W8`)nFialy^epYkXYg=CLS{6;aZups z0$t@|hIABPCEKRl>c=hXA$HFbf}wsp?GPerz0HF3J- zc1ot~vF!vw{*V|0-s^eX9!ketX3oNG5W%-CO0%L&U5t~x-)` zyGXBlblshrMu++RR_{9W{+F`jSrkmur`NuJDlfdtaFZMYDI@s!QThwLiaS)i#iu;e zZT}>=0Q)fs*ZUUG{UE|=v#%axZ%_|t?*G22{_WrwnTAUDb#=#=?!#b*mA25e|8evQ z-)$-;zCC>zQPsa){hdw+X^5M4(#w)I0nyE=Kim7u3;l)m zLX%X9C?+6=+UsdbR(eH(*N*o7GO;Jdguolu-pXHO@6xPK$6*BS#Xff`u{Lh;hb7thsC)x+>PkfLo$+_>Z4%Q*K zyZ|H(5XAedaOLN~06{ZYOe^P?b!{lo?0A?ChpHxOL5-xbq!1H_6ouzm2bTV+29pQo z-b%*Y*myr>Il9h+7{%F8ozTh=ofxmY%&+ zZ?ITxmh?lCe!y{fQJl{|;}b%?Pb$KLGzr(ubp-VT?XW^5SL?>{#7832XPqW`&%DLP zr};^FN3M8BbGR&1jsn&}95u%&v0sqUkItB;%%SbKgTMeuC<$k_zL&A+4dz(OB!f!gty>~CFn>qGg`>RE`SKO{~mjT>` zX}D>O0{0i}Z`Nx+n!t?Tg@2gC`^4jS#4eY=ub9ZG_Ng}dOkd*TwkS$>lyizoKnb&7 zge&FqZ~<~~bk)LnE;v=m$YHkeNRRsM35d^!u_ z;5b+W$>si>-b2>i;44lRmqk0f;6Id!2lzt}rDAQaKF%Su%ul(5kNw5TKB)TYK%SJ7ky!ncsT4{=#~preg6OS}?^^=BGNkaytyfa(k%o^eFA*+ln2h5hB(} zubq~@C53tf)|%F^)zG+b(NbfA^9|Gb=_|VapY83h zZ|6UkPd1B%R49G8x(L?Iz5P;dHcHDxCYrRE?c)an_xw+TGrE^^>c^*GgyZ^0y6{m{ z6G&wZK1AP5XP=zTK8ZS25L*o;e4c^H# zNq&w`0V;endp&%_j9>chdD;;jKFo-`W9qeJ^$z3<_k0VKqH8fw@8Z3vI8lt=PdF0~T8@EW!r^Rb_r)V|Xt2Tz#jc$r2Sla`1%+c|} zTb&Nl*!C(6*SNy-ryzJ0S+_`Vs^Dpg4<$Hge(7SlSjB0jg>1X`xsJQ$$KWNXV`>{@ z)_`VItO59p^m`Is91c`8zYPhmodgR?x)s{FFyvFPdncO+E%K-6`{Y;UywBnS(re18_3qw7SF2%zwk`60y?M}Km^aXmht(b6s zSd|%a#E5wsPL|Ul(AcjOFZbcMBQ2-+8|-~fzn{yp^es&wmoTMD0+|-lEOIa@Hd4wo ziuC-#ne{nT04R9o-c^8tGP>jJdK}?^CeQ&uZd9<=g z7NA!oS!RqceG^?%0Td%!8g))>gsyKVB>g*xMW-anR%EhCUerxa`5H-?&3_5_S244> zxWJ5#!@-+1@OQaQB)jQftbq-DS`~AFj7+cMYHAcN;_95qx0m2+c;Mc>b53yRySe9c zZ+X0lW`0CVcmi=uU{RClhIh(jUCm^2u#%gqp@Ay|qy1_s_KA4^7vyjNXu8k#Kq%x|UB%R=3uC6{&*LiKTvj^=X{Z$yUsnpvb-iLeHPvQ_PO zyPV`7;>~fg@V8N4nYJwSD|g{b#CRfTJehgFjCIVa(UhkJHsBYrjZ#z)wuyx15t>O< zAo$akMesXY((0v@=*rd~m4Nqt-Kv*@JFQavkw|s#7N&Yhb{or3eQ5}yQQOET5Y*p^hYAry;vdY_o*riMSnI5-HU~wUJC89 z?(;{Y(mh-1={GU9T;2IYnG^S5L8mgh45xtP^@X%Pmh~?DbqA;<-g1#tD_zqlE{7#8 zvHtaHgXcoj>iA~W1y(`wop;74_lAW5vPVaGAGO*?S#=v48+`4rHczw2V5KTez4%kL z_N~H*DOO71wu;A;uX&O#6?DF(-gUO-z^Hkb1#4eU_?^{5>jMs8IDQW%NyO`}wjfq3 zPDAUU#?P~bjq(7OX07A|-&JWg#QIM8_g1y9M!7pl`n*lBM$xO!TexmEs3WiY@d6{C zbr)Y{mB45f;;0U={&;INjDizcs2XbmWgCy+OQRA8uOBRX(L4?!lVi>D?%83WYWqh1 zW}-4~rV!Eghycm&BemK*wF6@|NuIO#ZBTtndJ9rIhnPX^+Ptqy!m0qMKG76ZSOn~G zl0*SWeRxG}r5KQ$9S2i(T|ZNa-&_6gp>l<~!2%Vyzm^717a_23b@h5*YOQE@6g<3Y z)lUm&vlPBUTQ{`9@bk*p+8S>)U0@N=X?TtrucJI93f#0BSig zu%X&jaoeD!JiZNd75JDBcY`QSfr6n*yk^#4Nr2wMUzk@T(}J#h`P1SLH|moL8l3gY zPDA=*mIqVQ~8e&Dv?oc%k?PS#Lg;XO2N@wA*%dURR+z!JQ-F5j&l zY+Z&9eA~4bWzv)}L(>7!dc-h_VXOYx=l){EFhuFWK`)U-07JtJ{My4+uocy*-trf@ zj!bEbDv}JCEy~T&Sh)=u>R&_bu6{^ai9RfKnL{;XoJ~=I>a~~56)ZbdT>DZQ5=k!F zjHLep(8(VBaE*qz&Q6uUo$L(6rKrQZBie>Bn15%3`Rk|rbyuT**JM4ON}*O|)MyLp zS7i}{=)Kh2kuA7%Bcnkd7pQ}V)P5f3a@ zCB{+_#dweuFxTN=wg`{CMJnz62rD+E#VOJ!f_MR{buaybB|h?x^dw4prKq5i#Wdop zMtzG%K<2|opXj-Zhw0&&34Hg?y&xIVpHea$0ST@ShFGvgp$9ns`fKIO;is$7NI}Lw z^bayAj<3Z^8+4*z=}Ia6luZ>nOTgTvv^^+6czpPV9%HOb{S$C4@)O+-@ZRij80>|6 z9<&XCj8gFRV|w8e7#T}H!j!W8&(*#3^oC9G{X7bi{4=dbv+rGv_Ja@M$3u_yJ1zSK zV|*_=u_*lfa1dWEkHf{_>+s9LU>{o?e4CSb6mcKtfp&j@=Gh0EtF5PyC=%r5!a*JMQd z8&U{>8t^-WcYXvuq0?Hrl)&-wSJPPf_WkQunW9gR(ug)Rpf&mwZ-8TlilP*!)s7CP z!(<-Bc!>0J4U%AmUTA)8emPB#C~sc!2YYln6ufzS5S$D@$0kbt(67Iy--8&O;gydE zw2U$Vjf~xE-FroM6T}|_9QXi_KE4zm9f}E#k zNkw?h%6vp;Onev>gQVRi>Cv~NgZOibi!Z)FB))@7c866~ZEY@-c!;Zf^{5N>x^oI1;PiJ5~jZogkAt-r2iU7m^{*ZCtN9j3U zkZ>q?v?Ya{9=lb%0=kH={=L^d%)JQyQPgb?GF4QaeJMFoG!;{OZ@stL`vlG{){bNi zklq=Lu14vLQwFItfN3|#?zNTJQ+mo^O_k;I4_K@eA)D0_PdXXwQR&f?5g-?ZyU%;V z?yum9N#4oR?DPPTd3;|z?Z-Qz#ptN3OW&S#JlgBEYr7t;2$?GuSf#kE@j9g>RwG_a zBaFHI`fCM#Z+iI-tQQ~j#(c2TgTbG^dHwqNckhqBd-Lr17hlw$rmcTOLkH25QV=UF!TfDB%w^*q zm5-ogl`w@bW#UhCV2BV(VjiEcaCLQ+T9X{~WW-?1(#(-g=N4MTRDK=!Uwo0ufnbR; z2oz&V0$Pq0(4D?9`*%!^EVh+50fx`;_UP{q+^twzNYqpqD`m z#sg^X4;j+>^#qszUuTV1qkRN}f^zk7w6_sc8WlxIt)@+R0Ovq~7u03;;rzqqV>&&X zHwts&1l9H?JK31kNB!%ElaE*V$$(Xzak#G8NmqM5{XWW+=_);$t*XzeHd(IJGQI@0 z_SaAS=maiU;dICwJf&wOROdkzBw*vK>F{!IkWI#E(Q!Q3`;e(dphSA$ZDvwZ2xO6Q;Q^nYIan|Duj+6f*1;o#HNKmtvDI6#*_pl9b$ zpOPN-2e-fY&n8R`&?-wm?WezL(5095(qGgy%-Uio0s5~iDtHz6%zk!YOJzskcOjne zoaWEQxO?%-_n1t6OiN!M4rG#VAOt;zkqKbne1l*FhV_|85B`}>(^G+H;zxEH7YKj)5$>caB)cqQ#z%!@j4iufu*$O`*M7PCekH<+cIlU|cL)mwKI`if1S04YV3<_LZEC*VKda_cRD*mPq!RH1N zJBn8}TG59u>Mt&b*qw-VE`Kl_75B$xyYOiw44-LF@&Eqe+4FamDXw=;@nTO@j#uT` zFaP-F`001w-~2RBXCzW8o0}iV-iHtBUUmcK7>Ky&GQX*2vl--^2WmF`@-e$uaM*XH zSSX%&kv<_k#xu^PGfg4&hw&_q)?2*eGfmDLUVYr_2W;Wbyu}t^y~6g=_kQeq(PS=B z)b|5M-*OaXPi+I=B*Y5WTLAcDaO_J@!Q@O~lg@|;=FZ?L5D-&;8`!od>hp__Q87ZM_@mA3pr&qLlI!KB!7nX zEF0SZ2#(uU;^Rkbk31)cINCoIYx35}izlE2w}P&YJ^3B00|5-yUwlh~_`P?LX;wj0 zCf>!7hY$P)v*X=quvo-`fW@@&?F04l%}cOC`r!$};C7Mo&*v$lNUyoVS~efLrOEIl2w&qFbJ3+B9gl->nLe;JEw^X?rwcC-mA(^fyERNRcx${3L_a*P z$tWH`8*hh62xyJAfd|`fV?kD5a4Q(XEC}h_FbaMFOXDRy#!C9fySDtsG+E_gJb#FdUT+D=c_t4G_@sC`rm-gkgj z-PD|i_w$mK7^2^g^zX_3pi#au&4Tsfb-3QF0k=V%9ETTzg(+#5-(siv(g(#hjG%-d zhA!&2Wr}|GExIk0x!ZZWsz0Ps@e@DaaY@U1-JjzyJe6_F7#91%vOla{+Id0No_aq- zlkBrJFSt?dN0*4>5s64RWPk@ckzti73K@j&zL7xJ4!MyCF|haESsyvSNep3 zB~#!na1krxs{>vgbXp>Okd;-hCbK-j?8-^BSsvqEXxVxk(?=L}_7yDej5SunRhSkp z(P;IO;t1(_(UOQ=8IP7Q=GSaN#z$;3E8ym(k8vaTK0_s^H!vq9!r1{Gl?7h265-=T z@JJAdR7NnS%Q$0pCX|0E%x?L5pce|a0y+E)7iH=GP6kLvL^&rNP%)fbHhLhxk|0_5 zMKShlqbWGo1>Djk!tCH_S`I(=GbE7Vew5I{^()-YlseWpvtmDq8Lz_Z;$$*M(RGP$ ztYp=sCuU23d8!(w{-UFfPKNDfuj!8XN1x$7qA2iTw3nY1jq~4gQmelc`%!R`a2T4_wd1Z6L76)9p<6Yv@O=^e0q^uDe)&F&o=bwH_6lxs%Nl z)$HRq_Tw0nOjAMd4dy|o7UFY+KbKN#%G6ZDE~v7U|mk7hm$5#FkkV>w7zul@#G zfa9iHF-fhYydqnfky>VZPUdL6(5?q{6D`kgYzwLXBk$jz9#<2R%}y!x4s;|))o(y{ z&K2Umo`%2OZy78u``cyGyxFiJu{Ws(Yqr9f52moJIt>J*w#E@Up6!LI;0*ezKd@zt6na06 zTXaMd&0Dmz&ztJyyoa~Xp2w{f2$%}Ec=0SaIq^~L;FstBNomnoPsGXXMMvIZ<{w91 zD+`U@l^$2`=1<~eUDntz{q(qYy-<1JxJtd*H740>d2dmRcrb|a_OV;e;-ZtxUm|cx* zkw%Wd+Z-8$U%O2%@W+|JKfx{SvMJoK$a&;6oqNIQ2DFGtX+l($aL~wp*c1UNQ&xn& zMs?I(9@zr2?AdHoL?t^svi<9)WsW z;w>Hkv$3a@UWcl;^*Sx8EYdb-u^PSw9@^#=VoauQ!(^z6#X7YG+c`q*4B3ljr2)uj zRq^lIt)lRBj6a}>YO5}{0T2dU4P1zkBi{qb!Q`g)17eN0>|4BNMFR_5hj}B`TIb$1 zxpyt@-L?un4VJ+ef5@je?>eVwt)0cY-G(0hy76)o`$zE_y(4l&1lnlrHtEq|(8rf33<(-Y8e+O-c7f=>ku%Ws#;Gr`6yZxxC* zR#H<>CPXGWzonM7)SxlHCAC8=;8a;P`&#BBqs@uk*E}DaQ1hObTk>MF^7`7u;i9ie z;ztp;AB(QH^)%@NrWX_oh{KRjjG_yJz8C70mw#=vY(wo31KOV!^1#7BzHI(CJ z#Y-=nhciE!`_UMuuuN^Q@hkiZBJ$lN+$2EnCjiiM`DJE@wYslt)vPolihwmpmt^HfRS&$l7th1nx7)ZRsar3mVwg=LY7UbYKYU@_0bMjABcy9H;}@K5cP*_b=-CSl?& zj_PpHjy`}1eIvuSTABYUgEE+w}}QC&ajKv3me_4qU)+{ z70tY}*-@}sZ<3=VJoQ(VkzgH}YAfK@zGibTf;rK=S)htz^za9c@ob}W;I6Kyrqeg> zyBKhf56&MnSR{_qO>GBsfot?q8(e4eZfV^zgZaY-OWiO#hiw6^tzKFystdRycQNj^ zvBw{-22}2z(}u0+y@Mr|uG@D)-BO)cic75C zp%s5l;@Qv>A8(^5a{ghXUqiOep^34cs-{@^y<4|8$3{61b+m2~j_el&` zTg`r=n0B{{Czh(X0y>IRZc3bU4 z*Az$9`scc>R}8{CBXdVohmQeOaR|p@GYA;7mH-A!m)r^AyMl^+qc&|0-H8K}LBNF9 zOvfy%5qjcFpGSz5`N0z=0#CquTd@_w8*2p2jidT*Y>Yz@)cy-MUx) zi^$L&#nrkco}n8`zo<7sU1;}0$ywqwcNEL%p6MC^f2E)1hN-LMehB(jGDh$4?e5ub z5m0o7h;`$t&V7p3$V6=LCP&H>)l_Vw-+8l$dN%=!j^)_CF{^HOtkMOy#TGE?RLi^@ zx1~Qdjz4@`;Viu9KyX`Nx)be=e73G+k-jloW_MgpbAjXnb6nRn>psZ1Z6zFxo^)$A z*9L3H6d}24kbC66RTAU+hOhG$mQ}?z1&kVyg@$$ymZ2)CD}BROIm0&4FBNcIa1lbk zPqti2AL~iybyFNq^)#5R}DMo&s-1%BdYa0uHxAQ)(_B)4Xq@Y4>V2TP;FvYeaykrIBvVE;bocI4 z!rJ@KdWWuaqK^)Es3J?~hNIv4*x9@K_`}x}mltoh6jKvmc2IJTe)s0<7A6hMVooZs zgPUF)k#Rfb-H*GoBugI(RJYq}wL$9}nh@t3`u#9)1l?9r&GKdmJ69KBdB;)j(wt%J zzU&*eZFk4waM@90rfars-ld@H+E3aycGV)B96@kk1Ohagdq*ow?TzyM;i`(wTTWEL zy6g8;bprG#-s+Y%)@C-YwuTxSJ0?rt)JyX$uK z4O=rgYY?`rsD_B(gOU48} zZYa8t!Ut#Hx|@vvebqqt!^ixJZg1W+6-R{sZPfgCF?@b`2>#H)eK9#{;w~CJLVJ9P zN$yvcL1&$`K7^b58sqt}PDmKSNz!;cFbWb%bTe}Z=M~<-b z3nN!8&XyTRu>|Zx@PJ&?xL>oPUg59@2^^b1(L}s|!?Ydku1x|V6<{bXCkKUCVwv-V zn3YKXZ>j@!EalUO6^B#V1h+_9A^{NH(vAD797NEd@iJ}+s8EqGkYQ=$UYQ5gqELO~ zw+)UoR4v62af~$0x?kH$I;*$#p+n`Qbr2P2jeyNWRM)ZZQQ8)#JTf02L~%5(`(zj* z$vrx`S4l3-BF=p{;N)$Bx)gDsG~#O3U8~EEx|_YD=VUQr0KpV76p3Nqt5}ji(Oy1Y z)aqsomj0|b%pAO^uCCpOwIfFT`M&WRyci<^R12i`Vh+1;KSjfq*w8t4gn{K41X6v8 zP4332=&yL%JA6(KsO~5>5eYJBnrYp&jNDp6YT!R`)>lA)+9DA)+>)%{wLc}Yo&+8; zXMIK0z+mhjE*MC*Y2SB+Ac75Bacy3zyJCrKUjy>rtHemr4O}pMQ|Dz%DYip|)ZpEu z!m{g}^M`B#_j1-#LG#&mf7Xnlnw71tcNSU^SltI8K>CPQ4 z0r9SP8+xuKP!_`v&fqQ&ZN1wnx02N0+^7Nc!n}AN;J;j!t?Q3N0@n$ zpM;Hu;d1Rwl5gqReT@>{yPZ#&9i8{eyY!=fMjjqA+H*9e{T0b{@ z6(*Qnh3Ayk4y61$p4aRJD?gGdC%+zhu|IgkKgLp{2MpHEVwTnyvM&8yyIAbQCweTh zk$%O)(C2t?GW|yTiNf>BH+sM-{aQPam0tOI6DPsRrF`rs=RS~&UPKUzx8avdnMi*( z@zs~IHt0Ezy*P_gF{Qufhp_V@LY&~qZ`Icucp!Nag}_f8r+LQ9v5_4dOP73^r+H>) zpwo^T=cE@0%~qZR#3TWIu#caMlfRiK)W`D(oWmR??9*`d#=-3ie}+edV>@X%W#2pr zW}C?WW1ZE$49|S|3=u!^7W{+SUlUEz&hQxU)*0=NL>$S$zm#>_w=KMb_t{6dKYsf3 zhwmRh{okCf6^cgwB0WMfSmB6dq$zZg9{ElW{@jnu@5y?9AmPc3hlBJ^J`^urhwBX> zX94Eaad?qk_b=c*Qy-qDb^WueC^MpP5$An=OZE%2#q=ayDPBsmIVh6iK=KBwq|Ib1 z$Jn>%X~NPn!t-+faGW#`qu>{mygj_M1b%T+evud)^IDULHfOS$B{?Z4b@5VN&P1CP zs2n`A7Km1ejdsI~IFq@52GZ(}MVV|+vAkK!)9`%NLmFEmrbPeBCW8RBV*@*!^lbJ+HS^Y&^}~JbgaZ;_`9&>!PmCE3ZBa&Vp%|p6^xhof=$68zJwi zrMGJ;n_yEF^mW(pr~=Dyk)5-hi$8|MTSs7TqF|$BWvRD#38E5B&cnguG|=3q?%Ma_ zb1#|9C1k`)?5Kj#3G(v(UtYYh9;}?6GZ|R%&c7f&9UTfyz!xYLnBzXF%b2I`^5UKQ&Dvkx&RA6Xr#Y7NeK$Ugci^7-|GFijxKFevPn!eD-+1Fg z`4`9HwCwo#7uB9)$<@!K60b3^tz|Tw1=aPn+ zKH{1tpUd)JWjP%-p7Ji!0Ery=XTGBB zDJT*}4tiV1{%SG@W9d|$MFCix6k*BSUjmJ15g0+)r8O2QLBpQPn20h6(CzrL`rLvBv5rG8n#y#_ z;Vb4^qxW{J!a;~+iW(Z!kSguI8qYN*e!R&9xH&iu7C~~E!rkso`ikDz8gjt6hn$l{ z^otZvLY!qVN8^k}@l|5Y>7D`^{TshV-t1R*Zq(?sD*SLu%G=hqD$eS_b@ zP1VU0>E-(tXtIZf0#E%>meOb_bxWe0r5_Nc2I2S-r{ z-vM`z@p}t2U9rq6Xa)tXhC4hE&St`9io%BF|BtdG+Xj#R0UxA)=q*_AptdS3I>JI~ zXrgsAnXp)*u%sy@yLN1hEH{SSPR z;G3k`PZ)3+wfTr{FF>YO0i$Q0F#&U~%l3?8$(j?vl!ouKTO$VeYTul|L&`KJ^zP2k zXN9Q?P+w4uVYa}-GsLRk5h_#68rkVlVr&43y4$pE9QRdBUtM~E*QFuR#+o@|fot|L zg8}o4J$K0N`lCFYt3c^HdPy=B7FKP$3Z_P9cmt_;LKhoM?Yc~Bb@Hmk<5i-VPOs`S z2ped9C&6S=q11syQ*S||*98(Dcf|(B*i|Aey^D%OdhJ~>0MpDCb*(y<2wNNxb0;jd zj*QB&W=MFTQVT*%PpnT9n_|yU8pPB(h$*aPvU8n|%5$B-Vk|y005k16U;tG>s=pTH zF~z9Q4M#BzHe~h=x>!@JTpPnj22h$-2g*&cRg)@9y#rUyjv_Em%_@Z-2s0}Fnh?^X zk!n&p6?<;YvcdzB*burq;moWEksNs@3p5r*^*W!K?CGHGsy>)U3{?l!+pO9({SmV+omt2=^d5nX{tDY_)wG>tP4)bqT+C<;$TYjo zk*S0sMuYHdmellJ32(P1jomqEG@UVN>^gOEvmhHpuq(zTOkY>>-G)P^u|MBI${mZ4%0iW7n(YPmh=ifGr+1lEGnQJ!wijSL%N)0 zO<3FHXQ~`A4p$iz>r2v)MlQ}5$7mb9 z^vsT)AQbp|Y_m<@#f!;K*2rWJOld|C3ZXK3^dU9Z*DMjbmUbZ0-CA7B zR9wrT>l@;lFn}{15pj*@KCtDAq3#xRuxepYixmcKL6`ekN+a;l5ui1MHAz}=_o#M$ z0{Zl?L+MBOiaLY=oZ1M>8M>KL*0Jm1e}m0{hIVs-T1I^KyKt3sg}vyZ2Yq2LO|jWL zY3RGs9Czqei2fG7wm`#!3HsulfYT0|8#~FvQb;pJbaRo1A)@j00JexWcJx`aRywdo+5( zDFh8^q=Gc}_lgKJ+kiCH9V5(TUHLM&JiM3(v-yYtrCuYjmWrjO`KYsnfVrzKGE4li zXsg*Gn4|UI*>LR^Fss&R&rwHM;OYIWhZWk{bHd!oXK5Bu&){RO8esKwmlY4KJJVQ1 zoeklROD=_E&s1So5L{CXc3`3wa>Qci?0A`J?gSsB9(fL`^y4eG+SVF5B!xt;8EdpC zIU7`KcZp8TwTk>^*=wW`cI=RYSo4GppS>kB*6Mo?9)&ssB*v0yZQ|0R-fVET)fer$ z+W-@LEsa*&uH%ZNUO_$5ZFf#wVV!~7T95L|*g!-0#L7^j1+n?-kH)e?vs&tPO@(qz zsM8SZTwtbktOX1MXvQYAMQ+&A;SHy0^6-YGJN?6=HL2Od(sq-acHM`tnZk&cjBYq* z3}K`#%?{x7UHG?E+qvLpO_he<=U3af#QM&B=EfS&Yz1x^JK8>tGR&m`^c~z_fT%ib zd%e!aSc;hqrzyU?o|SeuYFA4}V~AdpUZ>k>^|n<+%O&?O;Lx(kY*2+YQL}X6QC7Ne z@aSdwp#>Y(Gp@mw%2lV{{(z|F0;+`<7zU)fn6INSAq2MEa zq2R$IT1Qj@RhBhq0gQEm8_cU^k(#=;)w`S^%iT5B@3ht(9k#7UYx)YTJ35NRW?Ngm zK6B*tCR-fA_kQ(eIEld$Sd+H{aI%ZEH4Ew5mH>w0+I`tdCYwQRVZD@>v^^?I+8*&s z+Lo+J?=!%5nvSrO#fqD0FAiH9*!oQh2&P=%@`2V^oMiEV)s^RD8`qV)Y020^`bzKc zD+5T|5LiK7c4WZV=FQEsZV}8Az?!0xJ}lEJuWb0OYp#1R3$Zp-Pb-76;om8`*eu?e2k@AViQ4i*Rg=XDSiX}D<1W5l$CT`y zkhIQb|AeIVHrdb`yVy!rAv0P@5m*UR<=aZ23E80u$Gi>c&?gjl0-9E`?3W9)q9uDs z)=dnla|k`Kx7I;eupACu7CPoIP3~nT?<>+tuq}RcCjPzbxihdi<~On$(J- z(o&9$^XlAa%(#tQ#iSxrI}My9G9uHtYx6Y*+&LGuvr9rdx~4?d9M)oJx_uI2?dv zN~`r{@E_5=-{WG>@e#_cqu87I%J<)&zkK=28&ml?riBf#G>xB3<^yRGg>f84!3^sU zfeKq)E<=o90$wV++ocWeihgzSuV4Nf9toH0aOJNOc@r(l_ddzr`y;&eRfyZ2l_zU? z@z2>mGYsR8AId8j%cc=&TW${(=dXpR02< zU{qJgDhX6}>C@^2$VpyVMc^0E(vrViCztzaX{@}mLotEa6$!9v91O0kMsOi}n zGcLUg`2E2$)b+(jEHPzC>tk6nB(MpIB`s!HY)Yl`2Fvg*{G#AR@rtRRc)=o;Xr)Um zE*DGH3R&qJ>cz6KQj`ImewP2qiye3?Z*dt1u^gVrRBp+~fd|v}aduq_Pb69=Kzah# zX?26ozFI%}r!SpgRS_wfdx;c>OFyr#f!yRzr8oiGdzHY1jA5?Eehl{`A9yT^=Zi}T z(1u}5m_FFUN(wQ=6z_jEUPJe7fk9q}-`U^dr1ejN3y`=@xAXuopJK%h<;l5u`NNa{ zNp4GKU}1|^QQF=r)kzp0E&LUBUMXDbD5hiUrtYYOSgz@o2TFxU08UUnD9Qv?;5wG+ zI)+_G>jYQhH4qQW*bgIf#<+n{B(cW|M8ti)W)4tbf-q zD!uYhjP|?c>bkAzlu5a~K`hOJ(YcPJs|MNSxN|nd;8>^@v5cJ3-04Tjp&2@{ZQZfm z@}Lq1$n+c+L)A^yv2Eh&H+{}f*X(o+(-o9}Iyg+Or5Uzq>AI;oCWl|Z9R|wgqNF-I zw>lQ0i)}1oYNl=w0@P3qoeQ?6IEL-$M0X68m^_(+d7!%GLf?4$sH0n2FW7<*>xymw z!=(C8bzHvGc0ic8mTNnpL`>D-NkNc1FezxCAnB_B?{GpHTc+T87tq@-gm{lo0ay&% zHFb?xy3JvET(MRin5eb|vXc;Z=d%UaaGM}6EG-ijgezN6nW(PWR;q2NiHql&Ynn+i z@p4nA3bPF@^!|+sLvqz8*al!c*@Z;cILOBYD6sd6soy_Aw-hZHMlo^=NHw_XQ^#10 z10qGUKr`A}d8QwO#6~vy;a#>xGK-J!<)aC#k3wG4Rh77gt8>Mp zX@CyNWvSj_AvSVmg(s|Il)P;7Wew)?Yc6q#s#6iEafJ=&Xfx*+y=u8!=g_k*7XxOl z+t!^-#RE`AM?;DSm*hl8#>^UJqDO$g%k%fQty#LAPgM`RIK3=P2-vB zh64D98KXIdrGnjg&BJTECNMEUL|D0i=1yRk?O=A6L*)Q(D<+UBGxr0 zf2v$SagZB59GdOHVyOwM;!g0!f)W^nBq?lp4r4h=WllJt0k3tmd6ywap#QdM#+($c zLcnh%PWDgZP6`Xil@S57H(Mdpc6bO*1DM8^=>osn)MnAR*tS{*FjpIZ!<)(~kQ2n{ zngz8Q0D@4%bOaC9(Q@&_75@Yn9-JmH%M9?WK?!icHVK}w=GwXix{jz-)w|Vswq3}! zflbrk6*6LvnXj7$7?X}}=&J3yT!9ZZHKA-xgJah?hN?@zfaL|wkzpFt7BKoEdLY3y zSJPeFvQ(ZYh4TszO>(}GOsHN{U2qOe$FW?VQw|cvgo%?` zAeyla>|2U>;CQ0hrdjw-{6TfE6s9M)#OpY1hGMM5S?l!1j>M_djLmsg%R0d0Ip$(DVLq#XI8iZ3&)My)#C&~M)OXyiU6(K1F-`xk!4$8lxe1^ zTU=CwHDKrNAt0k)o><$85kmog)j*VT#4^rivRq}qDd1HZdGC2)fs z9cCgJZZ_YpGAtmF3lhQ9z;y;4&UN=34Vc7**#ipHFokSFOV}864V6Xv4y#<(S!-rY zy<=;n;yHHD$$E7%VFm|WTXwN{(zApF60Mk!8V76>*vVCkWj?hQA87E%Ho;Q@wy|cH zmVIB$U9E(MdGKPm45ngbu5hs2j=2j$uYv=k3fkWqR*|(F*R)Ayqj0dTD~1Yof@2Z7 zv`RXZO?5xLpbWv_Juq~>^u@Qmai4hAbmY4XplB=32>a~Z){Z?v8DhY!$&5N3QP>kI zbuHa63nP}hG@}#ZxT>R5e?;#@bfYw`_-Y19C*4X{mCPP>V;6MM!4Y-13}RZuwh9LC zguAinP&_6JTcS7!{GHI9?)Uxi6wvves!WZm0$mVqMiHy)nslb{dUGWs zgV=7Lu6Dra&(~))%ix;NI)GUOoQ&>CrrF2Bq6`?YVmdoLcm=0j7Y(_>+||p-o5gbh z%bm%>d(q*hEm< znJ_n0-BQ7ow{by2^n81^)Wt{Swt=lYm#nM0uQQ0t;ot>uvpn3$c=VgWQ^1!1XXb>*Ea5rA(9 z$mSw7i8sA01<0VWR(`9~%M?xzu*}k?viKe-qDrS}BDu%VEx2&&co8CnhnNai7mfkx z%zTy(f1;j9XK^E&wc%Gt-`tBl&h{dUFFsp(nMmjK6ZWaYg8HmoslN~nY{rZ zQ#g1G^VHOeB@m-00$%DmBr!ic=$e|IC*)RhS-Vi2Zub`U*;Fl+^=Tl_>0w|S%*#UJ zTHNHKS~T6@=?6_X-UFhDwanCI-e3lE@rQ3Vk}}IguP*=9jWK#CjIe*EYgV2Sp=$g! zW4iypme$0J1QN|P`xo$w^8SLnfQbPhYTEulhld8Pty9LaxGDk+k5WL}-2m^9jOYMQ zs+MVkA($>Ia+v~s@-lJ2;kv;#rX-lTEu@=lRbdHRVwM3t6~Py-0yb^M$#gH(tGu;I z6(o5+14`I6rq|%s7BUIKF4XEXsUx zFmn46P6?QzMWQS(uLzItscbtSge`|{VJ)ZI)lDu=Pf{QO!%Hj7VS}4PWH{hEXIj}L z{nn){zEeu8#Siqpof_ce8pR)ri(Kf`DrBJB!NFVjZdfzf6ipq%ygye^ZJ-?i-e8(0 zpc38gfK}Xw*FXaZXLn=1kmI`L?kI=v6@y`41otXBIjv+oZq$PoUmCZoiQ0V%E;Btd zlbP!lgR$-u9AsCzx-EDogYgJi%uECTyD?U+QT*tCpF|!@| zwVg6>>2|5N>>y)d$zWIVQb}*ftPI&1yo@&!S3>hV2}}{{wb=#5FCHLUNj~O>{&nPF zz-}H-H@VijN$g5mtH82~Bk0|0iU~AxG{dy$7BMbI;BigYJZxbz1HBBh= z2O=2MP_c!!W1THK>1m6#wsg@W&FJalhnkse@&gdAV~`>Qq4jP$5!V(lD=fn((wX@{ zQ%Rx~sRlhe3w7|log!WgFkiJwg8azJgTVkBv)DlKSbr6fc(Iu^TVeH*X04IVLLu+6 zf)0=4*pK5SemoD7x%7>{@*^(^&V1=P*5dHWhg+jN3a0%qoOsJ%aoOMQ%#UIQoTT1r zvj)uht=H1z45G@a(?9^AIrfvIU{=*U~l{Nt(* zn|^vl?@EuWcZ;dDNy24~4YLw5wd;B7iMN<+YQi}E^t5)pq9AGcki0e7`%pX~R({~) z9{Yj1Im2ni$g=EgU0K8Dv?+B_XCqsrkt6W7N+7%1h8r&wXeg8ZroG$x`X4q$KcH zAKj}o-fmAyVyH6Ovo?0u;_KHN>et#-w3|J?R3-r|8GR@2#vxvPZJx|QwT z-IkNJ&3$N_SBNp0z73N#mQ6YBY{7PpP`g^(huNx?#aeB}EF{*^pnIbt{{gYaTlOv9 zb5~}1M{AvX*W}){xOdwsl)K}ptDVKW-G;t1v5wnDYq#l$E!CRzZR_JUZYF?M8gtVw zyV^19aC9|hmUZiB%-GP~)v~>5u&ZTz$47Td=9Y!-=FFV}UCn!5UR~7bZc~JBx|>w8 z5ZR4t3!q(1`he*Lr50>zH(?adV>IfhOIB&g&ukNDSk2F>Hs7(_L!}FL9Jaebm!9ih z^7;6&DDCP+$;ywa9{9&B5+l4EznCD^A2w6lxsqhAZig2U3iJVs@^8JG*`yk<75g`xDuvR^fFVUgBGfy z>#A*);ZA?K#=zbts^Ge^1#Eq^F0ZdPi$y^l@8Sr!d%a1HlJL}D#Z@pJRV^w7Gq$5u z`zdZ8`6nk~l(1U?u8+ewNG{p8vFz(6@mpOKpf>6oN8wp8ZFAw{oOJc(b1wop5I0`E z<#r_ou}!5?Frr0lS69?>t-4KXqf900qV;VOSyyj@#u9gQYonLE#L{(px0$Re zuK8P&w>1xMk-D$LaD|4S^!VjgBo^2w#w4_9brae)6}`kZi_}&>sSVq;`BqpJEqAY@ ze@kM&0itb5XOV1fCZP?LbQ0RZJ*yl#Tm7n8)lG-kw>hP6lE@doslk-L~C7k#*K)p_?_&gg3`9$W0W-?KZ_R$*~mO4087p z+rqts50?1tSmJ$!)Y$Ykx(RM*SZA0wz56My;%-;>!O^}YN1Kxc&Lp>OYn=Iw8&SC? zy4EkwZO5#nkUm7uZd{!ASMXy}zLF%{ro08V6xa9G-Tt_#saHyCJW&4Hlni;R_fZdqbnAdqJ3l~~;$ z+ETkMPi)H~ehfNe4Oq@%mq8TOvaaWSsaWpW{T+o_?){t=*x|$^wP|yvG#-qo^VYib z{)ig5|BhkY|47SkG^I28>>3Ye6$NC{RnxlZs@aHH-W1mQ?Jm`Fi0kY+h5wMG-7yv} zx&yRLXTwz-r`x{9{ZTGYBkTQX396|%hW+43-}uTFw~{-X)E38;vUTu*U0w=`fL7KV zTelvj^vzFcp@Q2?qVp_n3k-5zoXio3lYzx;I}g+QHWN}zF-jX{y^I#AF#(MiDA8Qf z??k(rtyvFI@LP$KaL+(tlDpj6%n;@d=yhH`p-9Yxn2vheE7>;`8vC92SfbU`*!1Rv z$nZQo!b?HY1a!Bh>(+xHzvTrF5nAJcfZdjwp&JU%CRh8TWTt~PiykETEv;vE7Lj7o zx>6sx?PwFlwRrPdz$Q0LT_q3m`ex?!cE=5|>1}r_ZVT)189Xyh7m2I5>VrjoTY|j1 zKqr&hCdVizDxS%1@|G|A!=n~aHTOZ3--1!zQDu=yX4UP^HgvHB8AoGR{AZ4vZJEET z^^SXK*_CU2)#KfyKKu~l5n8|w*Bto!Ai-}>f_I*=!63J(I6C3v6=Ll% zo@HhUluOYbqGq?Y*`vE`H`dFG7ujOvL>cCDaYqQxK%I+U-Vbno>FVg_UKdiR`m>i1|=P z;pR-S&QlEdq&5^@9txggWbr~|x=1DpG5#b=)xW{sJkgp*K-fxPvb|PG6oYWn{tb~3 zNuM|M;5qNSs-bPOpT|%t$AGf}-Ia5%Svzb+G7&Uew;yKljV)q`CYczD)~$-MB8tr| zgg}Irs>s7MzU^tuKB%IP_C&RKGTh`&Yh4qW);h5sruEHC>sGycK6$}T<1Av}0lqq~ zkgO||W1<-k=|nd+w*^lPX42Z^#5#3LAqKhT#%k4YHSNK9-rafbIXiF`v7xqY@wFMV}RGN7Yq zDkmZ$P%9Yd>*&sdR=$nYf{sIq_!QR^mzUOUDW=A&=r5Ai4K9|WKal*EN|9|5DFHRi zsTu3w86LdCMIy7?bWHuhqq)TGL zyF$xWhVel6YZSPsIncIfzL1U?oSY1O+g4P=xTa&JK{pS?y_RFue_A`A%#NZN-J;vF z6rFpPsU~2QS-Pf^hbZ@Tl(fjPH>39Z2H9?CD}0(~q`I2Bm050q&gE_3 zP!)r>fkRVWfdhAJ({&z9w`-;Hp9GPA9!96}c(L%7o*^$ifH9by%E1bx$7Yhq$6lOw z%BQ)h*c=xevy!@V%rJ_si!6g!w)Q~s?D~tGc-Jlc?g+1G0+CgVb83g9So(ES{Z*%O z-qu4oo7<@pY@~ACB!&>xIbOJ;>D=A4hDiNb+j*GQ*DMfRal?gdb3<`CcT?Ah5^_=f0a#Yk*P*t)O>51ZK+@%$RR~w$_0=B!UlZQp|+d0QP-A$l6LYsns z)|@hh;HL3cS2Tg@b;Q-I2RH4;+%%x?i5Dl^EtZn%KI?X_v5gawM`i(Aof!5*U7TA< zXsRBqxKp-^TCMuea@8u!pA31fiA3L2SAUq~HjBeKCuWh1~(yh5bJu!TxV zKltO<+Bedxv$v$V&hg0&Jno(oizALe)+(`y`9SlVJ5RGnff3*Rhh*l|EYuyvCPw!X znImG4gGX*Yq`=%t0{Zsmn&v7tXTt$lb2is1H$=+L64QCG*RE%;Rn|^K_K|3g!r5b> zYKp--utyarFiZY__TF?ik|RqO{NGOz%?k`9Al+#DlBLvbZB-*&CAzw$zFn9?A>j-< z$tcFcjwP3=TR=a={JFr)%k`7YvDoOr2tR&2+&z_48YD1dF<-(zJ7#D9cyPw0$jPsv zky9psW5Hqlq?tu$0H@?K3Xv4?KFSS<%sqgyQCJ@x=DcWcY^8RUtuQX_&fIZI_)8jC zU6CR{u&Bji9jVNzF_!Sne^m9?`W)W!tH0 zyKS~{S@ohcTM(d8t7;cmu+_xnl3AK-B)HTOsVjYYV>3->(?)iaFf1UgQ!Lj0T~esE zVB_^7HL&XpWpO6XvAG7aNV8DgsZcUu4MmNXA#pgI2lV#8i@{-B9bouIH zJj<^y2D69d;?>0#+i5=8J<`oT-;VOdcW~nE4eowgK3?s{MRGYfcm6IPEwka>XnZ{{ zrf`|Bzq%g1g=^&tIQ?|CjE!0r#o%UKykGyxhO_bDpN6yPG9OKg`QR!a-orFpO|N6F zdNC-jZ#RG7z|kb1e>gnj;dyTJhfHopUmYAYczB=b!NZ444gN{JAHM`6BD>23-@IEN zKcBrTy?Zy$A2x5nzL+esAv`L$yY(l3T`iWQn-AGlv3yq))6IwR((iYtk8eI?V)F*B zD&eMKL!C`;_xBAyI9n9+(am64yoZ;QOFdYBxT(p4*Yjfg9QHr_5A*yQrrRK*F}V6L zcs;u=#$R1-&V2k>7I*W}^giQ*$6s{tW8z(^cxMm!ba&vmxGmt4aok{XJy_pBaUGX} z;`GRy(Jjc|>F*Zr^5yVu05fj+VO+erC}9g|&E_xv+xO0Uui@y+c=+z0UuLtx&&&B} zxZHjIE8b$z-??w((_8$v=@o!bTz>*L=zjl=`u@99&+w;W8by5n1x%3L-_l%pDseke z(fe!O6{Fj`&Eq$F8Gm2~pRV6!qs2I%UVr@$Up;*N&#U?1t9)=bFK%94{I^M*=UMSSpFE6< z#l_d__k-PgIQ;9uRgupJ%e&EH01Nvd8@z)hCjKP&EMGpJ>T-SPV!hG5DOs8qyN9v4 zRCIxj<4=k*H@hF`^)+rm{EEc@mchl{_`{3AJ6Pq4<8yDq3U8kFbw)1n$6p4fe1-7_ zxA6z|58n(Hqsa(trNL$|(`6qt?BVzSW zep`GwSiYOZ0M`5(9?#8S^Z0iA_h2|%jj!PW4)P_)+!bs_aEcjxJf9cCCCq`n_tGi63$ef<`-XoyMO)F<(=HjxM{Jw@d>COaKXt09^ncEVhB>Phzl~9L-E1q z15i7%|5z2Uuf%$)#Q^>EO_Km3#uvYq@pjf9&?guR3yc=|@Z_>VS<(phjoFV91pbWx&Ek?J~(ai{!mF0RO#3u;K z)eat*6Wgb-MPxQ#ZrFYXFZhinu0SrETaQST>pWOgN zB)=Mu7I!eo=c_B&2&2#e1fxm3lenGCVF=*#?rxKN4x&7dr-@IX7~V}`XSn^azRm2al^<}NIK}>MPOThq0b3NXP=CL9v0pRxzj!+$E*hY$#hZ?g_+~a9&)&tyck2G( zUY5n}Y(6R$FEdh9jDoH9b_I(JY@cw+gX{hTia0z*e~-+dW;-#DSZy8*-1m1Nq@ZbkJIm+Su;MQU4=Y$1hJ$bZIQad){>xtL87wx+ z=6bo3lL)cdKlfprVWZnAv0e>nTu5_MmYso5aLes?P8VC_=?Lw`L*UzE z-xcHQ3@nltaBHKtQH_ErIJ(|Q&34y#F(@*Sw(;ZBj_;PUm+K!eT>cXrvS9h>;_D|L ztYM1taWY%q#(uHg-sJ+ey@%!3UwrZ9tFQkC!a5kg`eN~7^kNp@k72u-zseUMro;93 zYsbwO>%W7=7hit)#g~77v3wPmBUn>LZ0+iK_3C1?r(XQ`?I$na&PLY*{$lv*Z}Zt~ z`8uE9j;1e{uU8mL$R6oXtwusyFZ2Hevi((Fs`HKSX1AfF8 z@!s;dmK9%)ZobIbXz@6Fdi&+yrVO;xVJxE=F2DHl#r%see|H7z_x<1h{$lv$?{;Tc z>+>_V^F@6*WwXtNz9_!@`|rM36yqEAO@#e_{jbuSSAYBamtV5Cqs0i;<5%O~!Tx9e?rVm%sZvJi6b(eA+zxuga$FU=`Jgf15AA{~s4| zuHKB^zq*)5;}ceLu%D*OY&9Q)CHZE3)aAGN{15Z|;f{TuJzV8~{ON)Hbx|H%ypI|w zC@7m%^#W#hR1Lv6IXKP6(Z{~~m)FJW=kL_|BvH42^!M$zm;VWy$dRWv)~Cj~di2Hq zup`GLC`E@($;Ehe`*c*aaSlg;@wBK)u!ocB93MKktvAHI@I%i_-@jQVbxYPGpu>jxNz^BV!&rGgc0A zltG?wHJ_Ep^&ZDO&8xlvv#ZDqd0-~$ht^ir)gNpqw^UKb)##80cUHl_+~|a;^5P%qh}smjb9_0&*sN9zi@*C_ADEfqtxMD)sBWUQpfJ5*unEA z9Mt3zv`MC#Ofq%I_K1vg#BrW9PG?|oJt}}FMy*@TC~=jImdB$`=3-V1s7@Hc51->H8`oyJb(?8L{Py5uC2BR#xRpU-ly;Ti9Ss7L__ylKQ z)kwJ=DPb@^@|aMxdgShx9dvT89)7a%bJg&gi0@$GW9%*SsI{Sy2|5T&M4G;hw| zNcJ0=+G#)-I8r_%bvYgTRI5-zO|#<8x;|6zh4i0DcIW<959dqA-ASqyJsj-FWm(;7 zEGWiN4u^V@ywnf2r+%=9jusqcP&={CfgU#efc5hylwTNbK99zMA6cG{%sU)uut0Tn zUcOaW@l<5qp%Bc&ToQdkEv*!f{SSZRa(!&J;Q&hzAUg z_ON7QB5f&^k;V&`crW{IAypZStHYm%>b7A%a@!*2mge%EV&ZM2UIW)MbWVL_KdZEpglu`)_)pOEjRFC-(% zw^7yd?JVPnRaWB1Lg?|V^v`BBfB&hsfvn73tbVQzc(rYV3N3 zB}R#e<2c6#y;wg$!ML^UtS|0N5aAiQS>Av8?8c|FQ0kyI4iRbLVP(f+U5Y?zEKP|R z`Ll0Yv&H9Jf4C&|bOl_cMA(vCe6pAbQ+Uv<7d4c%hzS2Cp*_dhL=S-Fal|IIpo5YKu{qDzJf3wpzljEWJ}UiFEfOU7fmJ2Rf#4TTsX` zKbfa=?RCoeB;I`{q!=2~T2{9LjR|O&qeO=K7Ox^mCvEbA_u?lxPG_!zTOxeL&cnWr zfgGS?Ib!08x2#2qpsuM186@3Xw3p#Ri)T< zNLz#O+XawqWW&}351NwFij8)4uGj!B1Y*90;wkZZ1!I^L5y z-jLNUR`y_JxtuwJx>YE7q}*tk@a_ih>P`oBICBJN9ix;j%K;*Zcu+|}>nz2ru1gmV zxMjv88N)WlGqFmqdcdX9_ScPTm%B9lwhXUw=a5>QRkb1>(^y2n3cUz^aVum^M&TzG z2KmG#yQ&dQ9ni{F`^~gqR$@RC^l<6nzqEq)w}Kz`oW$Wa%qxN96~^9Um}X`P?NVg& zn)O!f@uV+o9oSg;z0eBGE=;&3K|K>e$!8MGelDO__P?t`+f?t^I!W`6IJ}^zAwdk!Oi)-O*+9xMkvgRlbtc8(5-F$#oGn_q;V6Qvf=`0nksu#)%hckl zkYqs5Bu6|VgkHgbuU9_p%6+63RFD+mh)Wi*v~J6c#5(#@(z?OK@+v{zQ!BCto)XxG zk)E+Wa(G~kizw1&L>*j*GY{2~NC%d+mMv3!VHwv6=r%wjOqn>kqTJc} zlnxe@Zbh{N=z7nSW2C)fuN3X+OA>G?$T{)!+q_CCN3HKHaE)EQ6N6i;f zF@qDy(07h;ty>ya5>AuAZDm;1v$f+ednR@D(rqnMD3_@uC}mbjC6brB%_dEuC9lJo z`)YAk8dhUOnmH+&VZ!;%VUzZKqGNk3o*U;eY_)!sO^WN$Y7*<Jow#*o$0d$BC6(r9@QrIu866U@H=ExtF3011 zlIv`e!#$3M_t|I)tH)}%%&zjq?xZ;8hB?Fpk6B68IcCv|t|(y;>tup_`}yy8;I(c2 z?#VGt0GV5awZk*3+NSz*U*g+J?QHt#60A4tx)GX$I%0)0snBX#P3a%&d=l%H4T4K% zI7v6xEFkSJgsp4%P0`haDP@f(<-Dy8zMS0_%e!KJ8SCP+^^Se{Q~V3!{PPeV5m@5i zX2SlE{kYw|X%oRKjS=*Ci=;1bQo69RDtX!Vs`?hjL1zuC9<_7|;1cUBD!_)r(p5^i z?re0BfX8A(&}+bVU-n*NGr)GSfOU(*WO_e6eg6EEVNC>*HKw7X=c7nV_o+?NVQy zUJ{zh)QV?iyYOk8xJZ8d1u)!W>Zkm3}lU^vI&?^SR+!Q9jTRc zvd3H1jnwYh4KY`UO|B!|J>rWKo{+UFoG=OV?)WtKDKPZvzX&tZvru{Fgsu*ed5S&` z7P(0&Fuf$8H&@qWU=Arb0Bz2pTDhjQofXC>eXX6oR*Ica>PVz#NR0uPOk;#SoX}ua z@ZgL~k&|CTBd1IN$AZK9Ni&Pi08Yte6e20&eUuvznR@_Zqp&_Y%z4q?*h=jxTVY(< zow?(b@Ru~Ox*|n@U|%`skVGZI@~yck(KV(``J5&DIQMv*&u@!N8bM*zlbV@eu-s#A zJ)&!q%C=M0cH3;@vg$=?wje;GR@E-BV5^DCC9^cwNN}kmQdj!)#%7w%rj6_-VOT&~ zr&z50yQENS!N%)FYGBtH%Hm9%V{;8;k!GQ~Q=w$S8j2b%L*j5a59mGAfBgQ3e?q9mM|Sfa0ygai<8SqVEV@UB+AQ}*zuL4F#TM4AeNrtPa)mn$i=gSZwl_0#E+wfw1&ixa*vDzkO9&L%wb9tEGGv?-A)L;B$e zAX0tcAX^7kbvuxSG{Kuh5zr4sz*%(l0nIm(9>Lo>N&t(Skfge&daol=t|oa2HShPH#rQUk;uDEADLTM7Z0 z3DF*H5m4It2sfd+N9sBP%}<8e`k9b{91~kpVwzerja2l(_ADK?Y7^XPN|h4U60vnV zV(X`>*iC*lE;7>|X(2_q@!eTaTdDkKq<8nMhCSV@_m^u0v+^u((N3bRu{z}#wb`HV zN0ZTI{FB|sEV3r{a}B=Eu%H}*uOf8711sB{U3>eR)U{``>Edp-Y~5dBYdZV(NK3Am zt8S}6l4Kp=3L-7pot0ufiF>#v%s$>!*KtJJE=&!h@ODYzq)1NDUc=b!k=BlhU|T|M zoV6}N-3`>Gy;^mEZ(y3ccK8Y{x#%?)SXwC__nv1I&N_FaZpFEXBQAtxR%0P8N@B3$ zsRpl02EK}hIHm&$kLA|Zv=eay8n|~!GwVBs^%%sYkv++3+$HFH9Px_^M`?b=wNJ6j z{xtFioaZ>$ngz@;Dm)>P*l-?FqG3N8`5_TdYrLacj>Wi$!fl^}a6d=9b(YU>XVca= zTLFuBTe%K8xL!M$b*i3!>Oh~va*t!I%N3VrV$79KEcb0#?pqSU)If|B?GXWX!AhOl zC3IJpS!r@*?eP||HrKR-Tm-|NONnGXMKCQj+XzS6M_;&Tt9hT>xazmKcz4pQY~N7W zwud-ajLU}xv2@<)U)8u;&R4~6-+^Kq1+Q_Nn)7n2!SjqPErV7fE#f_SP%0^QiHyo# zA#GBa<%ru<(3o{g>4&NPO{*F%o8?i_Ymcm!pj#mU5J8|wm!?$fmuk4$#y`q1GeIoX ztAp9W*P$KE+-NT4b4m>U#DP9e3l{B>HnGfJwL}^gYOhjWa6fs$YwfIqP1$Isuyhm( z(!vVKRf=Wy71I(8D{yNdw~9F|d`%K=C!I~9tsd&?8@7kWp>~1=9a^f~%BzqvQ@YDc zdD85k%%h4!9YmR5@l2`a#%q_XtF8+3Oee7V&~mXBS{kX7Ce~MK*|tV2>p<}qBBqA5 z#M?OUWXdh;g~Ea9Y~v^6(q>#`qJv#pD8-a)tB`r;o>LE;=O^YHVm;G^)fKxX1lxAE z7Qs9hSITn{^(VO~wkz#SXw95y>rnw!+}RY?JU=Jp$x6Ok#CaDYZ7ow+VJ1ma#9^+8XVGjc9B#n{ z^QtA@(kmB|2c@q>(MWLH9%-pG3%cz-ElR!ZlFR!Hm-ms)3@7gZT<4ikqV4W%15Z^P z-B`@jhTqCE!>gHzWndv7xN<{aoHs^lownmXwClZpyFnD({R4NT4b-lS#_X6K+p1Wd zbgYVP+qP|69e2!%ZQFJ_R!1G3ylbCtopr_@`w#4YP-E1ndhX|*^SUOU1i?jlQqdjM zzE%<@3!@Tl`x6Yj*MPWPNGVwDL?<70Rw_Uj$%o>C46L^GSip9+`fu(3Lo{gDm{uP! zbxUP-qqc>7X-8_Zh38Sg!lUroc;{{HzT+eJ-fWh zVYTD~axA?BXAPz@u4X+75C)heMY!cD8B$hVRoQ@=>;f43Hh;YacidGYzO=%21gLho zA>(kq1_rg9Jpt2Q?Q`LkdZp`>2oLK)rb1DGPL2YeOcmjZV(%x>y1yDuE5GI&VBC%} zR0=6CbYSwWE_tmD$+Ib5_iu*bN~}?8<+YtrIyJMek~WuFx*l7zQy-4eHrw@C(iVD3k~PB&^U zY@s(_ihqOF+KIc&e#P$`Mw_zDa3MP$&iC{0407ajEEZMeo^-H=wb2$G3xy$r*bI%N zr6;#i4O`=I+S7K?N4kKXm@WARSK$3!*)z_sPfxfRXtzsyHbehWzs+GT_&+i; zF_p;fVk7NBj}F#S?QM5tfj#eU>5MVKM2C40t)_kCD%OGDBXJMIP(PLCCnzp*Q+B3R z8ha&?+J}uq|FXLr-B&DospSG1S`J((yb--di>*-FldD8?T#5eZv>l4tV$E+`nk4(c z)kY0bb`lP}+5pBzjPb@HoQv^FxtkrFAz&*OxAR+^qT?Bs4Uj|Psg`^A`)Up%;Gunf zav4m8X0jIG`N)n+qpzo0s3gba*D| zN2^SdA7fkW;C<}^mnL{TI~#9~jVLv0w!mlv^`<|^%aZ_aOS$(#r8iv4He6IUo%yw} zF;{sS5z1#$CV2)0r)h!OI&hh9G`du^1gZY)jycQ1SGekzBsl#VNBNKM>wBM|SXvdF+scx@QdQ2C(Y~&*l=q*GR+iGfVoTH>P%d3W1(WT8j zEX6ARSyFF}6k8wo_-UI2v=;YX;HCxt9-sG)aiaA_novk)@*LDwsy4*a%KqClYGOxt z%!V?pA`f;>^Px-xl$JlIX$j<$i*$+nZA?m}Y@5*(VREjJuUplWCXjjux-Vl-utvB* zNJy4!G^8R7@}k`%XdEn-JzCp}U@I3fMEX77-&9Uz4532nOOvX`nOO9ALY?~+nS4rw zS@A0#><4UI^Py$QnNQNP7wO9AGrm)5*=7K+HS>%Ii_q1^2uywBf(X#7bfs#!<^8Zm zN#?`RznN&qvQcv7@Z}~+&4_r$-+Qn#-o0e|_-O0oo{?*#bv;78<(khMG@$_Op&P4& zyDY0;+P))}n;Qn3KuhyPM1NdadD9PeOJq2zG z2Fg$0?#zdtXVdr>hV(2WDy}m6o3DPJ>RH7iGpU|FDVEeW`uOo>&j%b=c)f52-JT6g zfw5Dkyh3HrckxP;r0&n%yxJt_;ds%0HL*0Ix`1^c?MQ+J*1S|&hfu5N zN|8lx+TpoM8FYCfkWFYUH}{*l6D|4=`rN3Skqb6oCQErNe=2B)z8LXwZ-!wTqyUVp z8N4DBBfn>tRyJ^_OU7moZ-z*X=bxuk*C5#@>wGvdCX$*2UYm0vdl3Z45Hh&7#c@i1 zX?@u_bWa@UW{DO#KLC6 zE@1U89<=#&A?QBWp4a@Vgr^oW?ER-%3|pmI#3*%RMu=P`?J`~`%AdD+)=jj{=?@H0 z#i^#VB(Lb&-RCWqSXIiro^!nP+YD(hMZ~XgCX#)yf(dLN zrkRzLbS1Q%63WEarKL3wjwwqP>tf!Oez|BdPzgoqR`wzPsSISMcEkzR_yp+ZUj9Ni zdR6@n+Tgoss@IB+W9|hso&7tGw^v^O{4O=o44!@V`booED@wS#s;0+s@uzy~X}ml& z0lU*0npHe(%p^IQ-pziKRdnI+Q3e^tMaU|Sy=91y9Cj?f#9||BU(v#(uIj0 z$T=u8i?CiP@jnA~*^W?4fGn{`q%={{z>t|xxu+s(GKB?HFtY+^>`Y{c`XY{@IJZFj zT!vG~(!Y^h&DB3!h4#caj);`jSoHhw*=It#9+Uu^)=|7rtd z=2CT!qPNHbi%{^0&_~#FEGaZK6j>#OWa?-EhOkf9?yE@cjwyG6Y1XsaQ9E(|f8S~HWsOd5#u&Wmg<36r zR@EAmBL4ywkrz3Dqze=kV$~@!O!Jo~{#xX&kL_K|103Ii*@D;Vjk^2NQ4++ApeMX(K{a|D}gl z);KUWNwqc7;vbzY9qlOY5(rTr+DG4x8^hfQaLvI4 z8jeYXfz_|Pb>?dMn1F{@ll!oPe;&bGn2WKv@@&Q64zp*$hhjd9<|8zFD@P>q_^Unq zn4v37>>{lF(ikoZ(No_V(wm`SO!BhyCW1E6@xn&Lxj9b{w^9w%*9~IbZZm)pqAXYk zeFnYsNmSoYkbWU< z?MH`vbe(=|2HSEPE~zV?q*vVR5Z@;%eTGI1K@7+>8VbHew$O*aEh%PdI=HEwh1EgH znAG2qY-3HH5hqw`;o)t*<{r|nBPlZ3tku5pxZOI$izB+YPrGlND5exkNneTEAOcYB+=5P!O8 zp}b)HZ?8%{`$cgJyessrGDSS1q>TmXy`+ssnY0aw5%c6J&BlO`hNi*jidgEG-|w1oich3kVa}PboQH8fCfNmOn-uQ4b3T{APsJ`lkOYto@3R!5h|s zs`@}br?H`t#rP6_jy)N-{Na!eR(nu4!>F>Mnjo;I#fPw%&8QKNQqazHxFS{Zv?47t zsKFI(!b0SMXm;n5PEdl&H|c<#?v(KDOiYnU=(loZ^x$W*Z6Zb0;JXL%A_$Tn@jHPj zh@@$;6c;W7j_Tip5GdnOhfD6NwlSNe9DiuZ8hLBtt!t@tNc3U4x7%ihCu{vaG(MHn zrstbyKr)HrnfI|liv1$kPOqo_?AqTmHsKD zAvh@)I#5UUo1UieA0VjQQO%)(>5Vno+S$TEFYD1}2a#@hjmOW!W?LMO128JopDpsN zfy=I4MbOKTWlDvb?uK$#+?2LLV$#Ysgy{zC1x-2+ba*zH!y6Q{gkoG=Xcwn8>lRya zssx4W#CCnH!u%He3PG#u2#E%E&}{pnMS(VnQ<@%B2`P$lXR^aHNS4%PnyPAs#+uki z0$0=~ZHE%x2yCk^LFiAYC08j@X&Y3<3J489c~j>tFdS63?nQ%Rfj4OZX0|T<4-7iQ zpz|ds&aw27*nE_9``n}ZjkT10VZbCAR)zua-0CB4b$Sw=pQd!D!OF5^_MfxixQ)KQ zW^fx0gNnO9>?C0cbyOOa7)QrBlhhHffX%aa)JQw;{Z1Obr2%4_zd(8DOzc^!6n!P= zsR+;dGGWkPhMebk{vX9~E>-57)lHMpvdT!U5r(Epf|RHgWeKz4hHgny(Y|)I0OOp4Dx9K5^Ww=ZClJN?7=mlNB95Z%eIx-rtTog8nW)r=WDBnN`YrmmYecjLy1di4tqBkfa`Tbwm35$g!u z$j+0%l$s41v^vn`sQ370$BjT*|F@j8+Z=YSIr+E6vXsiuBX$HCvNl9}41Ew0lkH9Z zT$nF^pw|^;M=n@1;0i`{snx#4U#1MCLhG~@Ub|>^u*6=rG@Dj@EgBURDT|yim7|h# z#NckpcD=eN5UqlE!=!qiLsDUup~VFE17JkjB3G>S&OOk`|0rh}->3*InBWSG)6vY^ zq{}TJ5c_U-`@du5rDEI%+f_Xjn-tAyLQdIwr_xg)hk!ZbYQ>(Gt>!MwLGzJ@QcxX* z11Q0={wf?iQIqZ-`KN6{bFUH~HdH%6*{U%oxxBQfqk&I`p#zd|9iBpS)^fHlQ8g=> zY+<@|{2AH$lFSx+O~P!PgKGpVxY~wgPZhIz{rqH?(sO(5*VHTc%FkN%1(=N@=<>`S z%t)&;*^XJGOno`E=>4gE()^tsG?R2ECf!G$G(dqy<4Sw9fq4yVg4LjzsS;*q zojw=&F2`#wl60@yIMh)} zvAfm1JXFfOZ;Xl|jN>NA?y?S%p2ZGJH~r*q1>HzzLb(g}$w*B98%k`3F5<0c)DY(-6C zYm&Wa_(98v+~^K8;7`R2JnlbdtF9T8l98gZ?s1SIK2X*OkU`rt(WBP7-Ht#q<}={- z4HoDH=#*6ITBxyBqsHC-976-?ZJ0Q^%R5V2UFkyB)n%owBjv6Z{TTa=EKN~xRkD$@ zDf&BfGQ=Xj1v8GcX`pYBc~98V_`&&%^kvg)Mp2s>E-Vp0NQYMu$#pC_3K}Gx3#a_I zXy%VZWe%oZ+NQFm#VFAgWaw?ehG^N+Quifqz~W#x^?%z1gs3r%YE%sa)0|ghG;S@p zCjc?Sce-rMm7*mIPKF{wrf>7hu%48=~MRFS}*1j?K-Ce3i z19oNqh|r|)vgNb4ss09fA~9aU@ha_*ViTB}v9TuhvcHmp>cmeU{# zQ54@j<&kq9I*%3G%jU1&qGTFOr*LabiP5H^k*s5b#<|NG@di$^;{t*5>BR88xbuNeMFqwW#bzJ_e#-5M}tKb)r=UrVK z53Yon&bkx_8L5~^dt_L^`w%3wnkam_!H$u1=G=o;CIn?pwu$c3!kLvivM{(7fSKi% zV>GEcv=o)BT~uj{vG$TZe>&@SeS9@f&S-K0-L;7_r-t^bF z8z>O)Aj6Mw<> zDMF9-W?O+_73K0EOF*7&;*#!S`Y|HuQKQz@&L?)7JHm6KHgu6mKkm~vjHG?N#IcU~ z5}#<>wE9t~l7~95%Wt$;w`pru>j$guS&L1q6nLe6zt0-~8vqX~6L(QzUrz2;*TQTn zn`k*)D#7O3A>EM6#s-0e+RhE>B7RlWa|Zsjv>S4^=)aY!-T9vvKmbHq?8pm6&l~Lr zbf-U2zcV=B+XirzxPiDV`pAbvN2zXw=9_9fVM3iuWHe1b@&Z zsUYd(!dg@2PedwIcPk)8HB8q1Pc-xMWUpoEoc~`xV86nfedrKfwCBsYZa254+P!)T zPP3gytEzik&OpDUBM7YnD58PMz~+s7M84U%uETEvb!0^~4~_HxU|Yfm83`)|JTfU> zyQ(*Ip@hvXL?daz6?04mjQeV(ZJlb|wX%-o&Ij_D#=o!-qo|kRQ4&%${rRTSyq3~X zYJHqL0Jov?Q@C&(J;kX=raMCb+tsqfV{+)Z$^$oB!Om-_Ds8mk3c9e)Qo2$GD@U1j zs5S(nPxR5%!5fIA8-Ew=gzL5M3H&}*}6is?+XJ->fpGFPpQmau?6n4_dLW2PW2p7t}(*EHpnbznOoAkiQQG2>~3Y@ z3-p;W{p874hq!5$L=l~d5px=crY^>lxoVXgRueRHU8pIYX2YuREgeoJU!qn5(w<7i z$>dw`yp|qOSnhH-cu!C)!z-fmmOziaUJ8G-)tlx|QSK37VAF3d!}PTemgvHBRw0SE zo7g_+3jBnR;T{MQb($*<$R*rERt!{mQMUp;Z(v`2%+F1i&AKI0L0ATxk4Ms!t?w#I zf8^9zwp?|S3U zXXq!GX)GI8^sk~a^_2@e4n?JP9+K8tio|>Odio94HQAn$xERf8h9AfX&K$VtK!?o^ z!5OOBFJmd9G|^sl$I5O|pJjvLtb%Hu=@_KaM(ZW^Na4`4R52CIBqgv>iOoG31Ra>* z1z$B!-rPnm%45)iV+pdG`(9pn{(X)frWBC$Mr@La{EEp#T;ghKyF5mXG2+Fvom93p z#MM=Y_uS2cDEUf{=ro4bN9EBS`p8M4-Tr#DFWbSe_>(YG4QY928Xb-tifemf(X>E* zLuD8J5bpvI23EF^=i0Pz|IP;%r3q$^+>c3!|VG~imrC_;*SIJB%Yyf z`r#Q9!>sCC;sh>$AakIXeX*2@+xOTz05ng%W$-(VE zutP0!7*NX5G(s8N1`Q>lfnCY;t?%#c#h+=eb@OTiRfd%65uT>-aq@;aUTUHwGt`K! z;2jl9lQ(CRke3-KFr*?U6i<;3OyFCvvr}jGE4eHcPv^hBi~BIsKD4B1C%c{J6}OX0 zarRDOp;x-p_p>%(*cIsPlFpetE%e1{B1{JHL{eZ|YV2d}L#T(isgcpptB|Q&_JqE9 zGL~L~-)=kFImC#QrM3K?kxsxn^HN4p)M@@tJlt2un#`?$&)+I2nEdrWNA zF?WH-*XkFxbTDh2p)5_)!#rL39hw)SH;ZGCUwqJy?L;N0V}N${A@h-vM}BeFOq?3h zX}z_AI`}v0_{^1%B$V=4sZ#{q^%d_pS%iMT8jhhK%dc{HD8oxy72$HrlK4Sy_w**%P$O%w7iBrzbIC+%`5(c6d$F_iKxHj92>@plSz?btukG6 zku_!JANS(@HggaHu)oMRnkv-?OIY92uYHjzA7HUsIlZ~X5N2`p3kczUG-%-qS*9D# z!))0i+0lP}FFIjX7?b$Jy1gVVpz1!*Vp17WyG&xK_e9fhC0L3%QL?2HT{-GFMZgB! zgAgpekudj9dKJ(g7_i!`$gwmGX*3h8>mTNSoe?zC7j4GaYCJ3kJ4=i)5IbWgBvs&x z4&j5bY?HRPzP~Or*~rd(s25B&J73U7z8`7v4ni@PJB=}~cb_7W%4P(08e{L(r&r|g z`_QcmS!iJgIga71%RJwaNbC{ldmN6X;GzWFN6_meN+!rmF(!vMwT1s}ry;@?bM$ZX zPV8FhAizTCN03!WsOSILv^)ZWw{#RldU*odm;u|+1eep@o{ChQ#v9BvS+8)57@^p2 z%6=*=%NH@z4{pV2O_NJ;WHz0j->^6}O=&y8daIGQoe6OZso3q-+5wey`?H_!E#Hd>O)0t*lMl~~G%@@=v^79L#|{zq9dc z1bz0zsq>w+++x+5vc5_92ncc$|3ginjGClf#Vk()Z8X$5vR=qif(oqzBR|0=_2mK? z^iI0%N;{a==MJ(#jVbEb$y(-&GfxL& zn)}cjE9;@@fuq0}vwxI7TkiY+)$Q9-U7)PRFAO<#<0ZArvzAx#y>Dmt&Z+LICd_iB|3&Ds7Z>;?iL+neL8at22s|JO zoN{jLE@9sM?^TYK!kR`YWG)`OZS$gP4|DYwBb-Y+$nL=0pDiN#9_7ACx0fe9J3=dZ zqyG~T(N*-SWA8`ZblZ}grD-NA2lpLnfcBpqxD43$>Nq&Tdmbvl^jfCvIP9WIM@K-$ z4+flCrsh$(`?0GwyN?&n{Ld7Bn&VG_8Db(g_r}-M7Y`6VVfOxj*q!@}MP?u+cXu9+ za_jbrzHa<>zt}yrjtQ@}IHJAcH@o4X*N1!?ZhVxQyS?{IpXMyRkz2*^(vK>SY@87frp`U+uUY*(N<6`nIK8yLi7x>7(`^iE_yVu8mR$p&Bwx%|kcJq44sx3zrw&+|PhxuerpIA@tTN*J@lsZl}vqAf$ zgr{3NGm#leWw<)dMx&h|^S0pO!Z`$T_t;3Z(W_++oI}abW6EclhiAPvRkV@LxAfp4 z69;eD%CRrJUf=BQ9pcIayo{G1t=jzhw08JK6+i?$&TM`DH$dVqsMYy)#u~>*faL7Q z8}e|qe{SE5xNo-@*|(vLavjxjuj#`b-O>{LlzfZX-J#9&_0jaYy`SQ`lx;YBx>D-M zbo(Qs`|yI)afJ21T_GqCrDd-GhMqRoUe$4YdSR6YK5>Gx=R62PZ4hPAvSZEfx60P8awabXqmnH$AYrTl{Ljy|jdL!aFOMrR>LMCBFxmcYf>w#uL=ER_ z9Y5;MJ?s4BZiNI7Cpy4?@i;;n!kmAyLgwoaGb48+-)29*AGF*WUgV!2nSJykCPbJs zW0U2iML#h3xZZlX>BjJ(VxhICrjD$AZt<$S-prs>4*g_H57|}64w|Bb-I%It^0%Sh zj%<5mf!!)#_ir{|CuHg9BE)fQs3QK-3Ml6_XFvCqHYYa?ZTYZwg zxse;2;(PGx^t0x&pE|BSzGciY%8=hVj_%VsWd8C8%vZ7d2x<#e$IRNo4j`HNc98cCpp^XeSL4CQqGEZG-wC)G8do^@GO&If0VBMyy>hx^67`x`Dne9)N&Q z6E+N(o7cO(LJedyuHf+RG!GMK7P%-SFcaXDU@Q<4ENg4KJBYYmf*JyPglV(q5N43T zFr%9*JN8a`LKZL%r#I5LNC_J|)@_1dp6-wDI{{=a4l6~cUVmJ>Cl4-r3!4K#_Gs7V z$bMrTFBWYvjf51%0fI%Oo5x0+dezKNM&6Jw53X-pw>ye&5W_J=TKF^QAjgccU)3YT zUGHcZ3NtY3Hyb}C;@n>oW`qEY`Yduw4@LZ~tiM}5J9|H#L^YyX{b~e>0{-!Z!18C{ zeAYu~+ulCU>elQIMFyAZzdiP3**rB|FW2*A2Q- zPX`Yt-u(Gs*aG~8?k+b9F0mP5bPX< z&Y%Qh3o7-DUrpaN-~-IS{(&`M~`PN6!1_ z*vgD2TiKPNx#njScjo;iLlpJ;NB>YC0aN7uy6kO=)lMeQKH`jIO3aF&qdCwtdTB}DkUCtX+xyM! z=5l%QpXV!Mjzl#v90NL6(ZgG)ersNSHrm=g_%RqT@gWDya#gp|16;(;9{efdpaUWx zjF>l&r$b9Vw?k7UV7Ha9^xo%kT}Lo#h(Ruo`{L9TcXOTK_z|L;z&~Kf&|6>eO1%!4 z;&*0X&h@e4|It%uS?D;=fzMG+%q-dxww{Js?|J2P?hhJ1sXz<@2G80_Zg_n^{TIzG} z$_%z2{iHNE@73nU34}5rh}D~wEdFzI&{@7YTFKKGuWG;exo1!ir{|4^+29PDi(H}a z7{aZ)HKqTh?2Gw@WxTkX`qw*=zvBKJ?h$;^u@wI)pxyl*`XBR)f{z0H`qy6>l0{a& zfmkl186SGP0pednzh3F<;Z)}LD4!<}9-YkvJE{W1-3>N(K4+9KxB&rZ?ny<=kKu$R z$o}`?#K=Dwqi@e1Pk%%IrU@^qK~#8`KtZc3Qnov~#b33+i5}BpKj^;+H^N&9wdj7U z(~m=*Y44fhDmwq?-8yDC?)#eLe22x(48j%QUrq%xrj1&Jc`?syY z4;axK+io8x9FmAS!^!w7_nu869Gv=Sn)gOeT#Fw#S9ySVu8xjvJNk2S)(% zGHmvRRvF*#dVz~5)t3N$|GI)rB{5CQA`!ezO=y4BB9ScpX->hxLh&=DekfUbVy{1 z3@jP(WD~Sw7gp|`wyt4*UHe|Ui<~e*R_J2v@JD89d!NGEVKSfac{`qE?$d`-lm?V~ zI)8{a?A&x7NLt2XA@(7c>AD`*H#~^>OFNo-Q$nE4`;$poWBFkCg1!qX{XL6NZL6qR zMnSL!6=Yz8LJcGTcIwYN%ahBwO5zjl84jRG3T4bc5UNhL>5$AY!atBt+$Va5lOtjO zu3>s_B$>fZG#8^iLSjkxy0oSKVu4judK}vXI5!*uC?lL6uBr%?SDkeAS81#0K(>F1 zbc&Td1gYYcscmPan%1sXf_Ob!F`0 zT^7;OpbgPlt^1nA5pvmY{n&4lNo%z_U3>A+HUgqrsFpm&6TV&MeY-51yX9>qoCiw5 z1*jC#@dkP8Y_+uaQpXqMbNwovT%L>Fro`dzy|0 z<&cVQ$$hn-JS3`tuBD*O93)RTvhmtKM*+k%C(L+S`p0oGN@c*;K=zl6bKW}r+vzm9 zHWCjM*&;TeBnJYY+kloPL|4TL;P2#O{-gxk}`$ zJ}f1IJot(s+rY?y##UK{&#Dmec%J%*FUivgt`_*Bv^bjvn1291n%6|2poxUEJED|3 zCnO!hNrJs0WU?f4G8#UFqI8ZKy6xM@vee}kgI+ZEZ{_a_B+JvyF4+h#RW~2M&!3;b z)@Q)98o|_6Y266tmsQHL#=sh4ZC0~(g_M=F5}h#Q+U287x9J+|3fy1}qOB(Axv2=7 zb@>q7Lk@Y%4}O#%AeD0)o1&`YZzqY%nxpGkmXOnPP21u_p5Cn`^ zfu&+LO)E{FwY%&EXq)jT0c?!KRArj&Fp2!{%!_(u+_(ge_3u>5U8B(csw zds3{FCX+^ZLNs<;(Uu5wf(EAT1SA@vgI<7;eO&rWjJXp!fTNFff8WLaQr6? z!ZzjSm}r7r8^usAG~l-sRo2*9HMg6nqXjvX5Q1tX#jtPmtz}^p?*(+rDxRG}y;>64 zvh*s&@>axyVvje0-t0mGNtihmO;w@|$x()6?tCO~vTc>KH{mkOtt;I`2sR^PWgD^* z)xnF;L0gzWA+iBw!t8xJ2W1j~ibhv1C|#%Q3a4v~4b37FQw4Gwn{mqlQ%J3kTIfnL zA->Jj>Q+^f^_O`PIT4z0twHY!eZJd~fD$5|WR$}oC8CB_i>bH~;XAG6mVuFHD#>*@ zmL{Y;yv=~*j0Q0ef8DSc<|*g)Zxcn3C4y`ra?LSloizwYvxDM{pbutE+-v&4Z-BcrMpHw*a8qE>5<11g#oaRU&Xq+HuVA<*u zZf_zKpAaNN(09XQc8bVe$_91JHTX{UMTtyi4p^Ojou`Nvvw}d(TO8gPoNz?g%O}1w zye*3>#6{AYDeq-S5=ZY5KmATY>$7J&-^WW|QZ!$*vR9LEb=gppqXezADt#W}9Sws6 z3)56|j98x07isc`Z=`jZ5s4nYOlr58wd!nb-7(`wa=X`Qa~f^RgbI2JYh^L|*u<7u za_MZ+%HE_eh?^d%5N|G_H7=FS2P^q>z7Y^k*)^sysgC~ zz`D-T*DoRRnoeEks)Eon#xQGlIrHV(KX@5w5I4-S2Nt zd^hXOeou!PQ~{HW{p)*~s)un&qT?Hat zCt&(0*>Z}rt+5l_aQc@y-b(*i^`|rpCx6L2Vf6opXK}q<^BU>?Z_erX;RzQH$8h`q z~O;Q#)=v#%u9HJ4Cu-V;Kw>p3;?UI+_Tz zC{aWWx*>X{66P^dy?6iz)+|NU57I0dxXF-6*A%3^sY*l|hG$mR@YeQO|A{3cr#QY# zO)kA{fs1I{jmx|K7_F~iH6 zS`9Te=z6rBf6^*fV}9^|4QQjDVqsH5IvZ_p8fj@PU>b&%)}SV-tI*14;})fP*AA-B z#I2q#!ZPRwsa6uNh;E1AoNF>$>f^h3U%YX0VT7FG`*#^1BX{PC>3H6^-RR^z_=j zot2l@3(y&u#q6T5>xoQO1C?p{!h`v>$6EgaS@NuMq@wMiQMn!8RAwr6V4BiHQH)DD zxDiTP&IEe+J%Z$l4(I zEA|p;c*P zmNjynV%9Ha{zp}0jID|B+6JRk8&CyuKODkLHTEyP7 zzh1U#>W}9MIqghfGX-&si`<4Lrn`2v%al}YRbgy|vs5PH_n|I=* zrP0zlr#e8avswq1P^Xq~l2>&`(Dm1hVcQ5``6z0F4RAb8o%YJf@WdEtU_~Mx^%w!x zVNfD>iQZm0QN`ZY@aPe(k!`Id%oMaYT_W0$N0yP6Wg9|%S;VZvmrvjFnlButigNzx7JFB@uYIU<@#rRY|u)p?38EQ(ypg?TAC1jBKB<_|JM