diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py index 3b4c2e52..a4a436bf 100644 --- a/py/src/braintrust/framework.py +++ b/py/src/braintrust/framework.py @@ -1710,7 +1710,7 @@ async def run_evaluator_task(datum, trial_index=0): exc_info = None scores = {} classifications = {} - tags = datum.tags + tags = list(datum.tags) if datum.tags is not None else None event_dataset = ( experiment.dataset if experiment else evaluator.data if isinstance(evaluator.data, Dataset) else None diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py index c3284e53..8f8ce574 100644 --- a/py/src/braintrust/test_framework.py +++ b/py/src/braintrust/test_framework.py @@ -1444,6 +1444,74 @@ def task_with_hooks(input, hooks): assert root_span[0].get("tags") == expected_tags +@pytest.mark.parametrize("as_dict", [False, True], ids=["dataclass", "dict"]) +@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-tags", "trials"]) +@pytest.mark.parametrize("initial_tags", [None, [], ["seed-z", "seed-a"]], ids=["none", "empty", "nonempty"]) +def test_hooks_tags_mutation_is_isolated_per_evaluation(with_memory_logger, as_dict, use_trials, initial_tags): + initial_tags = None if initial_tags is None else list(initial_tags) + original_tags = None if initial_tags is None else list(initial_tags) + base_tags = list(initial_tags or []) + inputs = ["row"] if use_trials else ["first", "second"] + data = [ + {"input": value, "tags": initial_tags} if as_dict else EvalCase(input=value, tags=initial_tags) + for value in inputs + ] + + def task(input, hooks): + own_tag = f"{input}:{hooks.trial_index}" + hooks.tags.append(own_tag) + return own_tag + + def tag_isolation(output, tags): + return float(tags == [*base_tags, output]) + + result = Eval( + "isolated-hook-tags", + data=data, + task=task, + scores=[tag_isolation], + trial_count=2 if use_trials else 1, + max_concurrency=1, + no_send_logs=True, + enable_cache=False, + ) + + assert [row.error for row in result.results] == [None, None] + assert [row.scores["tag_isolation"] for row in result.results] == [1.0, 1.0] + assert result.summary.scores["tag_isolation"].score == 1.0 + assert [row.tags for row in result.results] == [[*base_tags, row.output] for row in result.results] + assert initial_tags == original_tags + assert with_memory_logger.pop() == [] + + +@pytest.mark.parametrize("initial_tags", [None, [], ["seed-z", "seed-a"]], ids=["none", "empty", "nonempty"]) +def test_hooks_tags_clear_is_local_to_each_trial(with_memory_logger, initial_tags): + initial_tags = None if initial_tags is None else list(initial_tags) + original_tags = None if initial_tags is None else list(initial_tags) + + def task(input, hooks): + before = list(hooks.tags) + hooks.tags.clear() + return before + + result = Eval( + "cleared-hook-tags", + data=[EvalCase(input="row", tags=initial_tags)], + task=task, + scores=[], + trial_count=2, + max_concurrency=1, + no_send_logs=True, + enable_cache=False, + ) + + assert [row.error for row in result.results] == [None, None] + assert [row.output for row in result.results] == [list(original_tags or [])] * 2 + assert [row.tags for row in result.results] == [None, None] + assert initial_tags == original_tags + assert with_memory_logger.pop() == [] + + @pytest.mark.asyncio @pytest.mark.parametrize( ("tags", "expected_tags"),