From 4d1d13b6d528433bf4523e85acb8b23a4108177c Mon Sep 17 00:00:00 2001
From: "Gareth.Y" <91742523+linjiaye929@users.noreply.github.com>
Date: Wed, 14 Oct 2026 10:49:15 -0700
Subject: [PATCH 01/11] fix(eval): isolate mutable tags for each evaluation

---
 py/src/braintrust/framework.py      |  2 +-
 py/src/braintrust/test_framework.py | 68 +++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+), 1 deletion(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index 3b4c2e52f..a4a436bf1 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1710,7 +1710,7 @@ async def run_evaluator_task(datum, trial_index=0):
         exc_info = None
         scores = {}
         classifications = {}
-        tags = datum.tags
+        tags = list(datum.tags) if datum.tags is not None else None
 
         event_dataset = (
             experiment.dataset if experiment else evaluator.data if isinstance(evaluator.data, Dataset) else None
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index c3284e537..8f8ce574a 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1444,6 +1444,74 @@ def task_with_hooks(input, hooks):
     assert root_span[0].get("tags") == expected_tags
 
 
+@pytest.mark.parametrize("as_dict", [False, True], ids=["dataclass", "dict"])
+@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-tags", "trials"])
+@pytest.mark.parametrize("initial_tags", [None, [], ["seed-z", "seed-a"]], ids=["none", "empty", "nonempty"])
+def test_hooks_tags_mutation_is_isolated_per_evaluation(with_memory_logger, as_dict, use_trials, initial_tags):
+    initial_tags = None if initial_tags is None else list(initial_tags)
+    original_tags = None if initial_tags is None else list(initial_tags)
+    base_tags = list(initial_tags or [])
+    inputs = ["row"] if use_trials else ["first", "second"]
+    data = [
+        {"input": value, "tags": initial_tags} if as_dict else EvalCase(input=value, tags=initial_tags)
+        for value in inputs
+    ]
+
+    def task(input, hooks):
+        own_tag = f"{input}:{hooks.trial_index}"
+        hooks.tags.append(own_tag)
+        return own_tag
+
+    def tag_isolation(output, tags):
+        return float(tags == [*base_tags, output])
+
+    result = Eval(
+        "isolated-hook-tags",
+        data=data,
+        task=task,
+        scores=[tag_isolation],
+        trial_count=2 if use_trials else 1,
+        max_concurrency=1,
+        no_send_logs=True,
+        enable_cache=False,
+    )
+
+    assert [row.error for row in result.results] == [None, None]
+    assert [row.scores["tag_isolation"] for row in result.results] == [1.0, 1.0]
+    assert result.summary.scores["tag_isolation"].score == 1.0
+    assert [row.tags for row in result.results] == [[*base_tags, row.output] for row in result.results]
+    assert initial_tags == original_tags
+    assert with_memory_logger.pop() == []
+
+
+@pytest.mark.parametrize("initial_tags", [None, [], ["seed-z", "seed-a"]], ids=["none", "empty", "nonempty"])
+def test_hooks_tags_clear_is_local_to_each_trial(with_memory_logger, initial_tags):
+    initial_tags = None if initial_tags is None else list(initial_tags)
+    original_tags = None if initial_tags is None else list(initial_tags)
+
+    def task(input, hooks):
+        before = list(hooks.tags)
+        hooks.tags.clear()
+        return before
+
+    result = Eval(
+        "cleared-hook-tags",
+        data=[EvalCase(input="row", tags=initial_tags)],
+        task=task,
+        scores=[],
+        trial_count=2,
+        max_concurrency=1,
+        no_send_logs=True,
+        enable_cache=False,
+    )
+
+    assert [row.error for row in result.results] == [None, None]
+    assert [row.output for row in result.results] == [list(original_tags or [])] * 2
+    assert [row.tags for row in result.results] == [None, None]
+    assert initial_tags == original_tags
+    assert with_memory_logger.pop() == []
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     ("tags", "expected_tags"),

From a6266d6f6e7c2ee524e51b7ad291f1e622a6b42d Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:14:43 +0000
Subject: [PATCH 02/11] fix(eval): keep rejecting non-sequence EvalCase tags

Copying tags with list() turned a string into per-character tags and a
dict into its keys, so invalid tags that used to fail validation were
logged without an error. Only copy list, tuple, and set values and pass
anything else through so validate_tags still rejects it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      |  3 ++-
 py/src/braintrust/test_framework.py | 23 +++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index a4a436bf1..8efd03172 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1710,7 +1710,8 @@ async def run_evaluator_task(datum, trial_index=0):
         exc_info = None
         scores = {}
         classifications = {}
-        tags = list(datum.tags) if datum.tags is not None else None
+        # Copy so hook mutations stay local to this trial. Leave other values as-is so validation rejects them.
+        tags = list(datum.tags) if isinstance(datum.tags, (list, tuple, set)) else datum.tags
 
         event_dataset = (
             experiment.dataset if experiment else evaluator.data if isinstance(evaluator.data, Dataset) else None
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 8f8ce574a..68f066314 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1512,6 +1512,29 @@ def task(input, hooks):
     assert with_memory_logger.pop() == []
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("tags", ["prod", "foo", {"prod": True}], ids=["str", "str-repeated-chars", "dict"])
+async def test_non_sequence_eval_case_tags_are_rejected(with_memory_logger, with_simulate_login, tags):
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-evaluator",
+        data=[EvalCase(input="row", tags=tags)],
+        task=lambda input: input,
+        scores=[],
+        experiment_name=None,
+        metadata=None,
+        summarize_scores=False,
+    )
+
+    with pytest.raises(ValueError, match="tags must be a list, set, or tuple of strings"):
+        await run_evaluator(
+            experiment=init_test_exp("test-evaluator", "test-project"),
+            evaluator=evaluator,
+            position=None,
+            filters=[],
+        )
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     ("tags", "expected_tags"),

From 38edf806119798e2e923ddbd8583410cef7dd587 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:15:23 +0000
Subject: [PATCH 03/11] fix(eval): copy tags in the DictEvalHooks constructor

The constructor stored the caller's tags list as-is, so any caller that
built DictEvalHooks directly still had hooks.tags.append() modify its own
list. The tags setter already copies. Copy list, tuple, and set values in
the constructor too, and drop the per-call-site copy in run_evaluator.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      | 13 ++++++++-----
 py/src/braintrust/test_framework.py | 12 ++++++++++++
 2 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index 8efd03172..f01b3dfba 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1236,10 +1236,14 @@ def __init__(
             self.update({"expected": expected})
         self.update({"trial_index": trial_index})
         self._span = None
-        if tags is not None:
-            self.update({"tags": tags})
-        else:
+        if tags is None:
             self.update({"tags": []})
+        elif isinstance(tags, (list, tuple, set)):
+            # Copy so hook mutations don't leak into the caller's tags.
+            self.update({"tags": list(tags)})
+        else:
+            # Leave other values as-is so span tag validation rejects them.
+            self.update({"tags": tags})
 
         self._report_progress = report_progress
         self._parameters = parameters
@@ -1710,8 +1714,7 @@ async def run_evaluator_task(datum, trial_index=0):
         exc_info = None
         scores = {}
         classifications = {}
-        # Copy so hook mutations stay local to this trial. Leave other values as-is so validation rejects them.
-        tags = list(datum.tags) if isinstance(datum.tags, (list, tuple, set)) else datum.tags
+        tags = datum.tags
 
         event_dataset = (
             experiment.dataset if experiment else evaluator.data if isinstance(evaluator.data, Dataset) else None
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 68f066314..7141da7ef 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -11,6 +11,7 @@
 from braintrust.util import LazyValue
 
 from .framework import (
+    DictEvalHooks,
     Eval,
     EvalAsync,
     EvalCase,
@@ -1512,6 +1513,17 @@ def task(input, hooks):
     assert with_memory_logger.pop() == []
 
 
+@pytest.mark.parametrize("tags", [["a"], ("a",), {"a"}], ids=["list", "tuple", "set"])
+def test_dict_eval_hooks_copies_tags(tags):
+    original = type(tags)(tags)
+    hooks = DictEvalHooks(tags=tags)
+
+    hooks.tags.append("b")
+
+    assert tags == original
+    assert hooks.tags == ["a", "b"]
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("tags", ["prod", "foo", {"prod": True}], ids=["str", "str-repeated-chars", "dict"])
 async def test_non_sequence_eval_case_tags_are_rejected(with_memory_logger, with_simulate_login, tags):

From 4aaaddcae986f6ef5573807b844d56e307b034f1 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:15:55 +0000
Subject: [PATCH 04/11] fix(eval): copy nested metadata for each evaluation

Eval metadata was only copied one level deep, so a task that changed a
nested dict or list in hooks.metadata affected later trials, other rows
sharing the same metadata, and the caller's dataset. Recursively copy
plain dicts and lists per evaluation. Other objects stay shared so the
copy stays cheap and can't fail on values that don't support copying.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      | 26 +++++++++++++++-
 py/src/braintrust/test_framework.py | 48 +++++++++++++++++++++++++++++
 2 files changed, 73 insertions(+), 1 deletion(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index f01b3dfba..933e18b5b 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1219,6 +1219,30 @@ def evaluate_filter(object, filter: Filter):
     return filter.pattern.match(serialize_json_with_plain_string(key)) is not None
 
 
+def _copy_containers(value: Any, memo: dict[int, Any] | None = None) -> Any:
+    """Recursively copy plain dicts and lists so one evaluation's mutations don't leak into another.
+
+    Other objects are shared rather than deep-copied, since they may be expensive or impossible to copy.
+    """
+    value_type = type(value)
+    if value_type is not dict and value_type is not list:
+        return value
+    if memo is None:
+        memo = {}
+    if id(value) in memo:
+        return memo[id(value)]
+    if value_type is dict:
+        copied_dict: dict[Any, Any] = {}
+        memo[id(value)] = copied_dict
+        for k, v in value.items():
+            copied_dict[k] = _copy_containers(v, memo)
+        return copied_dict
+    copied_list: list[Any] = []
+    memo[id(value)] = copied_list
+    copied_list.extend(_copy_containers(v, memo) for v in value)
+    return copied_list
+
+
 class DictEvalHooks(dict[str, Any]):
     def __init__(
         self,
@@ -1708,7 +1732,7 @@ async def run_evaluator_task(datum, trial_index=0):
         if isinstance(datum, dict):
             datum = EvalCase.from_dict(datum)
 
-        metadata = {**(datum.metadata or {})}
+        metadata = _copy_containers({**(datum.metadata or {})})
         output = None
         error = None
         exc_info = None
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 7141da7ef..e99f9b185 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -20,6 +20,7 @@
     Evaluator,
     Filter,
     _call_user_fn_args,
+    _copy_containers,
     await_or_run,
     evaluate_filter,
     parse_filters,
@@ -1513,6 +1514,53 @@ def task(input, hooks):
     assert with_memory_logger.pop() == []
 
 
+def test_copy_containers_copies_dicts_and_lists_only():
+    leaf = object()
+    shared = [1]
+    value = {"nested": {"items": [shared, shared]}, "leaf": leaf, "tuple": (shared,)}
+    value["self"] = value
+
+    copied = _copy_containers(value)
+
+    assert copied is not value
+    assert copied["nested"] is not value["nested"]
+    assert copied["nested"]["items"][0] is not shared
+    assert copied["nested"]["items"][0] is copied["nested"]["items"][1]
+    assert copied["self"] is copied
+    assert copied["leaf"] is leaf
+    assert copied["tuple"] is value["tuple"]
+
+
+@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-metadata", "trials"])
+def test_hooks_nested_metadata_mutation_is_isolated_per_evaluation(with_memory_logger, use_trials):
+    shared_metadata = {"ctx": {"seen": []}}
+    inputs = ["row"] if use_trials else ["first", "second"]
+
+    def task(input, hooks):
+        hooks.metadata["ctx"]["seen"].append(f"{input}:{hooks.trial_index}")
+        return list(hooks.metadata["ctx"]["seen"])
+
+    def metadata_isolation(output, metadata):
+        return float(metadata["ctx"]["seen"] == output)
+
+    result = Eval(
+        "isolated-hook-metadata",
+        data=[EvalCase(input=value, metadata=shared_metadata) for value in inputs],
+        task=task,
+        scores=[metadata_isolation],
+        trial_count=2 if use_trials else 1,
+        max_concurrency=1,
+        no_send_logs=True,
+        enable_cache=False,
+    )
+
+    assert [row.error for row in result.results] == [None, None]
+    assert [len(row.output) for row in result.results] == [1, 1]
+    assert [row.scores["metadata_isolation"] for row in result.results] == [1.0, 1.0]
+    assert [row.metadata["ctx"]["seen"] for row in result.results] == [row.output for row in result.results]
+    assert shared_metadata == {"ctx": {"seen": []}}
+
+
 @pytest.mark.parametrize("tags", [["a"], ("a",), {"a"}], ids=["list", "tuple", "set"])
 def test_dict_eval_hooks_copies_tags(tags):
     original = type(tags)(tags)

From 9390d3d298ecc2f99d94d82e79e1c24f98a6fa4f Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:16:26 +0000
Subject: [PATCH 05/11] fix(eval): copy expected for each evaluation

hooks.expected and the scorers shared the EvalCase's expected object, so
a task that changed it affected later trials, other rows sharing the same
value, and the caller's dataset. Give each evaluation its own copy of
plain dicts and lists in expected, and use that copy for the hooks,
scorers, logged span, and EvalResult.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      |  9 +++++----
 py/src/braintrust/test_framework.py | 30 +++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 4 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index 933e18b5b..397fb215d 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1733,6 +1733,7 @@ async def run_evaluator_task(datum, trial_index=0):
             datum = EvalCase.from_dict(datum)
 
         metadata = _copy_containers({**(datum.metadata or {})})
+        expected = _copy_containers(datum.expected)
         output = None
         error = None
         exc_info = None
@@ -1767,7 +1768,7 @@ async def run_evaluator_task(datum, trial_index=0):
             name="eval",
             span_attributes={"type": SpanTypeAttribute.EVAL},
             input=datum.input,
-            expected=datum.expected,
+            expected=expected,
             tags=tags,
             **({"origin": origin} if origin is not None else {}),
         )
@@ -1802,7 +1803,7 @@ def report_progress(event: TaskProgressEvent):
 
                 hooks = DictEvalHooks(
                     metadata,
-                    expected=datum.expected,
+                    expected=expected,
                     trial_index=trial_index,
                     tags=tags,
                     report_progress=report_progress,
@@ -1900,7 +1901,7 @@ async def ensure_spans_flushed():
 
                 scorer_kwargs = {
                     "input": datum.input,
-                    "expected": datum.expected,
+                    "expected": expected,
                     "metadata": metadata,
                     "output": output,
                     "trace": trace,
@@ -1987,7 +1988,7 @@ async def ensure_spans_flushed():
 
         return EvalResult(
             input=datum.input,
-            expected=datum.expected,
+            expected=expected,
             metadata=metadata,
             tags=tags,
             output=output,
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index e99f9b185..86a2a99aa 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1561,6 +1561,36 @@ def metadata_isolation(output, metadata):
     assert shared_metadata == {"ctx": {"seen": []}}
 
 
+@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-expected", "trials"])
+def test_hooks_expected_mutation_is_isolated_per_evaluation(with_memory_logger, use_trials):
+    shared_expected = ["a"]
+    inputs = ["row"] if use_trials else ["first", "second"]
+
+    def task(input, hooks):
+        hooks.expected.append(f"{input}:{hooks.trial_index}")
+        return list(hooks.expected)
+
+    def expected_isolation(output, expected):
+        return float(expected == output)
+
+    result = Eval(
+        "isolated-hook-expected",
+        data=[EvalCase(input=value, expected=shared_expected) for value in inputs],
+        task=task,
+        scores=[expected_isolation],
+        trial_count=2 if use_trials else 1,
+        max_concurrency=1,
+        no_send_logs=True,
+        enable_cache=False,
+    )
+
+    assert [row.error for row in result.results] == [None, None]
+    assert [len(row.output) for row in result.results] == [2, 2]
+    assert [row.scores["expected_isolation"] for row in result.results] == [1.0, 1.0]
+    assert [row.expected for row in result.results] == [row.output for row in result.results]
+    assert shared_expected == ["a"]
+
+
 @pytest.mark.parametrize("tags", [["a"], ("a",), {"a"}], ids=["list", "tuple", "set"])
 def test_dict_eval_hooks_copies_tags(tags):
     original = type(tags)(tags)

From 5d291e0d62cfca2abd2e4d8c34afcc6b9c3a3d6f Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:16:47 +0000
Subject: [PATCH 06/11] test(eval): cover tag isolation across concurrent
 trials

The existing tag isolation tests run with max_concurrency=1. Add a test
that keeps every trial of one row in flight at once, so sharing a tags
list between concurrently running trials is caught.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/test_framework.py | 35 +++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 86a2a99aa..9899eb7f5 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1486,6 +1486,41 @@ def tag_isolation(output, tags):
     assert with_memory_logger.pop() == []
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("initial_tags", [None, ["seed"]], ids=["none", "nonempty"])
+async def test_hooks_tags_mutation_is_isolated_across_concurrent_trials(with_memory_logger, initial_tags):
+    trial_count = 3
+    base_tags = list(initial_tags or [])
+    appended = 0
+    all_appended = asyncio.Event()
+
+    async def task(input, hooks):
+        nonlocal appended
+        own_tag = f"trial:{hooks.trial_index}"
+        hooks.tags.append(own_tag)
+        appended += 1
+        if appended == trial_count:
+            all_appended.set()
+        # Hold every trial open until all of them have appended, so they overlap.
+        await asyncio.wait_for(all_appended.wait(), timeout=5)
+        return list(hooks.tags)
+
+    result = await EvalAsync(
+        "concurrent-hook-tags",
+        data=[EvalCase(input="row", tags=initial_tags)],
+        task=task,
+        scores=[],
+        trial_count=trial_count,
+        no_send_logs=True,
+        enable_cache=False,
+    )
+
+    assert [row.error for row in result.results] == [None] * trial_count
+    assert sorted(row.output[-1] for row in result.results) == [f"trial:{i}" for i in range(trial_count)]
+    assert [row.output for row in result.results] == [[*base_tags, row.output[-1]] for row in result.results]
+    assert initial_tags == (None if initial_tags is None else base_tags)
+
+
 @pytest.mark.parametrize("initial_tags", [None, [], ["seed-z", "seed-a"]], ids=["none", "empty", "nonempty"])
 def test_hooks_tags_clear_is_local_to_each_trial(with_memory_logger, initial_tags):
     initial_tags = None if initial_tags is None else list(initial_tags)

From 1eb4699c889ff2642095a150e2353d598ae90781 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:17:06 +0000
Subject: [PATCH 07/11] test(eval): cover tuple and set EvalCase tags

Copying tags into a list lets hooks.tags.append() work when an EvalCase
gives tags as a tuple or set, which used to raise AttributeError. Add a
test so this behavior can't quietly regress.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/test_framework.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 9899eb7f5..61514ffde 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1486,6 +1486,31 @@ def tag_isolation(output, tags):
     assert with_memory_logger.pop() == []
 
 
+@pytest.mark.parametrize("initial_tags", [("seed",), {"seed"}], ids=["tuple", "set"])
+def test_hooks_tags_from_tuple_or_set_are_appendable_per_trial(with_memory_logger, initial_tags):
+    original_tags = type(initial_tags)(initial_tags)
+
+    def task(input, hooks):
+        hooks.tags.append(f"trial:{hooks.trial_index}")
+        return list(hooks.tags)
+
+    result = Eval(
+        "sequence-hook-tags",
+        data=[EvalCase(input="row", tags=initial_tags)],
+        task=task,
+        scores=[],
+        trial_count=2,
+        max_concurrency=1,
+        no_send_logs=True,
+        enable_cache=False,
+    )
+
+    assert [row.error for row in result.results] == [None, None]
+    assert [row.output for row in result.results] == [["seed", "trial:0"], ["seed", "trial:1"]]
+    assert [row.tags for row in result.results] == [row.output for row in result.results]
+    assert initial_tags == original_tags
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("initial_tags", [None, ["seed"]], ids=["none", "nonempty"])
 async def test_hooks_tags_mutation_is_isolated_across_concurrent_trials(with_memory_logger, initial_tags):

From a7892bcba31c8cf0aca05fbf62b05b65c31a7900 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:35:44 +0000
Subject: [PATCH 08/11] test(eval): consolidate hook isolation tests

Fold the separate tags, tuple/set tags, nested metadata, and expected
isolation tests into one parametrized test that changes all three in the
same task. Drop the hooks.tags.clear() test (it checks the same sharing
as append), the direct DictEvalHooks constructor test (the eval tests go
through the constructor now that the call-site copy is gone), and
duplicate parameter cases. 33 test cases become 12, and undoing any one
of the fixes still fails a test.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/test_framework.py | 189 +++++-----------------------
 1 file changed, 32 insertions(+), 157 deletions(-)

diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 61514ffde..25be64316 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -11,7 +11,6 @@
 from braintrust.util import LazyValue
 
 from .framework import (
-    DictEvalHooks,
     Eval,
     EvalAsync,
     EvalCase,
@@ -1446,32 +1445,32 @@ def task_with_hooks(input, hooks):
     assert root_span[0].get("tags") == expected_tags
 
 
-@pytest.mark.parametrize("as_dict", [False, True], ids=["dataclass", "dict"])
-@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-tags", "trials"])
-@pytest.mark.parametrize("initial_tags", [None, [], ["seed-z", "seed-a"]], ids=["none", "empty", "nonempty"])
-def test_hooks_tags_mutation_is_isolated_per_evaluation(with_memory_logger, as_dict, use_trials, initial_tags):
-    initial_tags = None if initial_tags is None else list(initial_tags)
-    original_tags = None if initial_tags is None else list(initial_tags)
+@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-data", "trials"])
+@pytest.mark.parametrize("initial_tags", [None, ["seed"], ("seed",), {"seed"}], ids=["none", "list", "tuple", "set"])
+def test_hook_mutations_are_isolated_per_evaluation(with_memory_logger, use_trials, initial_tags):
+    original_tags = None if initial_tags is None else type(initial_tags)(initial_tags)
     base_tags = list(initial_tags or [])
+    metadata = {"ctx": {"seen": []}}
+    expected = ["seed"]
     inputs = ["row"] if use_trials else ["first", "second"]
-    data = [
-        {"input": value, "tags": initial_tags} if as_dict else EvalCase(input=value, tags=initial_tags)
-        for value in inputs
-    ]
 
     def task(input, hooks):
-        own_tag = f"{input}:{hooks.trial_index}"
-        hooks.tags.append(own_tag)
-        return own_tag
-
-    def tag_isolation(output, tags):
-        return float(tags == [*base_tags, output])
+        own = f"{input}:{hooks.trial_index}"
+        hooks.tags.append(own)
+        hooks.metadata["ctx"]["seen"].append(own)
+        hooks.expected.append(own)
+        return own
+
+    def isolation(output, expected, metadata, tags):
+        return float(
+            tags == [*base_tags, output] and metadata["ctx"]["seen"] == [output] and expected == ["seed", output]
+        )
 
     result = Eval(
-        "isolated-hook-tags",
-        data=data,
+        "isolated-hook-mutations",
+        data=[EvalCase(input=value, expected=expected, metadata=metadata, tags=initial_tags) for value in inputs],
         task=task,
-        scores=[tag_isolation],
+        scores=[isolation],
         trial_count=2 if use_trials else 1,
         max_concurrency=1,
         no_send_logs=True,
@@ -1479,50 +1478,26 @@ def tag_isolation(output, tags):
     )
 
     assert [row.error for row in result.results] == [None, None]
-    assert [row.scores["tag_isolation"] for row in result.results] == [1.0, 1.0]
-    assert result.summary.scores["tag_isolation"].score == 1.0
-    assert [row.tags for row in result.results] == [[*base_tags, row.output] for row in result.results]
-    assert initial_tags == original_tags
-    assert with_memory_logger.pop() == []
-
-
-@pytest.mark.parametrize("initial_tags", [("seed",), {"seed"}], ids=["tuple", "set"])
-def test_hooks_tags_from_tuple_or_set_are_appendable_per_trial(with_memory_logger, initial_tags):
-    original_tags = type(initial_tags)(initial_tags)
-
-    def task(input, hooks):
-        hooks.tags.append(f"trial:{hooks.trial_index}")
-        return list(hooks.tags)
-
-    result = Eval(
-        "sequence-hook-tags",
-        data=[EvalCase(input="row", tags=initial_tags)],
-        task=task,
-        scores=[],
-        trial_count=2,
-        max_concurrency=1,
-        no_send_logs=True,
-        enable_cache=False,
-    )
-
-    assert [row.error for row in result.results] == [None, None]
-    assert [row.output for row in result.results] == [["seed", "trial:0"], ["seed", "trial:1"]]
-    assert [row.tags for row in result.results] == [row.output for row in result.results]
+    for row in result.results:
+        assert row.tags == [*base_tags, row.output]
+        assert row.metadata["ctx"]["seen"] == [row.output]
+        assert row.expected == ["seed", row.output]
+        assert row.scores["isolation"] == 1.0
     assert initial_tags == original_tags
+    assert metadata == {"ctx": {"seen": []}}
+    assert expected == ["seed"]
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("initial_tags", [None, ["seed"]], ids=["none", "nonempty"])
-async def test_hooks_tags_mutation_is_isolated_across_concurrent_trials(with_memory_logger, initial_tags):
+async def test_hooks_tags_mutation_is_isolated_across_concurrent_trials(with_memory_logger):
     trial_count = 3
-    base_tags = list(initial_tags or [])
+    initial_tags = ["seed"]
     appended = 0
     all_appended = asyncio.Event()
 
     async def task(input, hooks):
         nonlocal appended
-        own_tag = f"trial:{hooks.trial_index}"
-        hooks.tags.append(own_tag)
+        hooks.tags.append(f"trial:{hooks.trial_index}")
         appended += 1
         if appended == trial_count:
             all_appended.set()
@@ -1541,37 +1516,8 @@ async def task(input, hooks):
     )
 
     assert [row.error for row in result.results] == [None] * trial_count
-    assert sorted(row.output[-1] for row in result.results) == [f"trial:{i}" for i in range(trial_count)]
-    assert [row.output for row in result.results] == [[*base_tags, row.output[-1]] for row in result.results]
-    assert initial_tags == (None if initial_tags is None else base_tags)
-
-
-@pytest.mark.parametrize("initial_tags", [None, [], ["seed-z", "seed-a"]], ids=["none", "empty", "nonempty"])
-def test_hooks_tags_clear_is_local_to_each_trial(with_memory_logger, initial_tags):
-    initial_tags = None if initial_tags is None else list(initial_tags)
-    original_tags = None if initial_tags is None else list(initial_tags)
-
-    def task(input, hooks):
-        before = list(hooks.tags)
-        hooks.tags.clear()
-        return before
-
-    result = Eval(
-        "cleared-hook-tags",
-        data=[EvalCase(input="row", tags=initial_tags)],
-        task=task,
-        scores=[],
-        trial_count=2,
-        max_concurrency=1,
-        no_send_logs=True,
-        enable_cache=False,
-    )
-
-    assert [row.error for row in result.results] == [None, None]
-    assert [row.output for row in result.results] == [list(original_tags or [])] * 2
-    assert [row.tags for row in result.results] == [None, None]
-    assert initial_tags == original_tags
-    assert with_memory_logger.pop() == []
+    assert sorted(row.output for row in result.results) == [["seed", f"trial:{i}"] for i in range(trial_count)]
+    assert initial_tags == ["seed"]
 
 
 def test_copy_containers_copies_dicts_and_lists_only():
@@ -1591,79 +1537,8 @@ def test_copy_containers_copies_dicts_and_lists_only():
     assert copied["tuple"] is value["tuple"]
 
 
-@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-metadata", "trials"])
-def test_hooks_nested_metadata_mutation_is_isolated_per_evaluation(with_memory_logger, use_trials):
-    shared_metadata = {"ctx": {"seen": []}}
-    inputs = ["row"] if use_trials else ["first", "second"]
-
-    def task(input, hooks):
-        hooks.metadata["ctx"]["seen"].append(f"{input}:{hooks.trial_index}")
-        return list(hooks.metadata["ctx"]["seen"])
-
-    def metadata_isolation(output, metadata):
-        return float(metadata["ctx"]["seen"] == output)
-
-    result = Eval(
-        "isolated-hook-metadata",
-        data=[EvalCase(input=value, metadata=shared_metadata) for value in inputs],
-        task=task,
-        scores=[metadata_isolation],
-        trial_count=2 if use_trials else 1,
-        max_concurrency=1,
-        no_send_logs=True,
-        enable_cache=False,
-    )
-
-    assert [row.error for row in result.results] == [None, None]
-    assert [len(row.output) for row in result.results] == [1, 1]
-    assert [row.scores["metadata_isolation"] for row in result.results] == [1.0, 1.0]
-    assert [row.metadata["ctx"]["seen"] for row in result.results] == [row.output for row in result.results]
-    assert shared_metadata == {"ctx": {"seen": []}}
-
-
-@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-expected", "trials"])
-def test_hooks_expected_mutation_is_isolated_per_evaluation(with_memory_logger, use_trials):
-    shared_expected = ["a"]
-    inputs = ["row"] if use_trials else ["first", "second"]
-
-    def task(input, hooks):
-        hooks.expected.append(f"{input}:{hooks.trial_index}")
-        return list(hooks.expected)
-
-    def expected_isolation(output, expected):
-        return float(expected == output)
-
-    result = Eval(
-        "isolated-hook-expected",
-        data=[EvalCase(input=value, expected=shared_expected) for value in inputs],
-        task=task,
-        scores=[expected_isolation],
-        trial_count=2 if use_trials else 1,
-        max_concurrency=1,
-        no_send_logs=True,
-        enable_cache=False,
-    )
-
-    assert [row.error for row in result.results] == [None, None]
-    assert [len(row.output) for row in result.results] == [2, 2]
-    assert [row.scores["expected_isolation"] for row in result.results] == [1.0, 1.0]
-    assert [row.expected for row in result.results] == [row.output for row in result.results]
-    assert shared_expected == ["a"]
-
-
-@pytest.mark.parametrize("tags", [["a"], ("a",), {"a"}], ids=["list", "tuple", "set"])
-def test_dict_eval_hooks_copies_tags(tags):
-    original = type(tags)(tags)
-    hooks = DictEvalHooks(tags=tags)
-
-    hooks.tags.append("b")
-
-    assert tags == original
-    assert hooks.tags == ["a", "b"]
-
-
 @pytest.mark.asyncio
-@pytest.mark.parametrize("tags", ["prod", "foo", {"prod": True}], ids=["str", "str-repeated-chars", "dict"])
+@pytest.mark.parametrize("tags", ["prod", {"prod": True}], ids=["str", "dict"])
 async def test_non_sequence_eval_case_tags_are_rejected(with_memory_logger, with_simulate_login, tags):
     evaluator = Evaluator(
         project_name="test-project",

From 0d7af1fbb8453cf714651b7715d3b658b59d30d1 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:45:28 +0000
Subject: [PATCH 09/11] refactor(eval): simplify hook isolation helper and
 tests

- _copy_containers only recurses into dicts and lists, so plain values
  skip a function call (about 32% faster on small and large inputs).
- Reword the DictEvalHooks tags comment: span validation runs on the
  original tags before the hooks are built, so the pass-through exists
  to avoid splitting a string or dict into tags.
- Drop the concurrent-trials test. Each trial already gets its own copy
  in the constructor, so the sequential trials case catches the same
  regressions without the event and timeout machinery.
- Use copy.copy for the test's tags snapshot.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      |  8 ++++---
 py/src/braintrust/test_framework.py | 35 ++---------------------------
 2 files changed, 7 insertions(+), 36 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index 397fb215d..caf99bcdd 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1231,15 +1231,17 @@ def _copy_containers(value: Any, memo: dict[int, Any] | None = None) -> Any:
         memo = {}
     if id(value) in memo:
         return memo[id(value)]
+    # Only recurse into containers, so plain values skip the extra call.
     if value_type is dict:
         copied_dict: dict[Any, Any] = {}
         memo[id(value)] = copied_dict
         for k, v in value.items():
-            copied_dict[k] = _copy_containers(v, memo)
+            copied_dict[k] = _copy_containers(v, memo) if type(v) is dict or type(v) is list else v
         return copied_dict
     copied_list: list[Any] = []
     memo[id(value)] = copied_list
-    copied_list.extend(_copy_containers(v, memo) for v in value)
+    for v in value:
+        copied_list.append(_copy_containers(v, memo) if type(v) is dict or type(v) is list else v)
     return copied_list
 
 
@@ -1266,7 +1268,7 @@ def __init__(
             # Copy so hook mutations don't leak into the caller's tags.
             self.update({"tags": list(tags)})
         else:
-            # Leave other values as-is so span tag validation rejects them.
+            # Leave other values as-is instead of splitting a string or dict into tags.
             self.update({"tags": tags})
 
         self._report_progress = report_progress
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 25be64316..d2da5c136 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import importlib.util
 import re
 import sys
@@ -1448,7 +1449,7 @@ def task_with_hooks(input, hooks):
 @pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-data", "trials"])
 @pytest.mark.parametrize("initial_tags", [None, ["seed"], ("seed",), {"seed"}], ids=["none", "list", "tuple", "set"])
 def test_hook_mutations_are_isolated_per_evaluation(with_memory_logger, use_trials, initial_tags):
-    original_tags = None if initial_tags is None else type(initial_tags)(initial_tags)
+    original_tags = copy.copy(initial_tags)
     base_tags = list(initial_tags or [])
     metadata = {"ctx": {"seen": []}}
     expected = ["seed"]
@@ -1488,38 +1489,6 @@ def isolation(output, expected, metadata, tags):
     assert expected == ["seed"]
 
 
-@pytest.mark.asyncio
-async def test_hooks_tags_mutation_is_isolated_across_concurrent_trials(with_memory_logger):
-    trial_count = 3
-    initial_tags = ["seed"]
-    appended = 0
-    all_appended = asyncio.Event()
-
-    async def task(input, hooks):
-        nonlocal appended
-        hooks.tags.append(f"trial:{hooks.trial_index}")
-        appended += 1
-        if appended == trial_count:
-            all_appended.set()
-        # Hold every trial open until all of them have appended, so they overlap.
-        await asyncio.wait_for(all_appended.wait(), timeout=5)
-        return list(hooks.tags)
-
-    result = await EvalAsync(
-        "concurrent-hook-tags",
-        data=[EvalCase(input="row", tags=initial_tags)],
-        task=task,
-        scores=[],
-        trial_count=trial_count,
-        no_send_logs=True,
-        enable_cache=False,
-    )
-
-    assert [row.error for row in result.results] == [None] * trial_count
-    assert sorted(row.output for row in result.results) == [["seed", f"trial:{i}"] for i in range(trial_count)]
-    assert initial_tags == ["seed"]
-
-
 def test_copy_containers_copies_dicts_and_lists_only():
     leaf = object()
     shared = [1]

From f2a42a501a5e025ca2fd889069695e107d965330 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 21:01:58 +0000
Subject: [PATCH 10/11] revert(eval): drop per-trial deep copies of metadata
 and expected

Copying metadata and expected for every trial cost about 6ms per row and
nearly doubled peak memory for evals with large reference data, since
each EvalResult kept its own copy. Mutating nested metadata or expected
from a task isn't a documented pattern, while hooks.tags.append() is.
Keep the tags fix and go back to main's handling of metadata (shallow
copy) and expected (shared). This also removes _copy_containers.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      | 37 +++----------------
 py/src/braintrust/test_framework.py | 55 +++++++----------------------
 2 files changed, 18 insertions(+), 74 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index caf99bcdd..e5c229c6a 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1219,32 +1219,6 @@ def evaluate_filter(object, filter: Filter):
     return filter.pattern.match(serialize_json_with_plain_string(key)) is not None
 
 
-def _copy_containers(value: Any, memo: dict[int, Any] | None = None) -> Any:
-    """Recursively copy plain dicts and lists so one evaluation's mutations don't leak into another.
-
-    Other objects are shared rather than deep-copied, since they may be expensive or impossible to copy.
-    """
-    value_type = type(value)
-    if value_type is not dict and value_type is not list:
-        return value
-    if memo is None:
-        memo = {}
-    if id(value) in memo:
-        return memo[id(value)]
-    # Only recurse into containers, so plain values skip the extra call.
-    if value_type is dict:
-        copied_dict: dict[Any, Any] = {}
-        memo[id(value)] = copied_dict
-        for k, v in value.items():
-            copied_dict[k] = _copy_containers(v, memo) if type(v) is dict or type(v) is list else v
-        return copied_dict
-    copied_list: list[Any] = []
-    memo[id(value)] = copied_list
-    for v in value:
-        copied_list.append(_copy_containers(v, memo) if type(v) is dict or type(v) is list else v)
-    return copied_list
-
-
 class DictEvalHooks(dict[str, Any]):
     def __init__(
         self,
@@ -1734,8 +1708,7 @@ async def run_evaluator_task(datum, trial_index=0):
         if isinstance(datum, dict):
             datum = EvalCase.from_dict(datum)
 
-        metadata = _copy_containers({**(datum.metadata or {})})
-        expected = _copy_containers(datum.expected)
+        metadata = {**(datum.metadata or {})}
         output = None
         error = None
         exc_info = None
@@ -1770,7 +1743,7 @@ async def run_evaluator_task(datum, trial_index=0):
             name="eval",
             span_attributes={"type": SpanTypeAttribute.EVAL},
             input=datum.input,
-            expected=expected,
+            expected=datum.expected,
             tags=tags,
             **({"origin": origin} if origin is not None else {}),
         )
@@ -1805,7 +1778,7 @@ def report_progress(event: TaskProgressEvent):
 
                 hooks = DictEvalHooks(
                     metadata,
-                    expected=expected,
+                    expected=datum.expected,
                     trial_index=trial_index,
                     tags=tags,
                     report_progress=report_progress,
@@ -1903,7 +1876,7 @@ async def ensure_spans_flushed():
 
                 scorer_kwargs = {
                     "input": datum.input,
-                    "expected": expected,
+                    "expected": datum.expected,
                     "metadata": metadata,
                     "output": output,
                     "trace": trace,
@@ -1990,7 +1963,7 @@ async def ensure_spans_flushed():
 
         return EvalResult(
             input=datum.input,
-            expected=expected,
+            expected=datum.expected,
             metadata=metadata,
             tags=tags,
             output=output,
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index d2da5c136..2e52bd2d7 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -20,7 +20,6 @@
     Evaluator,
     Filter,
     _call_user_fn_args,
-    _copy_containers,
     await_or_run,
     evaluate_filter,
     parse_filters,
@@ -1446,32 +1445,26 @@ def task_with_hooks(input, hooks):
     assert root_span[0].get("tags") == expected_tags
 
 
-@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-data", "trials"])
+@pytest.mark.parametrize("use_trials", [False, True], ids=["shared-row-tags", "trials"])
 @pytest.mark.parametrize("initial_tags", [None, ["seed"], ("seed",), {"seed"}], ids=["none", "list", "tuple", "set"])
-def test_hook_mutations_are_isolated_per_evaluation(with_memory_logger, use_trials, initial_tags):
+def test_hooks_tags_mutation_is_isolated_per_evaluation(with_memory_logger, use_trials, initial_tags):
     original_tags = copy.copy(initial_tags)
     base_tags = list(initial_tags or [])
-    metadata = {"ctx": {"seen": []}}
-    expected = ["seed"]
     inputs = ["row"] if use_trials else ["first", "second"]
 
     def task(input, hooks):
-        own = f"{input}:{hooks.trial_index}"
-        hooks.tags.append(own)
-        hooks.metadata["ctx"]["seen"].append(own)
-        hooks.expected.append(own)
-        return own
-
-    def isolation(output, expected, metadata, tags):
-        return float(
-            tags == [*base_tags, output] and metadata["ctx"]["seen"] == [output] and expected == ["seed", output]
-        )
+        own_tag = f"{input}:{hooks.trial_index}"
+        hooks.tags.append(own_tag)
+        return own_tag
+
+    def tag_isolation(output, tags):
+        return float(tags == [*base_tags, output])
 
     result = Eval(
-        "isolated-hook-mutations",
-        data=[EvalCase(input=value, expected=expected, metadata=metadata, tags=initial_tags) for value in inputs],
+        "isolated-hook-tags",
+        data=[EvalCase(input=value, tags=initial_tags) for value in inputs],
         task=task,
-        scores=[isolation],
+        scores=[tag_isolation],
         trial_count=2 if use_trials else 1,
         max_concurrency=1,
         no_send_logs=True,
@@ -1479,31 +1472,9 @@ def isolation(output, expected, metadata, tags):
     )
 
     assert [row.error for row in result.results] == [None, None]
-    for row in result.results:
-        assert row.tags == [*base_tags, row.output]
-        assert row.metadata["ctx"]["seen"] == [row.output]
-        assert row.expected == ["seed", row.output]
-        assert row.scores["isolation"] == 1.0
+    assert [row.tags for row in result.results] == [[*base_tags, row.output] for row in result.results]
+    assert [row.scores["tag_isolation"] for row in result.results] == [1.0, 1.0]
     assert initial_tags == original_tags
-    assert metadata == {"ctx": {"seen": []}}
-    assert expected == ["seed"]
-
-
-def test_copy_containers_copies_dicts_and_lists_only():
-    leaf = object()
-    shared = [1]
-    value = {"nested": {"items": [shared, shared]}, "leaf": leaf, "tuple": (shared,)}
-    value["self"] = value
-
-    copied = _copy_containers(value)
-
-    assert copied is not value
-    assert copied["nested"] is not value["nested"]
-    assert copied["nested"]["items"][0] is not shared
-    assert copied["nested"]["items"][0] is copied["nested"]["items"][1]
-    assert copied["self"] is copied
-    assert copied["leaf"] is leaf
-    assert copied["tuple"] is value["tuple"]
 
 
 @pytest.mark.asyncio

From f37b20aff433e5356bce99869504dead637c2bdd Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 21:13:06 +0000
Subject: [PATCH 11/11] fix(eval): keep hook tags when the task raises

Tags are now copied into the hooks, but only the success path read
hooks.tags back. A task that appended a tag and then raised lost it from
EvalResult.tags, which main only kept because the list was shared. Read
hooks.tags on the error path too and log them with the error, so the
root span also gets them (main never logged them there).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      |  6 +++++-
 py/src/braintrust/test_framework.py | 30 +++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index e5c229c6a..4099a552f 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1760,6 +1760,7 @@ async def run_evaluator_task(datum, trial_index=0):
             # In most cases this will be a no-op span, but if the parent is set, it will use that ctx.
             root_span = start_span(state=state, **base_event)
 
+        hooks = None
         with root_span:
             try:
 
@@ -1954,7 +1955,10 @@ async def ensure_spans_flushed():
                     root_span.log(metadata=metadata, tags=tags)
             except Exception as e:
                 exc_type, exc_value, tb = sys.exc_info()
-                root_span.log(error=stringify_exception(exc_type, exc_value, tb))
+                if hooks is not None:
+                    # Keep tags the task added before it failed.
+                    tags = hooks.tags if hooks.tags else None
+                root_span.log(error=stringify_exception(exc_type, exc_value, tb), tags=tags)
 
                 error = e
                 # Python3.10 has a different set of arguments to format_exception than earlier versions,
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 2e52bd2d7..e01076cbc 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1477,6 +1477,36 @@ def tag_isolation(output, tags):
     assert initial_tags == original_tags
 
 
+@pytest.mark.asyncio
+async def test_hooks_tags_are_kept_when_task_raises(with_memory_logger, with_simulate_login):
+    def task(input, hooks):
+        hooks.tags.append("failed-step")
+        raise ValueError("task failed")
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-evaluator",
+        data=[EvalCase(input="row", tags=["seed"])],
+        task=task,
+        scores=[],
+        experiment_name=None,
+        metadata=None,
+        summarize_scores=False,
+    )
+
+    result = await run_evaluator(
+        experiment=init_test_exp("test-evaluator", "test-project"),
+        evaluator=evaluator,
+        position=None,
+        filters=[],
+    )
+
+    assert isinstance(result.results[0].error, ValueError)
+    assert result.results[0].tags == ["seed", "failed-step"]
+    root_spans = [log for log in with_memory_logger.pop() if not log["span_parents"]]
+    assert [span.get("tags") for span in root_spans] == [["seed", "failed-step"]]
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("tags", ["prod", {"prod": True}], ids=["str", "dict"])
 async def test_non_sequence_eval_case_tags_are_rejected(with_memory_logger, with_simulate_login, tags):