From 41dcb413ca8f4f6a1fbe76c86f76a28200b124fb Mon Sep 17 00:00:00 2001
From: Raphael Fakhri <153192858+RaphaelFakhri@users.noreply.github.com>
Date: Sat, 10 Oct 2026 19:24:50 +0000
Subject: [PATCH 1/6] fix(eval): track unhandled scores per row for the error
 score handler

unhandled_scores was shared across every row of an eval, so a row whose task raised used whatever the last finished row left behind. The error score handler then ran with a partial scorer list or not at all. Track it per row, matching the JavaScript SDK.
---
 py/src/braintrust/framework.py      |  3 +--
 py/src/braintrust/test_framework.py | 41 +++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index 58a8c08ce..f5ea3a76a 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1686,7 +1686,6 @@ async def await_or_run_classifier(root_span, classifier, name, **kwargs):
     scorer_names = [_scorer_name(scorer, i) for i, scorer in enumerate(scorers)]
     classifiers = list(evaluator.classifiers or [])
     classifier_names = [_classifier_name(classifier, i) for i, classifier in enumerate(classifiers)]
-    unhandled_scores = scorer_names
 
     if evaluator.parameter_values is not None:
         resolved_evaluator_parameters = evaluator.parameter_values
@@ -1706,6 +1705,7 @@ async def run_evaluator_task(datum, trial_index=0):
         error = None
         exc_info = None
         scores = {}
+        unhandled_scores = scorer_names
         classifications = {}
         tags = datum.tags
 
@@ -1919,7 +1919,6 @@ async def ensure_spans_flushed():
                 if classifications:
                     root_span.log(classifications=classifications)
 
-                nonlocal unhandled_scores
                 unhandled_scores = None
                 if failing_scorers_and_exceptions:
                     scorer_errors = {
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index c3284e537..658c83155 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -20,6 +20,7 @@
     Filter,
     _call_user_fn_args,
     await_or_run,
+    default_error_score_handler,
     evaluate_filter,
     parse_filters,
     run_evaluator,
@@ -1134,6 +1135,46 @@ def task_with_hooks(input_value: int, hooks: EvalHooks) -> int:
     assert input_3_trials == [0]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("first_input", ["ok", "scorer_error"])
+async def test_error_score_handler_scores_each_failed_row_independently(first_input):
+    """A failed task gets error scores for every scorer, regardless of how earlier rows finished."""
+
+    async def task(input_value):
+        if input_value == "task_error":
+            raise ValueError("task failed")
+        return input_value
+
+    def exact_match(output, expected):
+        return 1.0 if output == expected else 0.0
+
+    def picky(output):
+        if output == "scorer_error":
+            raise ValueError("scorer failed")
+        return 1.0
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-error-score-handler",
+        data=[
+            EvalCase(input=first_input, expected=first_input),
+            EvalCase(input="task_error", expected="task_error"),
+        ],
+        task=task,
+        scores=[exact_match, picky],
+        experiment_name=None,
+        metadata=None,
+        max_concurrency=1,
+        error_score_handler=default_error_score_handler,
+    )
+
+    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
+
+    failed = next(r for r in result.results if r.input == "task_error")
+    assert isinstance(failed.error, ValueError)
+    assert failed.scores == {"exact_match": 0, "picky": 0}
+
+
 @pytest.mark.vcr
 @pytest.mark.asyncio
 async def test_scorer_spans_have_purpose_attribute(with_memory_logger, with_simulate_login):

From f8d778a31483fd662f8f75714b9e16ba76c8b324 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 19:56:55 +0000
Subject: [PATCH 2/6] fix(eval): keep the eval running when the error score
 handler raises

The error score handler was called while building the EvalResult, outside
the row's try/except, so an exception from a user handler escaped
run_evaluator_task and cancelled the whole eval. Catch it, report it, and
keep the row without error scores.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      | 16 ++++++------
 py/src/braintrust/test_framework.py | 40 +++++++++++++++++++++++++++++
 2 files changed, 48 insertions(+), 8 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index f5ea3a76a..e980ceadb 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1953,20 +1953,20 @@ async def ensure_spans_flushed():
                 # so just capture the stack trace here.
                 exc_info = traceback.format_exc()
 
+        error_scores = {}
+        if evaluator.error_score_handler is not None and unhandled_scores:
+            try:
+                error_scores = evaluator.error_score_handler(root_span, datum, unhandled_scores) or {}
+            except Exception as e:
+                eprint(f"Error score handler failed for scorers: {', '.join(unhandled_scores)}", e)
+
         return EvalResult(
             input=datum.input,
             expected=datum.expected,
             metadata=metadata,
             tags=tags,
             output=output,
-            scores={
-                **(
-                    evaluator.error_score_handler(root_span, datum, unhandled_scores) or {}
-                    if evaluator.error_score_handler is not None and unhandled_scores
-                    else {}
-                ),
-                **scores,
-            },
+            scores={**error_scores, **scores},
             classifications=classifications or None,
             error=error,
             exc_info=exc_info,
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 658c83155..87bb19a80 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1175,6 +1175,46 @@ def picky(output):
     assert failed.scores == {"exact_match": 0, "picky": 0}
 
 
+@pytest.mark.asyncio
+async def test_error_score_handler_exception_does_not_abort_eval():
+    """An error score handler that raises only loses that row's error scores; the rest of the eval still runs."""
+
+    async def task(input_value):
+        if input_value == "task_error":
+            raise ValueError("task failed")
+        return input_value
+
+    def exact_match(output, expected):
+        return 1.0 if output == expected else 0.0
+
+    def broken_handler(root_span, data, unhandled_scores):
+        raise KeyError("handler failed")
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-error-score-handler-raises",
+        data=[
+            EvalCase(input="task_error", expected="task_error"),
+            EvalCase(input="ok", expected="ok"),
+        ],
+        task=task,
+        scores=[exact_match],
+        experiment_name=None,
+        metadata=None,
+        max_concurrency=1,
+        error_score_handler=broken_handler,
+    )
+
+    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
+
+    results = {r.input: r for r in result.results}
+    assert set(results) == {"task_error", "ok"}
+    assert isinstance(results["task_error"].error, ValueError)
+    assert results["task_error"].scores == {}
+    assert results["ok"].error is None
+    assert results["ok"].scores == {"exact_match": 1.0}
+
+
 @pytest.mark.vcr
 @pytest.mark.asyncio
 async def test_scorer_spans_have_purpose_attribute(with_memory_logger, with_simulate_login):

From 6cb6273db2e3d8186487f8520f671c7053064c52 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 19:57:31 +0000
Subject: [PATCH 3/6] fix(eval): pass each row its own copy of the unhandled
 scorer names

Each row started from the evaluator-wide scorer_names list itself, and that
same list was handed to the user's error score handler. A handler that
mutated its argument changed which scorers ran on every later row.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      |  2 +-
 py/src/braintrust/test_framework.py | 48 +++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index e980ceadb..c43f9e007 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1705,7 +1705,7 @@ async def run_evaluator_task(datum, trial_index=0):
         error = None
         exc_info = None
         scores = {}
-        unhandled_scores = scorer_names
+        unhandled_scores = list(scorer_names)
         classifications = {}
         tags = datum.tags
 
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 87bb19a80..112b56af3 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1215,6 +1215,54 @@ def broken_handler(root_span, data, unhandled_scores):
     assert results["ok"].scores == {"exact_match": 1.0}
 
 
+@pytest.mark.asyncio
+async def test_error_score_handler_mutating_unhandled_scores_does_not_affect_other_rows():
+    """Mutating the unhandled_scores list passed to the handler must not change which scorers run on later rows."""
+
+    async def task(input_value):
+        if input_value.startswith("task_error"):
+            raise ValueError("task failed")
+        return input_value
+
+    def exact_match(output, expected):
+        return 1.0 if output == expected else 0.0
+
+    def picky(output):
+        return 1.0
+
+    seen = []
+
+    def consuming_handler(root_span, data, unhandled_scores):
+        seen.append(list(unhandled_scores))
+        scores = {}
+        while unhandled_scores:
+            scores[unhandled_scores.pop()] = 0
+        return scores
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-error-score-handler-mutation",
+        data=[
+            EvalCase(input="task_error_1", expected="task_error_1"),
+            EvalCase(input="ok", expected="ok"),
+            EvalCase(input="task_error_2", expected="task_error_2"),
+        ],
+        task=task,
+        scores=[exact_match, picky],
+        experiment_name=None,
+        metadata=None,
+        max_concurrency=1,
+        error_score_handler=consuming_handler,
+    )
+
+    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
+
+    results = {r.input: r for r in result.results}
+    assert results["ok"].scores == {"exact_match": 1.0, "picky": 1.0}
+    assert results["task_error_2"].scores == {"exact_match": 0, "picky": 0}
+    assert seen == [["exact_match", "picky"], ["exact_match", "picky"]]
+
+
 @pytest.mark.vcr
 @pytest.mark.asyncio
 async def test_scorer_spans_have_purpose_attribute(with_memory_logger, with_simulate_login):

From e84cdcf5178538c32937cf65ca53da1e5c04acc0 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 19:58:02 +0000
Subject: [PATCH 4/6] fix(eval): run the error score handler before the root
 span ends

The handler ran after `with root_span:` exited, so the scores that
default_error_score_handler logs to the root span were written after
metrics.end. Call it inside the span block instead.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      | 13 ++++++-----
 py/src/braintrust/test_framework.py | 34 +++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index c43f9e007..92b3f0869 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1953,12 +1953,13 @@ async def ensure_spans_flushed():
                 # so just capture the stack trace here.
                 exc_info = traceback.format_exc()
 
-        error_scores = {}
-        if evaluator.error_score_handler is not None and unhandled_scores:
-            try:
-                error_scores = evaluator.error_score_handler(root_span, datum, unhandled_scores) or {}
-            except Exception as e:
-                eprint(f"Error score handler failed for scorers: {', '.join(unhandled_scores)}", e)
+            # Run inside the root span so error scores the handler logs land before the span ends.
+            error_scores = {}
+            if evaluator.error_score_handler is not None and unhandled_scores:
+                try:
+                    error_scores = evaluator.error_score_handler(root_span, datum, unhandled_scores) or {}
+                except Exception as e:
+                    eprint(f"Error score handler failed for scorers: {', '.join(unhandled_scores)}", e)
 
         return EvalResult(
             input=datum.input,
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 112b56af3..64fbf07f8 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1263,6 +1263,40 @@ def consuming_handler(root_span, data, unhandled_scores):
     assert seen == [["exact_match", "picky"], ["exact_match", "picky"]]
 
 
+@pytest.mark.asyncio
+async def test_error_score_handler_logs_before_root_span_ends(with_memory_logger, with_simulate_login):
+    """Error scores are logged to the root span before it ends, not as a late write after metrics.end."""
+
+    async def task(input_value):
+        raise ValueError("task failed")
+
+    def exact_match(output, expected):
+        return 1.0 if output == expected else 0.0
+
+    evaluator = Evaluator(
+        project_name="test-project",
+        eval_name="test-error-score-handler-span-end",
+        data=[EvalCase(input="task_error", expected="task_error")],
+        task=task,
+        scores=[exact_match],
+        experiment_name=None,
+        metadata=None,
+        summarize_scores=False,
+        error_score_handler=default_error_score_handler,
+    )
+    experiment = init_test_exp("test-error-score-handler-span-end", "test-project")
+
+    await run_evaluator(experiment=experiment, evaluator=evaluator, position=None, filters=[])
+
+    # Inspect the raw, unmerged writes so their order is visible.
+    records = [record for item in with_memory_logger.logs if (record := item.get()) is not None]
+    root_records = [record for record in records if not record["span_parents"]]
+    score_index = next(i for i, record in enumerate(root_records) if record.get("scores"))
+    end_index = next(i for i, record in enumerate(root_records) if record.get("metrics", {}).get("end"))
+    assert root_records[score_index]["scores"] == {"exact_match": 0}
+    assert score_index < end_index
+
+
 @pytest.mark.vcr
 @pytest.mark.asyncio
 async def test_scorer_spans_have_purpose_attribute(with_memory_logger, with_simulate_login):

From 28783c8dd2e86d1cb1460926067034cdf6795930 Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:06:28 +0000
Subject: [PATCH 5/6] test(eval): consolidate the error score handler tests

Merge the per-row leak test and the handler-mutation test into one test,
since both check that each row's handler gets the right scorer names; the
merged test also covers the scorer-error path. Share an evaluator builder
across the three remaining tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/test_framework.py | 137 ++++++----------------------
 1 file changed, 30 insertions(+), 107 deletions(-)

diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 64fbf07f8..7f4ac5313 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1135,13 +1135,11 @@ def task_with_hooks(input_value: int, hooks: EvalHooks) -> int:
     assert input_3_trials == [0]
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("first_input", ["ok", "scorer_error"])
-async def test_error_score_handler_scores_each_failed_row_independently(first_input):
-    """A failed task gets error scores for every scorer, regardless of how earlier rows finished."""
+def _error_score_evaluator(inputs, error_score_handler, **kwargs):
+    """Rows named "task_error*" fail in the task; the "picky" scorer fails on "scorer_error" rows."""
 
     async def task(input_value):
-        if input_value == "task_error":
+        if input_value.startswith("task_error"):
             raise ValueError("task failed")
         return input_value
 
@@ -1153,83 +1151,23 @@ def picky(output):
             raise ValueError("scorer failed")
         return 1.0
 
-    evaluator = Evaluator(
+    return Evaluator(
         project_name="test-project",
         eval_name="test-error-score-handler",
-        data=[
-            EvalCase(input=first_input, expected=first_input),
-            EvalCase(input="task_error", expected="task_error"),
-        ],
+        data=[EvalCase(input=i, expected=i) for i in inputs],
         task=task,
         scores=[exact_match, picky],
         experiment_name=None,
         metadata=None,
         max_concurrency=1,
-        error_score_handler=default_error_score_handler,
-    )
-
-    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
-
-    failed = next(r for r in result.results if r.input == "task_error")
-    assert isinstance(failed.error, ValueError)
-    assert failed.scores == {"exact_match": 0, "picky": 0}
-
-
-@pytest.mark.asyncio
-async def test_error_score_handler_exception_does_not_abort_eval():
-    """An error score handler that raises only loses that row's error scores; the rest of the eval still runs."""
-
-    async def task(input_value):
-        if input_value == "task_error":
-            raise ValueError("task failed")
-        return input_value
-
-    def exact_match(output, expected):
-        return 1.0 if output == expected else 0.0
-
-    def broken_handler(root_span, data, unhandled_scores):
-        raise KeyError("handler failed")
-
-    evaluator = Evaluator(
-        project_name="test-project",
-        eval_name="test-error-score-handler-raises",
-        data=[
-            EvalCase(input="task_error", expected="task_error"),
-            EvalCase(input="ok", expected="ok"),
-        ],
-        task=task,
-        scores=[exact_match],
-        experiment_name=None,
-        metadata=None,
-        max_concurrency=1,
-        error_score_handler=broken_handler,
+        error_score_handler=error_score_handler,
+        **kwargs,
     )
 
-    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
-
-    results = {r.input: r for r in result.results}
-    assert set(results) == {"task_error", "ok"}
-    assert isinstance(results["task_error"].error, ValueError)
-    assert results["task_error"].scores == {}
-    assert results["ok"].error is None
-    assert results["ok"].scores == {"exact_match": 1.0}
-
 
 @pytest.mark.asyncio
-async def test_error_score_handler_mutating_unhandled_scores_does_not_affect_other_rows():
-    """Mutating the unhandled_scores list passed to the handler must not change which scorers run on later rows."""
-
-    async def task(input_value):
-        if input_value.startswith("task_error"):
-            raise ValueError("task failed")
-        return input_value
-
-    def exact_match(output, expected):
-        return 1.0 if output == expected else 0.0
-
-    def picky(output):
-        return 1.0
-
+async def test_error_score_handler_gets_all_scorers_for_each_failed_row():
+    """Each row gets the names of exactly the scorers it is missing, regardless of earlier rows or handler mutations."""
     seen = []
 
     def consuming_handler(root_span, data, unhandled_scores):
@@ -1239,53 +1177,38 @@ def consuming_handler(root_span, data, unhandled_scores):
             scores[unhandled_scores.pop()] = 0
         return scores
 
-    evaluator = Evaluator(
-        project_name="test-project",
-        eval_name="test-error-score-handler-mutation",
-        data=[
-            EvalCase(input="task_error_1", expected="task_error_1"),
-            EvalCase(input="ok", expected="ok"),
-            EvalCase(input="task_error_2", expected="task_error_2"),
-        ],
-        task=task,
-        scores=[exact_match, picky],
-        experiment_name=None,
-        metadata=None,
-        max_concurrency=1,
-        error_score_handler=consuming_handler,
-    )
-
+    evaluator = _error_score_evaluator(["scorer_error", "task_error_1", "ok", "task_error_2"], consuming_handler)
     result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
 
     results = {r.input: r for r in result.results}
+    assert seen == [["picky"], ["exact_match", "picky"], ["exact_match", "picky"]]
+    assert results["scorer_error"].scores == {"exact_match": 1.0, "picky": 0}
+    assert results["task_error_1"].scores == {"exact_match": 0, "picky": 0}
     assert results["ok"].scores == {"exact_match": 1.0, "picky": 1.0}
     assert results["task_error_2"].scores == {"exact_match": 0, "picky": 0}
-    assert seen == [["exact_match", "picky"], ["exact_match", "picky"]]
 
 
 @pytest.mark.asyncio
-async def test_error_score_handler_logs_before_root_span_ends(with_memory_logger, with_simulate_login):
-    """Error scores are logged to the root span before it ends, not as a late write after metrics.end."""
+async def test_error_score_handler_exception_does_not_abort_eval():
+    """An error score handler that raises only loses that row's error scores; the rest of the eval still runs."""
 
-    async def task(input_value):
-        raise ValueError("task failed")
+    def broken_handler(root_span, data, unhandled_scores):
+        raise KeyError("handler failed")
 
-    def exact_match(output, expected):
-        return 1.0 if output == expected else 0.0
+    evaluator = _error_score_evaluator(["task_error", "ok"], broken_handler)
+    result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
 
-    evaluator = Evaluator(
-        project_name="test-project",
-        eval_name="test-error-score-handler-span-end",
-        data=[EvalCase(input="task_error", expected="task_error")],
-        task=task,
-        scores=[exact_match],
-        experiment_name=None,
-        metadata=None,
-        summarize_scores=False,
-        error_score_handler=default_error_score_handler,
-    )
-    experiment = init_test_exp("test-error-score-handler-span-end", "test-project")
+    results = {r.input: r for r in result.results}
+    assert isinstance(results["task_error"].error, ValueError)
+    assert results["task_error"].scores == {}
+    assert results["ok"].scores == {"exact_match": 1.0, "picky": 1.0}
 
+
+@pytest.mark.asyncio
+async def test_error_score_handler_logs_before_root_span_ends(with_memory_logger, with_simulate_login):
+    """Error scores are logged to the root span before it ends, not as a late write after metrics.end."""
+    evaluator = _error_score_evaluator(["task_error"], default_error_score_handler, summarize_scores=False)
+    experiment = init_test_exp("test-error-score-handler-span-end", "test-project")
     await run_evaluator(experiment=experiment, evaluator=evaluator, position=None, filters=[])
 
     # Inspect the raw, unmerged writes so their order is visible.
@@ -1293,7 +1216,7 @@ def exact_match(output, expected):
     root_records = [record for record in records if not record["span_parents"]]
     score_index = next(i for i, record in enumerate(root_records) if record.get("scores"))
     end_index = next(i for i, record in enumerate(root_records) if record.get("metrics", {}).get("end"))
-    assert root_records[score_index]["scores"] == {"exact_match": 0}
+    assert root_records[score_index]["scores"] == {"exact_match": 0, "picky": 0}
     assert score_index < end_index
 
 

From 4b5e8879dc1d19ef03e66ecc5a418fcb848fd84b Mon Sep 17 00:00:00 2001
From: Abhijeet Prasad <devabhiprasad@gmail.com>
Date: Wed, 14 Oct 2026 20:10:01 +0000
Subject: [PATCH 6/6] refactor(eval): copy unhandled scorer names when calling
 the error score handler

Copy the list at the point it is handed to user code instead of at the start
of every row. Only failed rows pay for the copy, and a handler that drains
its list before raising no longer leaves the warning without scorer names.
Collapse the per-row score asserts in the consolidated test into one dict
comparison.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 py/src/braintrust/framework.py      |  5 +++--
 py/src/braintrust/test_framework.py | 11 ++++++-----
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/py/src/braintrust/framework.py b/py/src/braintrust/framework.py
index 92b3f0869..31445f219 100644
--- a/py/src/braintrust/framework.py
+++ b/py/src/braintrust/framework.py
@@ -1705,7 +1705,7 @@ async def run_evaluator_task(datum, trial_index=0):
         error = None
         exc_info = None
         scores = {}
-        unhandled_scores = list(scorer_names)
+        unhandled_scores = scorer_names
         classifications = {}
         tags = datum.tags
 
@@ -1957,7 +1957,8 @@ async def ensure_spans_flushed():
             error_scores = {}
             if evaluator.error_score_handler is not None and unhandled_scores:
                 try:
-                    error_scores = evaluator.error_score_handler(root_span, datum, unhandled_scores) or {}
+                    # Pass a copy so a handler that mutates its argument can't change scorer_names.
+                    error_scores = evaluator.error_score_handler(root_span, datum, list(unhandled_scores)) or {}
                 except Exception as e:
                     eprint(f"Error score handler failed for scorers: {', '.join(unhandled_scores)}", e)
 
diff --git a/py/src/braintrust/test_framework.py b/py/src/braintrust/test_framework.py
index 7f4ac5313..95de12a1d 100644
--- a/py/src/braintrust/test_framework.py
+++ b/py/src/braintrust/test_framework.py
@@ -1180,12 +1180,13 @@ def consuming_handler(root_span, data, unhandled_scores):
     evaluator = _error_score_evaluator(["scorer_error", "task_error_1", "ok", "task_error_2"], consuming_handler)
     result = await run_evaluator(experiment=None, evaluator=evaluator, position=None, filters=[])
 
-    results = {r.input: r for r in result.results}
     assert seen == [["picky"], ["exact_match", "picky"], ["exact_match", "picky"]]
-    assert results["scorer_error"].scores == {"exact_match": 1.0, "picky": 0}
-    assert results["task_error_1"].scores == {"exact_match": 0, "picky": 0}
-    assert results["ok"].scores == {"exact_match": 1.0, "picky": 1.0}
-    assert results["task_error_2"].scores == {"exact_match": 0, "picky": 0}
+    assert {r.input: r.scores for r in result.results} == {
+        "scorer_error": {"exact_match": 1.0, "picky": 0},
+        "task_error_1": {"exact_match": 0, "picky": 0},
+        "ok": {"exact_match": 1.0, "picky": 1.0},
+        "task_error_2": {"exact_match": 0, "picky": 0},
+    }
 
 
 @pytest.mark.asyncio