From 2585ffb129da63e1028787587bd2b2506cc4f63c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 10 Sep 2026 22:28:20 -0700 Subject: [PATCH 01/54] feat(timing): book each turn's head and tail as their own buckets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured live on all five harnesses, generation + tool left 0.1%-42% of the turn unexplained, and the whole remainder sat in two places: before the first generation window opened, and after the last one closed. EventCollector now measures both between the agent's own AgentStart/AgentEnd stamps and the first/last AssistantMessage, and publishes them on TurnRecord. One live turn per harness, residual after all four buckets: antigravity wall 14348 ms startup 0.0 teardown 3.5 -0.010 ms claude-code wall 13295 ms startup 0.0 teardown 834.7 +0.086 ms codex wall 11842 ms startup 5075.2 teardown 13.9 -0.019 ms opencode wall 8157 ms startup 3047.9 teardown 33.1 +0.022 ms pi wall 6906 ms startup 345.4 teardown 26.6 +0.621 ms The turn now reconciles to under a millisecond everywhere. The residual sign flips, so the invariant is |residual| < 1 ms rather than <= wall: head and tail are measured between event stamps while duration_seconds is the agent's own monotonic span, and the field descriptions say so. The head is NOT decomposed further, deliberately. Its composition differs per harness and the stream carries no marker to split it: OpenCode's process spawns in 3 ms and its first event lands at 3921 ms, so CLI boot, provider resolution, dispatch and TTFT are fused. claude-code and Antigravity read a measured 0.0 because their first window already covers dispatch — which is also why nothing folds that time OUT of their generation: for an in-process SDK it IS the generation. Hence names for the interval measured, not for what it contains. `agents/_timing.py` moves to `coder_eval/timing.py`. It is stdlib-only, but importing anything under `agents/` executes that package's __init__, which imports every agent, which imports streaming — so the collector could not reach it. A cycle-free leaf beside the other shared arithmetic, mirroring models/cli_match.py's rationale. Both fields join the golden-stream scrub list. They are measured wall values like duration_seconds and generation_duration_ms beside them; left unscrubbed they drifted 24 of 68 golden tests on an unchanged re-run. Co-Authored-By: Claude Opus 5 (1M context) --- .../lib/__tests__/timing-union-parity.test.ts | 2 +- evalboard/lib/timing.ts | 2 +- src/coder_eval/agents/_timing.py | 42 ------- src/coder_eval/agents/antigravity_agent.py | 2 +- src/coder_eval/agents/codex_agent.py | 2 +- src/coder_eval/agents/opencode_agent.py | 4 +- src/coder_eval/agents/pi_agent.py | 4 +- src/coder_eval/models/results.py | 23 ++++ src/coder_eval/streaming/collector.py | 29 +++++ src/coder_eval/timing.py | 84 ++++++++++++++ tests/_fixtures/golden_streams/_scrub.py | 5 + .../antigravity_a_single_text_turn.json | 2 + .../antigravity_b_tool_call_resolved.json | 2 + ...y_c_thinking_and_tool_same_generation.json | 2 + .../expected/antigravity_d_orphaned_tool.json | 2 + .../antigravity_e_multi_generation.json | 2 + .../expected/claude_a_single_text_turn.json | 2 + .../expected/claude_b_tool_use_result.json | 2 + .../claude_c_multi_emission_delta.json | 2 + .../expected/claude_d_subagent_terminal.json | 2 + .../claude_e_model_usage_and_backfill.json | 2 + .../expected/claude_f_orphaned_tool.json | 2 + .../claude_g_crash_format_placeholder.json | 2 + .../claude_h1_timeout_process_error.json | 2 + .../claude_h2_process_error_crash.json | 2 + .../claude_i_in_loop_deadline_break.json | 2 + .../expected/codex_a_agent_message_only.json | 2 + .../expected/codex_b_command_execution.json | 2 + .../codex_c_reasoning_placeholder.json | 2 + .../codex_d_cross_flush_is_error.json | 2 + .../expected/codex_e_orphan_tool.json | 2 + .../expected/codex_f_collab_fallback.json | 2 + .../expected/codex_g_items_rebuild.json | 2 + .../codex_h_no_turn_completed_crash.json | 2 + .../expected/opencode_a_single_text_turn.json | 2 + .../opencode_b_tool_call_resolved.json | 2 + .../expected/pi_a_single_text_turn.json | 2 + .../expected/pi_b_tool_call_resolved.json | 2 + tests/_fixtures/timing_union_cases.json | 4 +- tests/test_antigravity_agent.py | 2 +- tests/test_event_collector.py | 103 +++++++++++++++++- tests/test_timing_union_parity.py | 6 +- 42 files changed, 311 insertions(+), 57 deletions(-) delete mode 100644 src/coder_eval/agents/_timing.py create mode 100644 src/coder_eval/timing.py diff --git a/evalboard/lib/__tests__/timing-union-parity.test.ts b/evalboard/lib/__tests__/timing-union-parity.test.ts index 734c33f6d..f2df82bb1 100644 --- a/evalboard/lib/__tests__/timing-union-parity.test.ts +++ b/evalboard/lib/__tests__/timing-union-parity.test.ts @@ -5,7 +5,7 @@ import { describe, expect, test } from "vitest"; import { busyMs, toolExecutionMs } from "../timing"; import type { MessageEvent, MessageToolUse } from "../runs"; -// Parity guard: `busyMs` here and `coder_eval.agents._timing.busy_ms` in the +// Parity guard: `busyMs` here and `coder_eval.timing.busy_ms` in the // Python harness answer the same question about the same task.json — how much // wall clock the tools occupied — one to subtract it from a generation window, // the other to subtract it from the task's duration. A divergence makes the diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts index 598195999..5833ae065 100644 --- a/evalboard/lib/timing.ts +++ b/evalboard/lib/timing.ts @@ -136,7 +136,7 @@ export function epochMs(value: string | null | undefined): number | null { // Milliseconds inside [lo, hi] where at least ONE span was running: the UNION, // not the sum. // -// The TypeScript twin of `coder_eval.agents._timing.busy_ms`, deliberately the +// The TypeScript twin of `coder_eval.timing.busy_ms`, deliberately the // same algorithm — the agents subtract tool time from a generation window with // it, and this file subtracts tool time from a task's wall clock, so the two // must agree about what "tool execution took N ms" means. Held in step by diff --git a/src/coder_eval/agents/_timing.py b/src/coder_eval/agents/_timing.py deleted file mode 100644 index 6d568e56c..000000000 --- a/src/coder_eval/agents/_timing.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Shared timing helpers for agent implementations. - -Two harnesses interleave tool execution into a single generation window — -Antigravity (the Step for the tool arrives and only a later ``usage_metadata`` -Step cuts the message) and Codex (``_flush_message``'s window is extended to -the last item's ``completed_at_ms``). Both must therefore subtract the tool -time from the window before publishing ``generation_duration_ms``, and both -must subtract the same thing: the UNION of the closed intervals, clipped to -the window. -""" - -from datetime import datetime - - -def busy_ms(spans: list[tuple[datetime, datetime]], lo: datetime, hi: datetime) -> float: - """Wall milliseconds inside ``[lo, hi]`` where at least ONE span was running. - - The union, not the sum. Tool intervals overlap in practice — Antigravity - resolves several calls from one ``Step`` and backgrounds anything over ten - seconds; Codex spawns collab agents that run concurrently — so adding - their durations over-counts the busy time by exactly the overlap. - Subtracting such a sum from a generation window understates generation - and, with enough concurrency, drives it negative: four concurrent 400 ms - calls inside a 1000 ms window sum to 1600 ms, clamping the result to the - ``0.0`` that "unknown timing says unknown" exists to eliminate. - - Clipping to ``[lo, hi]`` is the other half: a tool that opened before this - window only spent part of its life inside it, and only that part is not - generation time here. - """ - clipped = sorted((max(s, lo), min(e, hi)) for s, e in spans if min(e, hi) > max(s, lo)) - if not clipped: - return 0.0 - total = 0.0 - open_start, open_end = clipped[0] - for start, end in clipped[1:]: - if start > open_end: # disjoint — bank the run and start a new one - total += (open_end - open_start).total_seconds() * 1000.0 - open_start, open_end = start, end - else: # overlapping or adjacent — extend the run - open_end = max(open_end, end) - return total + (open_end - open_start).total_seconds() * 1000.0 diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 6c3269ece..f1a339144 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -31,7 +31,6 @@ from coder_eval.agent import Agent, AgentState from coder_eval.agents._logging import PrefixedAdapter -from coder_eval.agents._timing import busy_ms from coder_eval.agents.registry import AgentRegistry from coder_eval.agents.watchdog import ThreadedWatchdog from coder_eval.config import settings @@ -68,6 +67,7 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.timing import busy_ms from coder_eval.utils import expand_env_vars diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index b8ffea865..0b5f39b33 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -17,7 +17,6 @@ from coder_eval.agent import Agent, AgentState from coder_eval.agents._logging import PrefixedAdapter, log_raw_sdk_event -from coder_eval.agents._timing import busy_ms from coder_eval.agents.registry import AgentRegistry from coder_eval.agents.watchdog import ThreadedWatchdog from coder_eval.config import settings @@ -54,6 +53,7 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.timing import busy_ms from coder_eval.utils import expand_env_vars diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 419042536..be6920117 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -43,7 +43,6 @@ from typing import Any, ClassVar, Literal, NoReturn from coder_eval.agent import Agent -from coder_eval.agents._timing import busy_ms from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES from coder_eval.models import ( @@ -77,6 +76,7 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.timing import busy_ms from ._skills import _plugin_skill_dirs from .registry import AgentRegistry @@ -331,7 +331,7 @@ def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str # milliseconds twice — once here and once as the tool's own # duration_ms. Intervals, not a running total: they overlap # whenever the harness runs tools concurrently, and only their - # union may be subtracted (agents/_timing.py::busy_ms). + # union may be subtracted (timing.py::busy_ms). self.step_tool_spans: list[tuple[datetime, datetime]] = [] # callID -> (telemetry, started_at) for tools awaiting a result. diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index f7ae7d6d8..e327e66a1 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -77,7 +77,6 @@ from coder_eval.agent import Agent from coder_eval.agents._skills import _plugin_skill_dirs # shared plugin->skills resolver -from coder_eval.agents._timing import busy_ms from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.isolation.docker_runner import STDOUT_LINE_LIMIT_BYTES from coder_eval.models import ( @@ -110,6 +109,7 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.timing import busy_ms from .registry import AgentRegistry @@ -294,7 +294,7 @@ def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str # milliseconds twice — once here and once as the tool's own # duration_ms. Intervals, not a running total: they overlap # whenever the harness runs tools concurrently, and only their - # union may be subtracted (agents/_timing.py::busy_ms). + # union may be subtracted (timing.py::busy_ms). self.turn_tool_spans: list[tuple[datetime, datetime]] = [] # toolCallId -> telemetry for tools awaiting a result. diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index a12bddd26..765fc2816 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -325,6 +325,29 @@ class TurnRecord(BaseModel): ) timestamp: datetime = Field(default_factory=datetime.now, description="When this turn occurred") duration_seconds: float = Field(default=0.0, description="How long this turn took") + harness_startup_ms: float | None = Field( + default=None, + description=( + "Wall milliseconds between the agent turn starting and the first generation window " + "opening, measured between AGENT EVENT stamps — not from timestamp/duration_seconds " + "above, which are orchestrator-level and a slightly different clock, so a consumer " + "recomputing this from those will get a near-but-not-equal number. Its COMPOSITION " + "differs per harness and is deliberately not decomposed: on an in-process SDK the " + "first window already covers dispatch and time-to-first-token so this reads ~0, while " + "on a subprocess harness it fuses CLI boot, provider resolution, dispatch and TTFT, " + "which the event stream gives no marker to separate. See docs/agents/HARNESS_PARITY.md. " + "None when the turn produced no assistant message — never 0.0, which would mean " + "'measured, and instant'." + ), + ) + harness_teardown_ms: float | None = Field( + default=None, + description=( + "Wall milliseconds between the last generation window closing and the agent turn " + "ending: SDK/CLI finalization, result assembly and process teardown. Same clock " + "caveat as harness_startup_ms. None when the turn produced no assistant message." + ), + ) token_usage: TokenUsage | None = Field( default=None, description="Token usage for this turn (if available from agent SDK)" ) diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index ebac3ee93..03f794556 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -22,6 +22,8 @@ from __future__ import annotations +from datetime import datetime + from coder_eval.models import ( AssistantMessage, CommandTelemetry, @@ -37,6 +39,7 @@ ToolEndEvent, TurnStartEvent, ) +from coder_eval.timing import decompose_turn class EventCollector: @@ -54,6 +57,8 @@ def __init__(self) -> None: self._user_input: str = "" self._model: str | None = None self._turn_starts: int = 0 + # Stamped by AgentStartEvent; the head is measured from it. + self._agent_start_at: datetime | None = None # tool_id -> finalized telemetry (last ToolEnd wins, mirroring last-result-wins). self._commands: dict[str, CommandTelemetry] = {} self._agent_end: AgentEndEvent | None = None @@ -71,6 +76,7 @@ def on_event(self, event: StreamEvent) -> None: if isinstance(event, AgentStartEvent): self._iteration = event.iteration self._user_input = event.prompt + self._agent_start_at = event.timestamp if event.model: self._model = event.model elif isinstance(event, TurnStartEvent): @@ -103,6 +109,25 @@ def visible_turn_count(self) -> int: def _ordered_commands(self) -> list[CommandTelemetry]: return sorted(self._commands.values(), key=lambda c: c.sequence_number) + def _overhead_ms(self, messages: list[TranscriptMessage]) -> tuple[float | None, float | None]: + """The turn's head and tail — the wall clock the generations do not cover. + + Measured against the FIRST and LAST ``AssistantMessage``, not + ``messages[0]`` / ``messages[-1]``: a simulation turn interleaves + ``UserMessage`` entries, and a reconciled turn ends with a + ``ReconciliationMessage`` that carries no timestamps at all, so indexing + the raw list would measure the wrong thing or raise. + """ + generations = [m for m in messages if isinstance(m, AssistantMessage)] + if not generations: + return None, None + return decompose_turn( + generations[0].started_at, + generations[-1].completed_at, + self._agent_start_at, + self._agent_end.timestamp if self._agent_end is not None else None, + ) + @staticmethod def _reconciled_messages(messages: list[TranscriptMessage], usage: TokenUsage) -> list[TranscriptMessage]: """Append a ``ReconciliationMessage`` so the transcript's token buckets @@ -193,6 +218,8 @@ def build_turn_record(self) -> TurnRecord: if token_usage is not None: messages = self._reconciled_messages(messages, token_usage) + startup_ms, teardown_ms = self._overhead_ms(messages) + return TurnRecord( iteration=end.iteration or self._iteration, user_input=end.user_input or self._user_input, @@ -208,4 +235,6 @@ def build_turn_record(self) -> TurnRecord: result_summary=end.result_summary, crashed=end.crashed, crash_reason=end.crash_reason, + harness_startup_ms=startup_ms, + harness_teardown_ms=teardown_ms, ) diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py new file mode 100644 index 000000000..b2950896a --- /dev/null +++ b/src/coder_eval/timing.py @@ -0,0 +1,84 @@ +"""Shared timing helpers for agent implementations. + +Two harnesses interleave tool execution into a single generation window — +Antigravity (the Step for the tool arrives and only a later ``usage_metadata`` +Step cuts the message) and Codex (``_flush_message``'s window is extended to +the last item's ``completed_at_ms``). Both must therefore subtract the tool +time from the window before publishing ``generation_duration_ms``, and both +must subtract the same thing: the UNION of the closed intervals, clipped to +the window. +""" + +from datetime import datetime + + +def busy_ms(spans: list[tuple[datetime, datetime]], lo: datetime, hi: datetime) -> float: + """Wall milliseconds inside ``[lo, hi]`` where at least ONE span was running. + + The union, not the sum. Tool intervals overlap in practice — Antigravity + resolves several calls from one ``Step`` and backgrounds anything over ten + seconds; Codex spawns collab agents that run concurrently — so adding + their durations over-counts the busy time by exactly the overlap. + Subtracting such a sum from a generation window understates generation + and, with enough concurrency, drives it negative: four concurrent 400 ms + calls inside a 1000 ms window sum to 1600 ms, clamping the result to the + ``0.0`` that "unknown timing says unknown" exists to eliminate. + + Clipping to ``[lo, hi]`` is the other half: a tool that opened before this + window only spent part of its life inside it, and only that part is not + generation time here. + """ + clipped = sorted((max(s, lo), min(e, hi)) for s, e in spans if min(e, hi) > max(s, lo)) + if not clipped: + return 0.0 + total = 0.0 + open_start, open_end = clipped[0] + for start, end in clipped[1:]: + if start > open_end: # disjoint — bank the run and start a new one + total += (open_end - open_start).total_seconds() * 1000.0 + open_start, open_end = start, end + else: # overlapping or adjacent — extend the run + open_end = max(open_end, end) + return total + (open_end - open_start).total_seconds() * 1000.0 + + +def decompose_turn( + first_started_at: datetime | None, + last_completed_at: datetime | None, + agent_started_at: datetime | None, + agent_ended_at: datetime | None, +) -> tuple[float | None, float | None]: + """Wall ms before the first generation window opens, and after the last closes. + + The turn's two unexplained ends. Between them the windows tile (each + harness's generation mark runs to the next) and tool execution is already + subtracted inside them, so head + generation + tool + tail is the whole + turn. Defined once here rather than in five agents, and consumed by + ``EventCollector``, the golden-stream sensor, and + ``scripts/timing/decompose_run.py``. + + What the head CONTAINS differs per harness and is deliberately NOT split. + On an in-process SDK the first window already covers dispatch and + time-to-first-token, so this reads ~0; on a subprocess harness it fuses CLI + boot, provider resolution, dispatch and TTFT, and the stream carries no + marker between them — measured on OpenCode, the process spawns in 3 ms and + the first event lands at 3921 ms. Naming these for the interval they + MEASURE rather than for what they contain is the whole point; see + docs/agents/HARNESS_PARITY.md for the per-harness composition. + + ``None`` means never measured — a turn that produced no generation, or a + snapshot taken before the terminal event. Never 0.0, which would claim a + measurement was taken and came back instant (CE058). A measured inversion + (the two clocks disagreeing) IS a real zero and clamps, because both ends + were observed. + + NOTE a second implementation of this arithmetic lives in the evalboard's + Unaccounted cell (``_sections.tsx``), as ``pricing.ts`` mirrors + ``pricing.py``. Change one, change the other. + """ + head = tail = None + if first_started_at is not None and agent_started_at is not None: + head = max((first_started_at - agent_started_at).total_seconds() * 1000.0, 0.0) + if last_completed_at is not None and agent_ended_at is not None: + tail = max((agent_ended_at - last_completed_at).total_seconds() * 1000.0, 0.0) + return head, tail diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index c84cc77d9..9450bc8ea 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -24,6 +24,11 @@ "duration_ms", "duration_seconds", "generation_duration_ms", + # Measured wall intervals like the two above, so they vary run to run; + # masking keeps None-vs-set (the meaningful distinction) visible while + # the value itself stays out of the snapshot. + "harness_startup_ms", + "harness_teardown_ms", # Cost is a rate-card-dependent float (and is backfilled from the rate # card on timeout/kill), so it is masked too — keeping the snapshot # rate-card-independent. The integer TOKEN buckets stay EXACT; those are diff --git a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json index e388dae48..fe601ee40 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json index 062c9556a..3729ef537 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json index c767a6521..f75f40fbf 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json index 2cc1c723e..ee177ff69 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json index ee0f36cb4..1a47c81f1 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json index febe95e0b..7dd57bd45 100644 --- a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json index 3ef5f669e..d94df272d 100644 --- a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json +++ b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json index 00d6f778c..0644c58c3 100644 --- a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json +++ b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json index 943e0ef79..37c18d97f 100644 --- a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json +++ b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json index 782fa2030..88ee7a648 100644 --- a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json +++ b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json index cae8354c3..9195adde8 100644 --- a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json @@ -26,6 +26,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json index d615eae28..4e60e81d1 100644 --- a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json +++ b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json @@ -5,6 +5,8 @@ "crash_reason": "Communication with agent failed: crash after poison\nStderr output:\nNo stderr captured", "crashed": true, "duration_seconds": "", + "harness_startup_ms": null, + "harness_teardown_ms": null, "iteration": 1, "max_turns_exhausted": false, "messages": [], diff --git a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json index 2e0218969..c87fb5be2 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json +++ b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json @@ -5,6 +5,8 @@ "crash_reason": "Agent turn timed out after 30s", "crashed": true, "duration_seconds": "", + "harness_startup_ms": null, + "harness_teardown_ms": null, "iteration": 1, "max_turns_exhausted": false, "messages": [], diff --git a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json index dcdd6042c..bf0c67150 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json +++ b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json @@ -5,6 +5,8 @@ "crash_reason": "CLI process failed (exit code 1): bad config", "crashed": true, "duration_seconds": "", + "harness_startup_ms": null, + "harness_teardown_ms": null, "iteration": 1, "max_turns_exhausted": false, "messages": [], diff --git a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json index da8254bdd..e0a00c6bd 100644 --- a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json +++ b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json @@ -5,6 +5,8 @@ "crash_reason": "Agent turn timed out after 100s", "crashed": true, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json b/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json index 1ef1685a6..b3b0e23e6 100644 --- a/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json +++ b/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json index 4f1ce309c..df6526577 100644 --- a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json +++ b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json b/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json index 4db03065c..93b837621 100644 --- a/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json +++ b/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json index 55959efe1..320923796 100644 --- a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json +++ b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json index 438c798e3..81459ad05 100644 --- a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json +++ b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json index 74d457e35..b1f83b099 100644 --- a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json +++ b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json @@ -46,6 +46,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json index aed8db923..1c88ea853 100644 --- a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json +++ b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json b/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json index d58244c03..4828acb6c 100644 --- a/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json +++ b/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json @@ -5,6 +5,8 @@ "crash_reason": "Codex turn failed: Turn did not complete (no turn/completed notification received)", "crashed": true, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json index 3d65a8797..bc1b0deaf 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json index 571de9971..8abfd4996 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json @@ -25,6 +25,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json index 49083eacc..025af6ce6 100644 --- a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json @@ -5,6 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json index a1c1aa859..f256747cf 100644 --- a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json @@ -45,6 +45,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/_fixtures/timing_union_cases.json b/tests/_fixtures/timing_union_cases.json index 1154740ad..a9e0ce604 100644 --- a/tests/_fixtures/timing_union_cases.json +++ b/tests/_fixtures/timing_union_cases.json @@ -2,9 +2,9 @@ "_comment": [ "Shared replay corpus for the tool-execution UNION, in milliseconds relative", "to an arbitrary base instant. Two implementations must agree on it:", - " * Python — coder_eval.agents._timing.busy_ms, which subtracts tool time", + " * Python — coder_eval.timing.busy_ms, which subtracts tool time", " from an agent's generation window.", - " * TypeScript — evalboard/lib/runs.ts::busyMs, which subtracts tool time", + " * TypeScript — evalboard/lib/timing.ts::busyMs, which subtracts tool time", " from a task's wall clock to produce the Unaccounted residual.", "They answer the same question about the same task.json, so a divergence", "means the evalboard and the harness disagree about how long the tools ran.", diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index ef6f3587e..0924de62a 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -1546,7 +1546,7 @@ def _at(ms: float) -> datetime: return _CLOCK_BASE + timedelta(milliseconds=ms) def _busy(self, spans, lo=0, hi=10_000) -> float: - from coder_eval.agents._timing import busy_ms + from coder_eval.timing import busy_ms return busy_ms([(self._at(s), self._at(e)) for s, e in spans], self._at(lo), self._at(hi)) diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 76e49e495..324f10fa8 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -8,6 +8,8 @@ from datetime import datetime from typing import ClassVar +import pytest + from coder_eval.models import ( AssistantMessage, CommandTelemetry, @@ -202,7 +204,17 @@ class TestFullFieldParity: # provider_call_costs -> joined in post-run by the orchestrator from the # LiteLLM proxy cost log (litellm_cost.apply_actual_cost), # not emitted by the agent/EventCollector. - _DERIVED: ClassVar[set[str]] = {"commands", "token_usage", "timestamp", "provider_call_costs"} + _DERIVED: ClassVar[set[str]] = { + "commands", + "token_usage", + "timestamp", + "provider_call_costs", + # Measured by the collector between the agent's own start/end event + # stamps and the first/last generation window — not carried on + # AgentEndEvent, because no agent computes them. + "harness_startup_ms", + "harness_teardown_ms", + } def _full_agent_end(self) -> AgentEndEvent: """An AgentEndEvent with every verbatim field set to a non-default sentinel.""" @@ -458,3 +470,92 @@ def test_minimal_record_without_agent_end(self): assert record.model_used == "gpt-x" assert record.assistant_turn_count == 1 assert [c.tool_id for c in record.commands] == ["a"] + + +class TestHarnessOverheadBuckets: + """The turn's two unexplained ends: before the first generation, after the last. + + Measured live across all five harnesses, these two plus generation plus tool + execution account for the turn to within 0.1 ms — so what the evalboard shows + as "Unaccounted" is fully explained rather than merely displayed. The head is + where the harnesses differ most (OpenCode ~3.0 s of CLI boot + TTFT fused, + claude-code a measured 0.0 because its first window already covers dispatch), + which is exactly why it is booked as its own bucket instead of being folded + into generation. + """ + + @staticmethod + def _msg(started: datetime, completed: datetime) -> AssistantMessage: + return AssistantMessage(started_at=started, completed_at=completed, generation_duration_ms=1.0) + + def _record(self, messages, *, start: datetime, end: datetime) -> TurnRecord: + collector = EventCollector() + _feed( + collector, + [ + AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1, timestamp=start), + AgentEndEvent( + task_id=TASK_ID, + usage=TokenUsage(output_tokens=1), + messages=messages, + timestamp=end, + ), + ], + ) + return collector.build_turn_record() + + def test_head_and_tail_are_measured_from_the_agent_event_stamps(self): + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [self._msg(t0.replace(second=2), t0.replace(second=5))], + start=t0, + end=t0.replace(second=9), + ) + assert rec.harness_startup_ms == pytest.approx(2000.0) + assert rec.harness_teardown_ms == pytest.approx(4000.0) + + def test_a_turn_with_no_generation_says_so_rather_than_claiming_zero(self): + """None means never measured; 0.0 would mean measured-and-instant (CE058).""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record([], start=t0, end=t0.replace(second=9)) + assert rec.harness_startup_ms is None + assert rec.harness_teardown_ms is None + + def test_a_measured_zero_head_is_zero_not_none(self): + """claude-code and Antigravity really do open their first window at turn + start, so their head is a genuine 0.0 — the distinction from None is the + whole point of the field.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record([self._msg(t0, t0.replace(second=5))], start=t0, end=t0.replace(second=5)) + assert rec.harness_startup_ms == 0.0 + assert rec.harness_teardown_ms == 0.0 + + def test_the_tail_ignores_a_trailing_reconciliation_entry(self): + """It is always last when present and carries no timestamps at all, so + indexing messages[-1] would raise rather than measure.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [ + self._msg(t0.replace(second=1), t0.replace(second=4)), + ReconciliationMessage(input_tokens=5, note="residual"), + ], + start=t0, + end=t0.replace(second=6), + ) + assert rec.harness_teardown_ms == pytest.approx(2000.0) + + def test_a_clock_inversion_clamps_rather_than_going_negative(self): + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [self._msg(t0.replace(minute=59, hour=11), t0.replace(second=5))], + start=t0, + end=t0.replace(second=1), + ) + assert rec.harness_startup_ms == 0.0 + + def test_a_snapshot_before_the_terminal_event_measures_nothing(self): + collector = EventCollector() + _feed(collector, [AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1)]) + rec = collector.build_turn_record() + assert rec.harness_startup_ms is None + assert rec.harness_teardown_ms is None diff --git a/tests/test_timing_union_parity.py b/tests/test_timing_union_parity.py index 1fac0c979..f1a70ec2d 100644 --- a/tests/test_timing_union_parity.py +++ b/tests/test_timing_union_parity.py @@ -1,7 +1,7 @@ """The Python and TypeScript tool-execution unions must agree. -``coder_eval.agents._timing.busy_ms`` subtracts tool time from an agent's -generation window; ``evalboard/lib/runs.ts::busyMs`` subtracts tool time from a +``coder_eval.timing.busy_ms`` subtracts tool time from an agent's +generation window; ``evalboard/lib/timing.ts::busyMs`` subtracts tool time from a task's wall clock to produce the task page's ``Unaccounted`` residual. They answer the same question about the same ``task.json``, so a divergence is not a style difference — it is the harness and the evalboard reporting two different @@ -20,7 +20,7 @@ import pytest -from coder_eval.agents._timing import busy_ms +from coder_eval.timing import busy_ms _FIXTURE = Path(__file__).parent / "_fixtures" / "timing_union_cases.json" From 7637f36b8ce6d1fb37b0f6300412f94cc4c1e807 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 10 Sep 2026 22:46:32 -0700 Subject: [PATCH 02/54] =?UTF-8?q?test(lint):=202/4=20=E2=80=94=20widen=20C?= =?UTF-8?q?E058=20to=20the=20turn=20head/tail=20buckets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `harness_startup_ms` / `harness_teardown_ms` were cited as CE058-guarded but matched neither `_TIMING_NAME` nor `_TIMING_CONSTRUCTORS`, so the guard the head/tail work leans on did not exist for the two fields it was named for. Add one alternation arm (`[a-z_]*_(?:startup|teardown)_ms`, leading segment required like the `_duration_ms` arm) and `TurnRecord` to the constructor set, which is what arms form 1. Mutating the real collector call site from `harness_startup_ms=startup_ms` to `0.0` now fires the rule. Co-Authored-By: Claude Opus 5 (1M context) --- tests/lint/rules/ce058_no_timing_literal.py | 19 +++++++++++++++--- tests/test_custom_lint.py | 22 +++++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/lint/rules/ce058_no_timing_literal.py b/tests/lint/rules/ce058_no_timing_literal.py index 45bae0473..9021fed8e 100644 --- a/tests/lint/rules/ce058_no_timing_literal.py +++ b/tests/lint/rules/ce058_no_timing_literal.py @@ -15,6 +15,15 @@ milliseconds by a command count of which 70 of 211 in one nightly had never been timed at all. +A third field family joined the first two: ``TurnRecord.harness_startup_ms`` +and ``harness_teardown_ms``, the turn's head and tail buckets. They are the +same invariant one level up — a turn whose stream carried no assistant message +was never timed at either end, and a ``0.0`` there would claim the harness +started instantly, which is exactly the reading that sends a real gap into the +evalboard's ``Unaccounted`` cell while a named bucket says it was measured at +zero. ``0.0`` IS the right answer for an in-process SDK whose first generation +window already covers dispatch, so the two values must stay distinguishable. + Five syntactic forms, one invariant, one id — the shapes the codebase actually produced: @@ -47,16 +56,20 @@ # Trailing-segment match, so `cmd.duration_ms` and `generation_duration_ms` -# fire while `duration_ms_limit` does not. +# fire while `duration_ms_limit` does not. The `_startup_ms` / `_teardown_ms` +# arms need a leading segment for the same reason the `_duration_ms` arm does: +# the shipped fields are `harness_*`, and a bare `startup_ms` is more likely a +# budget than a measurement. _TIMING_NAME = re.compile( - r"^(duration_ms|generation_duration_ms|total_command_time_ms|avg_command_time_ms|[a-z_]*_duration_ms)$" + r"^(duration_ms|generation_duration_ms|total_command_time_ms|avg_command_time_ms" + r"|[a-z_]*_duration_ms|[a-z_]*_(?:startup|teardown)_ms)$" ) # The constructors that carry a timing field. Keying on the callee name is what # makes the alias hazard above real; it is also the only thing an AST rule can # see without type inference. _TIMING_CONSTRUCTORS = frozenset( - {"AssistantMessage", "AssistantMessageTelemetry", "CommandTelemetry", "SlowestCommandInfo"} + {"AssistantMessage", "AssistantMessageTelemetry", "CommandTelemetry", "SlowestCommandInfo", "TurnRecord"} ) _SRC_ROOT = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index c28523535..84a95d532 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4355,6 +4355,28 @@ def test_ignores_a_dict_literal_that_is_not_an_update_kwarg(self): # Scoped to `update=` so an unrelated fixture dict cannot fire. assert not self._run('row = {"duration_ms": 0.0}') + # The head/tail family — the turn-level buckets on TurnRecord. + def test_flags_a_zero_harness_startup(self): + assert self._run("rec = TurnRecord(iteration=0, harness_startup_ms=0.0)") + + def test_flags_a_zero_harness_teardown(self): + assert self._run("rec = TurnRecord(iteration=0, harness_teardown_ms=0)") + + def test_allows_an_unmeasured_harness_startup(self): + assert not self._run("rec = TurnRecord(iteration=0, harness_startup_ms=None)") + + def test_allows_a_measured_harness_startup(self): + assert not self._run("rec = TurnRecord(iteration=0, harness_startup_ms=head_ms)") + + def test_flags_the_head_coalesce(self): + assert self._run("x = rec.harness_startup_ms or 0") + + def test_ignores_a_name_that_merely_starts_with_startup(self): + # Anchored at both ends, and the family needs a leading segment: a + # limit is not a measurement, and a bare `startup_ms` is not ours. + assert not self._run("cfg = TurnRecord(startup_ms_limit=0)") + assert not self._run("x = startup_ms_limit or 0") + # Scope + suppression. def test_is_out_of_scope_outside_src(self): assert not self._run( From 1f6f2a8f16e45a3c0af107e21a85ba5c556c0958 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 10 Sep 2026 22:54:33 -0700 Subject: [PATCH 03/54] =?UTF-8?q?feat(evalboard):=203/4=20=E2=80=94=20name?= =?UTF-8?q?=20the=20harness=20head=20and=20tail=20in=20the=20timeline=20st?= =?UTF-8?q?rip?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Unaccounted cell was reporting a harness's CLI boot as unexplained time: opencode's ~3.4s head and claude-code's ~1.0s tail are measured intervals, not residual. Parse `harness_startup_ms` / `harness_teardown_ms` off each turn, sum them across the task's iterations, render them as their own Startup and Teardown cells, and subtract both so Unaccounted is a true residual. Aggregation is `null` — never 0 — when no turn measured that end, mirroring the TurnRecord fields' own contract; a measured 0 (an in-process SDK whose first generation window already covers dispatch) is preserved and renders as `0ms`. An older run without either field renders exactly as before, including the 25% red threshold, which now reads the corrected number in both directions. Co-Authored-By: Claude Opus 5 (1M context) --- .../__tests__/message-timeline.test.tsx | 139 ++++++++++++++++++ .../app/runs/[id]/[...task]/_sections.tsx | 65 ++++++-- evalboard/app/runs/[id]/[...task]/page.tsx | 2 + .../lib/__tests__/harnessOverhead.test.ts | 79 ++++++++++ evalboard/lib/__tests__/runs.test.ts | 55 +++++++ evalboard/lib/runs.ts | 45 ++++++ 6 files changed, 375 insertions(+), 10 deletions(-) create mode 100644 evalboard/lib/__tests__/harnessOverhead.test.ts diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx index afa011028..de4c431d5 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx @@ -678,6 +678,145 @@ describe("MessageTimelineSection — Unaccounted cell", () => { }); }); +describe("MessageTimelineSection — Startup and Teardown cells", () => { + function cell(label: string): HTMLElement { + const parent = screen.getByText(label).parentElement as HTMLElement; + return parent.children[1] as HTMLElement; + } + + // Same 4s generation + 1s tool exec fixture the Unaccounted block uses, so + // the two blocks' numbers are directly comparable. + function renderStrip(props: { + taskDurationSeconds?: number | null; + harnessStartupMs?: number | null; + harnessTeardownMs?: number | null; + }) { + const m = makeMessage({ + generationMs: 4000, + textMs: 4000, + toolUses: [ + { + toolName: "Bash", + toolUseId: "tu_1", + summary: "ls", + argText: "ls", + description: null, + genMs: null, + durationMs: 1000, + isError: false, + resultPreview: null, + outputTokens: null, + resultTokens: null, + execStartMs: null, + execEndMs: null, + }, + ], + }); + return render(); + } + + test("both buckets render their measured value", () => { + renderStrip({ + taskDurationSeconds: 10, + harnessStartupMs: 3000, + harnessTeardownMs: 1500, + }); + expect(cell("Startup").textContent).toBe("3.0s"); + expect(cell("Teardown").textContent).toBe("1.5s"); + }); + + test("a measured zero renders as 0ms, not as an em-dash", () => { + // claude-code and antigravity really do measure ~0 here — their first + // generation window already covers dispatch. "—" would report that + // honest measurement as a missing one. + renderStrip({ + taskDurationSeconds: 10, + harnessStartupMs: 0, + harnessTeardownMs: 834.7, + }); + expect(cell("Startup").textContent).toBe("0ms"); + expect(cell("Teardown").textContent).toBe("835ms"); + }); + + test("Unaccounted shrinks by exactly startup + teardown", () => { + // 10s − 4s gen − 1s tool = 5s before; minus 3s + 1.5s = 500ms after. + renderStrip({ + taskDurationSeconds: 10, + harnessStartupMs: 3000, + harnessTeardownMs: 1500, + }); + expect(cell("Unaccounted").textContent).toBe("500ms (5%)"); + }); + + test("a corrected residual still above 25% stays red", () => { + // The other direction: naming the buckets must not disable the tint, + // only move the number it reads. 20s − 4s gen − 1s tool − 3s − 1s + // = 11s, still 55% unexplained. + renderStrip({ + taskDurationSeconds: 20, + harnessStartupMs: 3000, + harnessTeardownMs: 1000, + }); + expect(cell("Unaccounted").textContent).toBe("11.0s (55%)"); + expect(cell("Unaccounted").className).toContain("text-red-700"); + }); + + test("a residual that was red goes grey once the buckets are named", () => { + // The 25% threshold applies to the CORRECTED residual: 50% before, + // 5% after, so the red tint must follow the correction. + renderStrip({ + taskDurationSeconds: 10, + harnessStartupMs: 3000, + harnessTeardownMs: 1500, + }); + expect(cell("Unaccounted").className).not.toContain("text-red-700"); + }); + + test("an older run with neither field renders — and today's residual", () => { + const { container } = renderStrip({ taskDurationSeconds: 10 }); + expect(cell("Startup").textContent).toBe("—"); + expect(cell("Teardown").textContent).toBe("—"); + // Byte-identical to the pre-existing Unaccounted expectation. + expect(cell("Unaccounted").textContent).toBe("5.0s (50%)"); + expect(cell("Unaccounted").className).toContain("text-red-700"); + expect(container.textContent).not.toContain("NaN"); + }); + + test("only the present bucket is subtracted", () => { + renderStrip({ taskDurationSeconds: 10, harnessStartupMs: 3000 }); + expect(cell("Startup").textContent).toBe("3.0s"); + expect(cell("Teardown").textContent).toBe("—"); + expect(cell("Unaccounted").textContent).toBe("2.0s (20%)"); + }); + + test("the residual still goes negative and stays amber", () => { + // Naming the buckets does not clamp the overlap signal. + renderStrip({ + taskDurationSeconds: 5, + harnessStartupMs: 1000, + harnessTeardownMs: 500, + }); + expect(cell("Unaccounted").textContent).toBe("-1.5s (-30%)"); + expect(cell("Unaccounted").className).toContain("text-amber-700"); + }); + + test("each bucket says what it measures and that it is not decomposed", () => { + renderStrip({ + taskDurationSeconds: 10, + harnessStartupMs: 3000, + harnessTeardownMs: 1500, + }); + expect(screen.getByText("Startup").parentElement).toHaveAttribute( + "title", + expect.stringContaining("time-to-first-token"), + ); + expect(screen.getByText("Teardown").parentElement).toHaveAttribute( + "title", + expect.stringContaining("teardown"), + ); + }); +}); + // A mixed-kind emission's per-kind split is apportioned by content size, so // the page must say so and must not let the unattributable part distort the // thinking share. diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index eecdf1ade..aae93cf99 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -318,6 +318,8 @@ export function MessageTimelineSection({ subAgentUsageByToolId = {}, impactByIndex, taskDurationSeconds, + harnessStartupMs, + harnessTeardownMs, }: { messages: MessageEvent[]; // Per-Agent-call sub-agent token breakdown (input/output/cache-create/ @@ -332,6 +334,13 @@ export function MessageTimelineSection({ // tool execution do NOT account for. Null/absent on a run predating // duration capture — the cell then renders "—" rather than a fake residual. taskDurationSeconds?: number | null; + // The turn-level head and tail, summed over the task's turns: wall clock + // before the first generation window opened and after the last one closed. + // Turn-scoped, so they cannot be derived from the per-message stream the + // other stats come from. Null/absent on a run predating the capture, and + // the cells then read "—" while Unaccounted keeps exactly its old meaning. + harnessStartupMs?: number | null; + harnessTeardownMs?: number | null; }) { // Token columns can be shown as counts or as their estimated USD value. const [unit, setUnit] = useState("tokens"); @@ -398,11 +407,23 @@ export function MessageTimelineSection({ const attributableGenMs = totalGenMs - mixedMs; const thinkingShare = attributableGenMs > 0 ? thinkingMs / attributableGenMs : 0; - // Wall clock the agent stream does not explain. Negative means generation - // and tool execution overlapped, which is a real signal — never clamped. + // Wall clock the agent stream does not explain, AFTER every named bucket. + // Startup and teardown are subtracted because they are measured intervals, + // not residual — leaving them in reported a harness's CLI boot as + // unexplained time. `?? 0` subtracts only what was actually measured, so an + // older run with neither field keeps exactly its previous number. + // Negative means generation and tool execution overlapped, which is a real + // signal — never clamped. const taskMs = taskDurationSeconds != null ? taskDurationSeconds * 1000 : null; - const unaccountedMs = taskMs != null ? taskMs - totalGenMs - toolExecMs : null; + const unaccountedMs = + taskMs != null + ? taskMs - + totalGenMs - + toolExecMs - + (harnessStartupMs ?? 0) - + (harnessTeardownMs ?? 0) + : null; const unaccountedShare = taskMs != null && taskMs > 0 && unaccountedMs != null ? unaccountedMs / taskMs @@ -419,19 +440,28 @@ export function MessageTimelineSection({

MIXED = multiple block types · red = slow (gen ≥10s, tool ≥5s)

- {/* TWO LEVELS, two rows. The top row's Generation, Tool exec and - Unaccounted sum to the task's wall clock; the bottom row splits - Generation alone and sums to IT. Rendering the split as a - sub-cell of one top-row cell put both sums on one line, where - nothing said which total each part belonged to. */} + {/* TWO LEVELS, two rows. The top row's five time cells — Startup, + Generation, Tool exec, Teardown, Unaccounted — sum to the task's + wall clock; the bottom row splits Generation alone and sums to + IT. Rendering the split as a sub-cell of one top-row cell put + both sums on one line, where nothing said which total each part + belonged to. The time cells are ordered as the turn runs. */}
-
+
Messages
{messageCount}
+
+
+ Startup +
+
+ {fmtMs(harnessStartupMs ?? null)} +
+
Generation @@ -448,7 +478,15 @@ export function MessageTimelineSection({ {fmtMs(toolExecMs)}
-
+
+
+ Teardown +
+
+ {fmtMs(harnessTeardownMs ?? null)} +
+
+
Unaccounted
@@ -771,6 +809,8 @@ export function CostExplorerSection({ tokens, recordedCostUsd, taskDurationSeconds, + harnessStartupMs, + harnessTeardownMs, }: { messages: MessageEvent[]; subAgentUsageByToolId?: Record; @@ -778,6 +818,9 @@ export function CostExplorerSection({ recordedCostUsd: number | null; // Forwarded verbatim to the timeline's Unaccounted cell. taskDurationSeconds?: number | null; + // Forwarded verbatim to the timeline's Startup/Teardown cells. + harnessStartupMs?: number | null; + harnessTeardownMs?: number | null; }) { const [scale, setScale] = useState(1); const [toolScale, setToolScale] = useState(1); @@ -816,6 +859,8 @@ export function CostExplorerSection({ subAgentUsageByToolId={subAgentUsageByToolId} impactByIndex={impactByIndex} taskDurationSeconds={taskDurationSeconds} + harnessStartupMs={harnessStartupMs} + harnessTeardownMs={harnessTeardownMs} /> {model && tokens.total > 0 && (
diff --git a/evalboard/app/runs/[id]/[...task]/page.tsx b/evalboard/app/runs/[id]/[...task]/page.tsx index 12e8b334f..84efa4610 100644 --- a/evalboard/app/runs/[id]/[...task]/page.tsx +++ b/evalboard/app/runs/[id]/[...task]/page.tsx @@ -365,6 +365,8 @@ export default async function TaskPage({ tokens={task.tokens} recordedCostUsd={task.totalCostUsd} taskDurationSeconds={task.durationSeconds} + harnessStartupMs={task.harnessStartupMs} + harnessTeardownMs={task.harnessTeardownMs} /> )} diff --git a/evalboard/lib/__tests__/harnessOverhead.test.ts b/evalboard/lib/__tests__/harnessOverhead.test.ts new file mode 100644 index 000000000..a39fc6ac4 --- /dev/null +++ b/evalboard/lib/__tests__/harnessOverhead.test.ts @@ -0,0 +1,79 @@ +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; + +// End-to-end: the two turn-level timing buckets survive the trip from +// task.json's `iterations` onto TaskDetail. `sumHarnessOverhead` is unit-tested +// in runs.test.ts; what only a read off disk can catch is a misspelled raw key, +// since every TurnEntry field is optional and a typo would just parse as +// absent. Mirrors providerCalls.test.ts's env-stub + fresh-import pattern. +const RUN = "2026-01-01_00-00-00"; +const TASK = "demo-task"; +let tmp: string; + +async function write(rel: string, body: string): Promise { + const abs = path.join(tmp, rel); + await fs.mkdir(path.dirname(abs), { recursive: true }); + await fs.writeFile(abs, body); +} + +async function loadRuns() { + vi.resetModules(); + vi.stubEnv("EVALBOARD_LOCAL_RUNS_DIR", tmp); + return import("../runs"); +} + +async function writeTask(iterations: unknown[]): Promise { + await write( + `${RUN}/run.json`, + JSON.stringify({ + run_id: RUN, + task_results: [{ task_id: TASK, status: "success" }], + }), + ); + await write( + `${RUN}/default/${TASK}/00/task.json`, + JSON.stringify({ final_status: "success", iterations }), + ); +} + +beforeEach(async () => { + tmp = await fs.mkdtemp(path.join(os.tmpdir(), "evalboard-overhead-")); +}); + +afterEach(async () => { + vi.unstubAllEnvs(); + await fs.rm(tmp, { recursive: true, force: true }); +}); + +describe("readTaskDetail: harness startup/teardown", () => { + test("sums both buckets across the task's turns", async () => { + await writeTask([ + { harness_startup_ms: 3047.9, harness_teardown_ms: 33.1 }, + { harness_startup_ms: 120.5, harness_teardown_ms: 4.2 }, + ]); + const { readTaskDetail } = await loadRuns(); + const detail = await readTaskDetail(RUN, TASK); + expect(detail?.harnessStartupMs).toBeCloseTo(3168.4, 3); + expect(detail?.harnessTeardownMs).toBeCloseTo(37.3, 3); + }); + + test("an older run without the fields reports null, not zero", async () => { + await writeTask([{ model_used: "claude-haiku-4-5" }]); + const { readTaskDetail } = await loadRuns(); + const detail = await readTaskDetail(RUN, TASK); + expect(detail?.harnessStartupMs).toBeNull(); + expect(detail?.harnessTeardownMs).toBeNull(); + }); + + test("a measured zero head is preserved as 0", async () => { + // An in-process SDK's first generation window already covers dispatch, + // so 0.0 is its honest answer and must not read as "never measured". + await writeTask([{ harness_startup_ms: 0.0, harness_teardown_ms: 834.7 }]); + const { readTaskDetail } = await loadRuns(); + const detail = await readTaskDetail(RUN, TASK); + expect(detail?.harnessStartupMs).toBe(0); + expect(detail?.harnessTeardownMs).toBeCloseTo(834.7, 3); + }); +}); diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index 0ac7b635b..bef4fa8dd 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -24,6 +24,7 @@ import { parseCriterionResults, type RawTaskResult, sortArtifacts, + sumHarnessOverhead, toTaskRow, visibleTurnsFromRaw, walkArtifacts, @@ -246,6 +247,60 @@ describe("aggregateSubAgentUsage", () => { }); }); +describe("sumHarnessOverhead", () => { + test("sums both buckets across iterations", () => { + expect( + sumHarnessOverhead([ + { harness_startup_ms: 3000, harness_teardown_ms: 800 }, + { harness_startup_ms: 120, harness_teardown_ms: 40 }, + ]), + ).toEqual({ startupMs: 3120, teardownMs: 840 }); + }); + + test("a measured zero is a measurement and still sums", () => { + // An in-process SDK whose first generation window already covers + // dispatch legitimately reports 0.0 — that is a number, not a gap. + expect( + sumHarnessOverhead([{ harness_startup_ms: 0, harness_teardown_ms: 3.5 }]), + ).toEqual({ startupMs: 0, teardownMs: 3.5 }); + }); + + test("is null when EVERY iteration is null — never 0", () => { + // 0 would claim the harness started instantly; null says nobody looked. + expect( + sumHarnessOverhead([ + { harness_startup_ms: null, harness_teardown_ms: null }, + {}, + ]), + ).toEqual({ startupMs: null, teardownMs: null }); + }); + + test("sums the measured iterations and ignores the unmeasured ones", () => { + expect( + sumHarnessOverhead([ + { harness_startup_ms: 500 }, + { harness_teardown_ms: 90 }, + ]), + ).toEqual({ startupMs: 500, teardownMs: 90 }); + }); + + test("is null on an empty turn list", () => { + expect(sumHarnessOverhead([])).toEqual({ + startupMs: null, + teardownMs: null, + }); + }); + + test("a non-finite value is dropped rather than poisoning the sum", () => { + expect( + sumHarnessOverhead([ + { harness_startup_ms: NaN, harness_teardown_ms: 10 }, + { harness_startup_ms: 25 }, + ]), + ).toEqual({ startupMs: 25, teardownMs: 10 }); + }); +}); + describe("isExcludedArtifact", () => { test("hides build artifacts, local state, and secrets", () => { for (const rel of [ diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 3cfaf8944..824316598 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -364,6 +364,12 @@ export interface TaskDetail extends TaskResultSummary { // the Agent row). The cost simulator consumes the values via Object.values(). // Empty for runs/turns with no spawned sub-agents. subAgentUsageByToolId: Record; + // The task's harness head and tail, summed over its turns. `null` when no + // turn measured that end — never 0, which would claim the harness started + // or finished instantly. Subtracted from the timeline's Unaccounted cell so + // that residual is what is left after every named bucket. + harnessStartupMs: number | null; + harnessTeardownMs: number | null; // Per-call ACTUAL cost + cache audit rows, grouped by turn iteration. Only // turns whose `provider_call_costs` list is non-empty appear (LiteLLM/ // open-weight backend; empty on Claude/Bedrock). Rendered as a standalone @@ -394,6 +400,35 @@ export interface SubAgentTotals { cacheRead: number; } +// Sum one optional per-turn measurement across a task's turns. `null` — never +// 0 — when no turn carried the value, because 0 means "measured, and instant" +// while null means nobody measured (the `TurnRecord` fields' own contract, and +// what CE058 guards on the Python side). Non-finite values are dropped rather +// than poisoning the total with NaN. +function sumMeasured(values: (number | null | undefined)[]): number | null { + let total: number | null = null; + for (const v of values) { + if (typeof v !== "number" || !Number.isFinite(v)) continue; + total = (total ?? 0) + v; + } + return total; +} + +// The task's harness head and tail, summed over its turns. The per-turn values +// are measured by `coder_eval/timing.py::decompose_turn`; the summation is +// evalboard-only, and the arithmetic that consumes it — the Unaccounted +// residual in `_sections.tsx` — is the deliberate second implementation that +// helper's docstring names (as `pricing.ts` mirrors `pricing.py`). +export function sumHarnessOverhead(turns: TurnEntry[]): { + startupMs: number | null; + teardownMs: number | null; +} { + return { + startupMs: sumMeasured(turns.map((t) => t.harness_startup_ms)), + teardownMs: sumMeasured(turns.map((t) => t.harness_teardown_ms)), + }; +} + // Group the parsed assistant messages by `parentToolUseId` into a per-sub-agent // token breakdown. A sub-agent's generations all carry the spawning Agent call's // tool_use_id; main-thread messages (parentToolUseId null/undefined) are skipped. @@ -1531,6 +1566,12 @@ export interface TurnEntry { // reconciliation row, which carries no model of its own. model_used?: string | null; token_usage?: TokenUsageEntry | null; + // The turn's head and tail: wall ms before the first generation window + // opened and after the last one closed. Absent on runs predating the + // capture, and null on a turn that produced no assistant message — in both + // cases nobody measured, which is a different fact from a measured 0. + harness_startup_ms?: number | null; + harness_teardown_ms?: number | null; // Per-call actual cost + cache audit rows (LiteLLM/open-weight backend); // empty/absent on Claude/Bedrock. Surfaced as a standalone per-call table. provider_call_costs?: ProviderCallEntryRaw[]; @@ -2522,6 +2563,8 @@ export async function readTaskDetail( const tokens = selectTokenTotals(messages, task?.iterations ?? []); const subAgentUsageByToolId = aggregateSubAgentUsage(messages); + const { startupMs: harnessStartupMs, teardownMs: harnessTeardownMs } = + sumHarnessOverhead(task?.iterations ?? []); const taskDescription = task?.task_config?.resolved?.initial_prompt ?? @@ -2562,6 +2605,8 @@ export async function readTaskDetail( messages, tokens, subAgentUsageByToolId, + harnessStartupMs, + harnessTeardownMs, providerCalls, }; } From 8d370cec415e67b67f6bb7d4a371dd8cb4929d7a Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 10 Sep 2026 23:18:53 -0700 Subject: [PATCH 04/54] =?UTF-8?q?feat(timing):=204/4=20=E2=80=94=20assert?= =?UTF-8?q?=20the=20buckets=20in=20replay,=20and=20record=20what=20they=20?= =?UTF-8?q?contain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend `assert_timing_captured` with the one thing the golden replays can support: a turn that produced an assistant message reports both buckets, and a turn that produced none reports neither. Keyed on that message rather than on `expect_generation_window` — `codex_e_orphan_tool` and `claude_i_in_loop_deadline_break` clear the flag while still having a head and a tail, so the flag would have left them unchecked. No golden regeneration: all 27 dumps already carried both fields and still match. `HARNESS_PARITY.md` gains the rows this change exists to publish — what the FIRST generation window covers per harness, and the measured head and tail — plus the reason the head is deliberately not split into CLI boot vs TTFT, and a Known-divergences note for `TurnStartEvent`'s inconsistent emission point. Live verification (15 runs, 3 turns × 5 harnesses) corrected the identity itself: `Σ tool` books overlapping tool calls twice, and one Pi turn overlapped a Write and a Bash by 18.4 ms, producing exactly an 18.3 ms residual. The tool term is the UNION (`timing.py::busy_ms`), as it already is where a harness subtracts tool time out of a generation window. With all four buckets and the union, every harness reconciles to under 0.012% of wall clock. Co-Authored-By: Claude Opus 5 (1M context) --- docs/agents/HARNESS_PARITY.md | 57 ++++++++- scripts/timing/decompose_run.py | 142 +++++++++++++++++++++++ src/coder_eval/models/telemetry.py | 10 +- src/coder_eval/timing.py | 16 ++- tests/_fixtures/golden_streams/_scrub.py | 33 +++++- tests/test_agent_golden_master.py | 35 +++++- 6 files changed, 282 insertions(+), 11 deletions(-) create mode 100644 scripts/timing/decompose_run.py diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 4b4290328..d0ad4a975 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -25,10 +25,13 @@ wall clock its numbers account for. | Field | claude-code | codex | antigravity | opencode | pi | |---|---|---|---|---|---| | `generation_duration_ms` source | harness clock: previous SDK event → this message | SDK item stamps, minus tool execution inside the window | harness clock: previous flush → this flush, minus tool execution inside the window | harness clock per CLI step, minus tool execution inside the step | harness clock per CLI turn, minus tool execution inside the turn | +| what the **first** window covers | turn start → msg0, so dispatch + TTFT are INSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | turn start → first flush, so dispatch + TTFT are INSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | +| `harness_startup_ms` (turn head) | 0.0 — the window above already covers it | ~3.2 s — CLI boot fused with TTFT | 0.0 — the window above already covers it | ~2.5 s — CLI boot fused with TTFT | ~0.24 s — CLI boot fused with TTFT | +| `harness_teardown_ms` (turn tail) | ~1.4 s | ~12 ms | ~5 ms | ~28 ms | ~13 ms | | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | -| `Σ generation + Σ tool ≈ turn duration` | yes | yes | yes | yes | yes | +| `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes | yes | yes | yes | yes | **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** Four of the five harnesses interleave tool execution into a single generation @@ -39,7 +42,7 @@ opens its window at `step_start` and closes it at `step_finish`, and Pi at `turn_start` / `turn_end`, with every tool call running inside. In all four the span between the recorded bounds legitimately CONTAINS tool time that the model did not spend generating, so all four subtract it — the **union** of the closed tool intervals -clipped to the window (`agents/_timing.py::busy_ms`), never the sum, because +clipped to the window (`coder_eval/timing.py::busy_ms`), never the sum, because tool calls overlap: Antigravity resolves several from one `Step` and backgrounds anything over ten seconds, and Codex spawns collab agents concurrently. Summing them over-subtracts by exactly the overlap and, with enough concurrency, drives @@ -53,6 +56,42 @@ subtraction: it marks the end of the previous SDK event and reads again when the next message arrives, so a tool's execution falls between two windows rather than inside one. +**The head and tail are measured, not normalized.** Generation and tool are +only two of the four buckets. The turn's **head** (turn start → first +generation window) and **tail** (last window → turn end) are booked as +`TurnRecord.harness_startup_ms` / `harness_teardown_ms`, computed once at the +`EventCollector` seam by `coder_eval/timing.py::decompose_turn`. The tool term +is the **union** of the command intervals, for the same reason the subtraction +above is — Pi resolved a `Write` and a `Bash` overlapping by 18.4 ms in one +measured turn, and summing their durations books that overlap twice. With all +four buckets and the union, three live turns per harness reconcile to within +1.3 ms of `duration_seconds` (worst case 0.012% of wall clock; the residual is +clock skew, since head and tail are measured between wall-clock event stamps +while `duration_seconds` is the agent's own monotonic span, and its sign flips +between harnesses). `scripts/timing/decompose_run.py` reproduces the table. The head and tail +figures in the table above are means of three live `tasks/hello_date` turns +per harness and move with CLI cache warmth, so read their ORDER OF +MAGNITUDE, not the digits. + +What the head CONTAINS differs per harness and is deliberately **not** +decomposed, because the divergence is real and unfixable in both directions: + +- On an **in-process SDK** (claude-code, antigravity) the first generation + window starts at turn entry, so dispatch and time-to-first-token are already + inside it and the head reads a measured ~0. Excluding them is not possible — + neither harness stamps a per-message arrival to fall back to, and + `started_at == completed_at` would be the CE059 defect. +- On a **subprocess harness** (codex, opencode, pi) the first window cannot + start before the first event the CLI emits, so the head is one opaque + interval fusing CLI boot, provider resolution, dispatch and TTFT. Measured on + OpenCode: the process spawns in ~3 ms and its first `step_start` lands at + ~3.9 s, with no marker in between. + +So the fields are named for the **interval they measure**, never for what they +contain. Do not rename them `cli_boot_ms` or `ttft_ms` — that would claim a +split nobody performed. A measured `0.0` head is an answer; `None` is what +"never measured" looks like (a turn that produced no assistant message). + **Why Codex leaves `generation_completed_at` as `None`.** It means "when the model finished emitting the `tool_use` block". Codex's stream does not carry that per tool; deriving it from the flush time would be a guess. Note also that @@ -82,7 +121,19 @@ tool call rather than shell commands alone. records `execution_completed_at` while leaving `duration_ms` as `None` (audit P2-1). -Both are deliberately deferred; see `c/time-bugs-audit.md` for the measurements. +- **`TurnStartEvent` is emitted at inconsistent points.** Antigravity and Codex + fire it at turn entry, before the pump; claude-code, OpenCode and Pi fire it + when a generation begins. Nothing in the timing accounting reads it — the + head and tail are measured from the first and last `AssistantMessage` + instead, which is uniform across all five — so this is recorded rather than + fixed. It is NOT a `max_turns` hazard: `EventCollector.visible_turn_count` is + `len(self._commands)`, derived from `ToolEndEvent`, and `_turn_starts` feeds + only `assistant_turn_count` on the no-`AgentEndEvent` fallback path. The real + cost of normalizing it is that the event drives the live renderers, so moving + it changes the turn boundaries users watch during a run. + +All three are deliberately deferred; see `c/time-bugs-audit.md` for the +measurements. ## `max_turns` counts visible turns on Codex and Antigravity diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py new file mode 100644 index 000000000..df60ad4b5 --- /dev/null +++ b/scripts/timing/decompose_run.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Decompose recorded turns into the four wall-clock buckets, per harness. + +Reads `task.json` files, groups their turns by `agent_type`, and prints the +mean generation / tool / startup / teardown against the mean turn duration, +plus the residual as a percentage of wall clock. A healthy harness reconciles +to well under 1%. The tool bucket is the UNION of the command intervals, never +their sum — tool calls overlap, and summing them books the overlap twice. + + uv run python scripts/timing/decompose_run.py runs//default/*/00/task.json + +Not wired into `make`: it needs live runs, not fixtures. NOTE `scripts/` is +outside the Makefile's LINT_PATHS, so this file is neither formatted nor +ruff-checked — keep it small and dependency-free. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from datetime import datetime +from pathlib import Path + +from coder_eval.timing import busy_ms + + +def _parse(stamp: object) -> datetime | None: + if not isinstance(stamp, str): + return None + try: + return datetime.fromisoformat(stamp) + except ValueError: + return None + + +def _tool_ms(turn: dict) -> float: + """Wall ms this turn spent executing tools — the UNION, not the sum. + + The same rule `coder_eval.timing.busy_ms` applies when a harness subtracts + tool time out of a generation window, and it has to be the same rule here + or the identity does not close: Pi resolved a `Write` and a `Bash` that + overlapped by 18.4 ms in one measured turn, and summing their durations + booked that overlap twice, which is precisely the 18.3 ms residual that + found this. A command with no recorded bounds cannot be placed on the + timeline at all, so it contributes nothing rather than being summed in + blind — see docs/agents/HARNESS_PARITY.md's Delegate divergence. + """ + spans = [] + for command in turn.get("commands") or []: + start = _parse(command.get("execution_started_at")) + end = _parse(command.get("execution_completed_at")) + if start is not None and end is not None and end >= start: + spans.append((start, end)) + if not spans: + return 0.0 + return busy_ms(spans, min(s for s, _ in spans), max(e for _, e in spans)) + + +def _turn_buckets(turn: dict) -> tuple[float, float, float, float, float] | None: + """(wall_ms, generation_ms, tool_ms, startup_ms, teardown_ms) for one turn. + + None when the turn was never timed at all — a crash partial with no + generation. A bucket the harness could not measure counts as 0 toward the + sums while the turn still contributes its wall clock, so an unmeasured + bucket shows up as residual rather than silently vanishing. + """ + duration_seconds = turn.get("duration_seconds") + if not isinstance(duration_seconds, (int, float)): + return None + messages = turn.get("messages") or [] + generation_ms = sum( + m.get("generation_duration_ms") or 0.0 + for m in messages + if m.get("role") == "assistant" and isinstance(m.get("generation_duration_ms"), (int, float)) + ) + startup_ms = turn.get("harness_startup_ms") + teardown_ms = turn.get("harness_teardown_ms") + return ( + duration_seconds * 1000.0, + generation_ms, + _tool_ms(turn), + startup_ms if isinstance(startup_ms, (int, float)) else 0.0, + teardown_ms if isinstance(teardown_ms, (int, float)) else 0.0, + ) + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("task_json", nargs="+", type=Path, help="task.json files to decompose") + args = parser.parse_args(argv) + + by_harness: dict[str, list[tuple[float, float, float, float, float]]] = defaultdict(list) + for path in args.task_json: + try: + record = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"skipping {path}: {exc}", file=sys.stderr) + continue + harness = (record.get("environment_info") or {}).get("agent_type") or record.get("agent_type") or "unknown" + for turn in record.get("iterations") or []: + buckets = _turn_buckets(turn) + if buckets is not None: + by_harness[harness].append(buckets) + + if not by_harness: + print("no timed turns found", file=sys.stderr) + return 1 + + header = ( + f"{'harness':<14} {'n':>3} {'wall':>10} {'generation':>11} {'tool':>9} " + f"{'startup':>9} {'teardown':>9} {'residual':>10} {'%':>7} {'worst turn':>11}" + ) + print(header) + print("-" * len(header)) + worst_share = 0.0 + worst_turn = 0.0 + for harness in sorted(by_harness): + turns = by_harness[harness] + n = len(turns) + wall, gen, tool, up, down = (sum(col) / n for col in zip(*turns, strict=True)) + residual = wall - gen - tool - up - down + share = (residual / wall * 100.0) if wall else 0.0 + # The MEAN residual can hide an outlier by cancellation — the sign + # flips between harnesses because head/tail are measured between event + # stamps while duration_seconds is the agent's own monotonic span. So + # report the worst single turn beside it; that is the real bound. + per_turn = max(abs(w - g - t - u - d) for w, g, t, u, d in turns) + worst_share = max(worst_share, abs(share)) + worst_turn = max(worst_turn, per_turn) + print( + f"{harness:<14} {n:>3} {wall:>9.1f}ms {gen:>10.1f}ms {tool:>8.1f}ms " + f"{up:>8.1f}ms {down:>8.1f}ms {residual:>9.3f}ms {share:>6.2f}% {per_turn:>9.3f}ms" + ) + print(f"\nworst mean |residual| = {worst_share:.2f}% of wall clock") + print(f"worst single-turn |residual| = {worst_turn:.3f}ms") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/src/coder_eval/models/telemetry.py b/src/coder_eval/models/telemetry.py index 15a184303..0bef609f9 100644 --- a/src/coder_eval/models/telemetry.py +++ b/src/coder_eval/models/telemetry.py @@ -230,8 +230,14 @@ class AssistantMessage(BaseModel): "generation delivered as a tool result). Equals completed_at - started_at only when " "no tool execution closed inside the window; a harness whose stream interleaves tool " "calls into one generation (Antigravity) subtracts those. The property this field exists " - "to make true — once every harness records a real window — is: " - "sum(generation_duration_ms) + sum(command duration_ms) ~= turn duration_seconds. " + "to make true — once every harness records a real window — is the FOUR-bucket identity: " + "sum(generation_duration_ms) + UNION(command execution intervals) " + "+ TurnRecord.harness_startup_ms + TurnRecord.harness_teardown_ms ~= turn duration_seconds. " + "The tool term is the union and not the sum for the same reason the subtraction above " + "uses one (timing.py::busy_ms): concurrent tool calls otherwise book their overlap twice. " + "The last two are the turn's " + "head and tail, which no message can carry because they are the wall clock OUTSIDE every " + "generation window; without them the identity holds only on a harness with no CLI to boot. " "Per-harness status is in docs/agents/HARNESS_PARITY.md; do not assume it holds " "for a harness that table does not yet claim it for." ), diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index b2950896a..c64bece19 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -52,10 +52,18 @@ def decompose_turn( The turn's two unexplained ends. Between them the windows tile (each harness's generation mark runs to the next) and tool execution is already - subtracted inside them, so head + generation + tool + tail is the whole - turn. Defined once here rather than in five agents, and consumed by - ``EventCollector``, the golden-stream sensor, and - ``scripts/timing/decompose_run.py``. + subtracted inside them, so head + generation + UNION(tool) + tail is the + whole turn — the union and not the sum, because concurrent tool calls + otherwise book their overlap twice (``busy_ms`` above, and measured: one + live Pi turn overlapped a ``Write`` and a ``Bash`` by 18.4 ms). + + ``EventCollector`` is the SOLE caller, and deliberately so: this is the one + place the two values are computed, after which they are persisted on + ``TurnRecord`` and every later consumer READS them rather than recomputing. + The golden-stream sensor asserts on the dumped record, and + ``scripts/timing/decompose_run.py`` reads the stored fields — neither can + call this, because ``task.json`` carries no ``AgentStartEvent`` stamp to + recompute a head from. What the head CONTAINS differs per harness and is deliberately NOT split. On an in-process SDK the first window already covers dispatch and diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index 9450bc8ea..27263a4d0 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -135,6 +135,22 @@ def assert_timing_captured(record: dict[str, Any], *, expect_generation_window: bounds are the same ``ast.Name``; when they are two different names holding the same value it cannot, and this is the check that does. + **Unconditional, and keyed on the messages rather than on the flag.** A + turn's head and tail (``harness_startup_ms`` / ``harness_teardown_ms``) are + set exactly when the turn produced an assistant message, because that is + what the collector measures them against — so both are non-``None`` when + one exists and both are ``None`` when none does. The flag is the wrong key + for this one: ``codex_e_orphan_tool`` streams a generation whose window + subtracts to zero, so it clears the flag while still having a head and a + tail to report. + + PRESENCE is all the fixtures can support, and it is the thing worth + asserting: the replays run in ~0.3 ms of synthetic wall clock, so their + head and tail are microseconds and any bound or ordering check would be + noise. A ``>= 0`` check would be worse than noise — ``decompose_turn`` + clamps with ``max(..., 0.0)``, so it would restate the implementation and + could never fail. + Why a scenario-level floor rather than a per-entry rule: no per-entry form works against the real snapshots. ``claude_d_subagent_terminal`` holds two content-bearing assistant messages of which exactly one is legitimately @@ -154,9 +170,24 @@ def assert_timing_captured(record: dict[str, Any], *, expect_generation_window: "returned was timed, so the record must say when and for how long" ) + assistant = [m for m in record.get("messages") or [] if m.get("role") == "assistant"] + for field in ("harness_startup_ms", "harness_teardown_ms"): + value = record.get(field) + if assistant: + assert value is not None, ( + f"{field} is None on a turn that produced {len(assistant)} assistant message(s): " + "the collector measures the head and tail against the first and last generation, " + "so a turn that generated has both — None here says the bucket was never measured" + ) + else: + assert value is None, ( + f"{field} is {value!r} on a turn that produced NO assistant message: there is no " + "generation window to measure against, and a number here claims a measurement " + "nobody could have taken" + ) + if not expect_generation_window: return - assistant = [m for m in record.get("messages") or [] if m.get("role") == "assistant"] windows = [(m.get("generation_duration_ms"), m.get("started_at"), m.get("completed_at")) for m in assistant] assert any( duration is not None and duration > 0 and started is not None and completed is not None and completed > started diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index 4c54d0a24..54d7e98d5 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -265,8 +265,15 @@ def _record( windows: list[float | None] = (), commands: list[dict[str, Any]] = (), bounds_collapse: bool = False, + overhead: tuple[float | None, float | None] = (0.0, 3.5), ) -> dict[str, Any]: - """A record whose bounds span each window, unless `bounds_collapse`.""" + """A record whose bounds span each window, unless `bounds_collapse`. + + `overhead` is the (head, tail) pair. It defaults to a MEASURED pair — + a 0.0 head is antigravity's real answer — because every record here + carries an assistant message unless a test says otherwise, and the + sensor requires both buckets on such a turn. + """ return { "messages": [ { @@ -278,6 +285,8 @@ def _record( for w in windows ], "commands": list(commands), + "harness_startup_ms": overhead[0], + "harness_teardown_ms": overhead[1], } def test_a_positive_window_passes(self): @@ -362,3 +371,27 @@ def test_an_unresolved_command_is_exempt(self): def test_a_scenario_with_no_commands_is_vacuously_fine(self): assert_timing_captured(self._record(windows=[5.0]), expect_generation_window=True) + + # The turn's head and tail. Presence only — the replays run in ~0.3 ms of + # synthetic wall clock, so any bound check here would be noise. + def test_a_generating_turn_must_report_a_head(self): + with pytest.raises(AssertionError, match="harness_startup_ms is None"): + assert_timing_captured(self._record(windows=[5.0], overhead=(None, 3.5)), expect_generation_window=True) + + def test_a_generating_turn_must_report_a_tail(self): + with pytest.raises(AssertionError, match="harness_teardown_ms is None"): + assert_timing_captured(self._record(windows=[5.0], overhead=(0.0, None)), expect_generation_window=True) + + def test_a_turn_with_no_generation_must_report_neither(self): + # A number here claims a measurement nobody could have taken: the + # collector measures both against the first and last generation. + with pytest.raises(AssertionError, match=r"harness_startup_ms is 0\.0"): + assert_timing_captured(self._record(windows=[], overhead=(0.0, 3.5)), expect_generation_window=False) + assert_timing_captured(self._record(windows=[], overhead=(None, None)), expect_generation_window=False) + + def test_the_buckets_are_checked_even_when_no_window_is_expected(self): + # codex_e_orphan_tool clears the flag (its window subtracts to zero) + # while still having a head and a tail — so the flag is the wrong key + # for this half of the sensor, and the early return must not skip it. + with pytest.raises(AssertionError, match="harness_teardown_ms is None"): + assert_timing_captured(self._record(windows=[None], overhead=(0.0, None)), expect_generation_window=False) From b9eadf5408fcedc900cae6d9b5583274d8528459 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 10 Sep 2026 23:39:56 -0700 Subject: [PATCH 05/54] fix: code review fixes for turn head/tail timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects the final review found, each breaking the invariant the change exists to establish. **A placeholder stamp was read as a window bound.** Codex's rollout rebuild, both its sub-agent recovery builders and Claude's synthesized terminal message all stamp `started_at == completed_at == now()` at APPEND time and declare `generation_duration_ms=None` to say no window was measurable. `_overhead_ms` read those stamps anyway, so a Codex turn rebuilt from its rollout — stamped at turn end — booked the ENTIRE TURN as harness startup. Skip them, the same exemption CE059 already makes for the same reason. **The bounds depended on append order.** Codex appends recovered sub-agent messages after the parent's last flush, so `generations[-1]` is not the last generation. Use min/max instead of the first and last list entries. **The four buckets were not disjoint.** Generation windows are tool-subtracted; the head and tail were not. A tool that escapes every window — Antigravity force-closes an orphan at finalization, inside the tail, and backgrounds anything over ten seconds — was counted both as tool and as head or tail. On the committed `antigravity_d_orphaned_tool` fixture that is a residual of -86% of wall clock. `decompose_turn` now subtracts tool time from both ends via the same `busy_ms` the windows use. Also: reset the terminal event when a new turn starts, so the one collector that outlives a turn (EarlyStopWatcher, across retries) cannot pair this attempt's start with the last attempt's end and publish the clamped inversion as a measured 0.0; stop `decompose_run.py` double-counting a sub-agent's generation against its parent Agent call's interval; and say plainly in HARNESS_PARITY.md that claude-code's and antigravity's `0.0` head is a clamped value rather than a measured interval. One golden dump changes, by two lines: `codex_g_items_rebuild` now honestly reports `null` for both buckets instead of a number derived from a placeholder. Co-Authored-By: Claude Opus 5 (1M context) --- docs/agents/HARNESS_PARITY.md | 19 ++- scripts/timing/decompose_run.py | 12 +- src/coder_eval/streaming/collector.py | 41 +++++- src/coder_eval/timing.py | 34 ++++- tests/_fixtures/golden_streams/_scrub.py | 32 +++-- .../expected/codex_g_items_rebuild.json | 4 +- tests/test_agent_golden_master.py | 19 ++- tests/test_event_collector.py | 125 +++++++++++++++++- 8 files changed, 247 insertions(+), 39 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index d0ad4a975..d34b9921b 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -63,7 +63,12 @@ generation window) and **tail** (last window → turn end) are booked as `EventCollector` seam by `coder_eval/timing.py::decompose_turn`. The tool term is the **union** of the command intervals, for the same reason the subtraction above is — Pi resolved a `Write` and a `Bash` overlapping by 18.4 ms in one -measured turn, and summing their durations books that overlap twice. With all +measured turn, and summing their durations books that overlap twice. The head +and tail exclude tool execution by that same rule and that same helper, which +is what keeps the four buckets disjoint: a tool is not confined to a +generation window (Antigravity force-closes an orphan at finalization, inside +the tail, and backgrounds anything over ten seconds), so a span that escapes +one would otherwise be counted both as tool and as head or tail. With all four buckets and the union, three live turns per harness reconcile to within 1.3 ms of `duration_seconds` (worst case 0.012% of wall clock; the residual is clock skew, since head and tail are measured between wall-clock event stamps @@ -78,9 +83,15 @@ decomposed, because the divergence is real and unfixable in both directions: - On an **in-process SDK** (claude-code, antigravity) the first generation window starts at turn entry, so dispatch and time-to-first-token are already - inside it and the head reads a measured ~0. Excluding them is not possible — - neither harness stamps a per-message arrival to fall back to, and - `started_at == completed_at` would be the CE059 defect. + inside it. Excluding them is not possible — neither harness stamps a + per-message arrival to fall back to, and `started_at == completed_at` would + be the CE059 defect. **Read their `0.0` head as "nothing is left over", not + as a measured interval**: the window actually opens marginally BEFORE the + `AgentStartEvent` stamp (claude-code builds its turn state, then + `_build_claude_query`, and only then emits the event), so the raw figure is + negative and clamps. The practical consequence is that harness setup on + these two is booked as generation, and a regression in it would not show up + in the Startup cell. - On a **subprocess harness** (codex, opencode, pi) the first window cannot start before the first event the CLI emits, so the head is one opaque interval fusing CLI boot, provider resolution, dispatch and TTFT. Measured on diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index df60ad4b5..ba6e3c410 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -69,11 +69,19 @@ def _turn_buckets(turn: dict) -> tuple[float, float, float, float, float] | None duration_seconds = turn.get("duration_seconds") if not isinstance(duration_seconds, (int, float)): return None + # MAIN THREAD ONLY. A sub-agent's generations bubble into the same stream + # tagged with the spawning Agent call's tool_use_id, and that call's own + # interval already spans the sub-agent's entire run. Counting both books the + # sub-agent twice — the evalboard's timeline strip filters on exactly this + # field for exactly this reason (a 120 s Agent call containing 90 s of + # sub-agent generation drove its residual to -57%). messages = turn.get("messages") or [] generation_ms = sum( m.get("generation_duration_ms") or 0.0 for m in messages - if m.get("role") == "assistant" and isinstance(m.get("generation_duration_ms"), (int, float)) + if m.get("role") == "assistant" + and m.get("parent_tool_use_id") is None + and isinstance(m.get("generation_duration_ms"), (int, float)) ) startup_ms = turn.get("harness_startup_ms") teardown_ms = turn.get("harness_teardown_ms") @@ -98,7 +106,7 @@ def main(argv: list[str]) -> int: except (OSError, json.JSONDecodeError) as exc: print(f"skipping {path}: {exc}", file=sys.stderr) continue - harness = (record.get("environment_info") or {}).get("agent_type") or record.get("agent_type") or "unknown" + harness = record.get("agent_type") or "unknown" for turn in record.get("iterations") or []: buckets = _turn_buckets(turn) if buckets is not None: diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 03f794556..2a4e1e173 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -77,6 +77,12 @@ def on_event(self, event: StreamEvent) -> None: self._iteration = event.iteration self._user_input = event.prompt self._agent_start_at = event.timestamp + # A new turn has begun, so the previous turn's terminal event is no + # longer this turn's. Every agent builds a fresh collector per + # communicate(), but EarlyStopWatcher keeps ONE across retries: left + # stale, it would pair this attempt's start with the last attempt's + # end and publish the clamped inversion as a measured 0.0. + self._agent_end = None if event.model: self._model = event.model elif isinstance(event, TurnStartEvent): @@ -112,20 +118,43 @@ def _ordered_commands(self) -> list[CommandTelemetry]: def _overhead_ms(self, messages: list[TranscriptMessage]) -> tuple[float | None, float | None]: """The turn's head and tail — the wall clock the generations do not cover. - Measured against the FIRST and LAST ``AssistantMessage``, not - ``messages[0]`` / ``messages[-1]``: a simulation turn interleaves - ``UserMessage`` entries, and a reconciled turn ends with a + Measured against ``AssistantMessage`` entries only: a simulation turn + interleaves ``UserMessage`` entries, and a reconciled turn ends with a ``ReconciliationMessage`` that carries no timestamps at all, so indexing the raw list would measure the wrong thing or raise. + + Two further restrictions, both of which are the difference between a + measurement and an invention: + + A message whose ``generation_duration_ms`` is ``None`` is SKIPPED. That + field is the codebase's own marker for "no window was measurable here", + and every producer of one stamps ``started_at == completed_at == + datetime.now()`` at *append* time as an admitted placeholder — Codex's + rollout rebuild (``_messages_from_items``), both Codex sub-agent + recovery builders, and Claude's ``_synthesize_subagent_terminal_message``. + Reading those stamps as window bounds turns a placeholder into a + measurement: a Codex turn rebuilt from its rollout stamps every message + at turn END, which would book the entire turn as harness startup. It is + the same exemption CE059 makes for exactly the same reason. + + ``min`` / ``max`` rather than the first and last list entries, because + the list is not ordered by time — Codex appends recovered sub-agent + messages after the parent's last flush. Positional access made the + result depend on append order, which nothing enforces. """ - generations = [m for m in messages if isinstance(m, AssistantMessage)] + generations = [m for m in messages if isinstance(m, AssistantMessage) and m.generation_duration_ms is not None] if not generations: return None, None return decompose_turn( - generations[0].started_at, - generations[-1].completed_at, + min(m.started_at for m in generations), + max(m.completed_at for m in generations), self._agent_start_at, self._agent_end.timestamp if self._agent_end is not None else None, + [ + (c.execution_started_at, c.execution_completed_at) + for c in self._commands.values() + if c.execution_started_at is not None and c.execution_completed_at is not None + ], ) @staticmethod diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index c64bece19..66f4a1a1f 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -1,4 +1,8 @@ -"""Shared timing helpers for agent implementations. +"""Wall-clock arithmetic for a turn, defined once and shared. + +A cycle-free leaf (the ``models/cli_match.py`` rationale): it sits outside +``agents/`` because ``EventCollector`` consumes it, and importing anything +under ``agents/`` pulls in every agent, which imports ``streaming/``. Two harnesses interleave tool execution into a single generation window — Antigravity (the Step for the tool arrives and only a later ``usage_metadata`` @@ -47,6 +51,7 @@ def decompose_turn( last_completed_at: datetime | None, agent_started_at: datetime | None, agent_ended_at: datetime | None, + tool_spans: list[tuple[datetime, datetime]] | None = None, ) -> tuple[float | None, float | None]: """Wall ms before the first generation window opens, and after the last closes. @@ -57,6 +62,17 @@ def decompose_turn( otherwise book their overlap twice (``busy_ms`` above, and measured: one live Pi turn overlapped a ``Write`` and a ``Bash`` by 18.4 ms). + ``tool_spans`` is what keeps those four buckets DISJOINT, and omitting it + is a double-count rather than a lost refinement. A tool is not confined to + a generation window: Antigravity force-closes an orphan at finalization + (``antigravity_agent.py``), which stamps its completion inside the tail, + and it backgrounds anything over ten seconds, which can straddle either + end. Such a span is subtracted out of the windows AND counted in the tool + bucket, so leaving it in the head or tail books it twice — measured on the + committed ``antigravity_d_orphaned_tool`` fixture as a residual of -86% of + wall clock. So the head and tail exclude tool time by the same rule and + the same helper the windows use. + ``EventCollector`` is the SOLE caller, and deliberately so: this is the one place the two values are computed, after which they are persisted on ``TurnRecord`` and every later consumer READS them rather than recomputing. @@ -80,13 +96,19 @@ def decompose_turn( (the two clocks disagreeing) IS a real zero and clamps, because both ends were observed. - NOTE a second implementation of this arithmetic lives in the evalboard's - Unaccounted cell (``_sections.tsx``), as ``pricing.ts`` mirrors - ``pricing.py``. Change one, change the other. + NOTE the four-bucket identity has a second implementation in TypeScript — + the evalboard's Unaccounted cell (``_sections.tsx``) subtracts the same + buckets from the same wall clock, as ``pricing.ts`` mirrors ``pricing.py``. + It does not recompute a head or a tail (it reads the stored fields), so a + change HERE needs a TS change only when it alters what the buckets mean; + adding a fifth bucket means touching that cell and ``sumHarnessOverhead``. """ + spans = tool_spans or [] head = tail = None if first_started_at is not None and agent_started_at is not None: - head = max((first_started_at - agent_started_at).total_seconds() * 1000.0, 0.0) + elapsed = (first_started_at - agent_started_at).total_seconds() * 1000.0 + head = max(elapsed - busy_ms(spans, agent_started_at, first_started_at), 0.0) if last_completed_at is not None and agent_ended_at is not None: - tail = max((agent_ended_at - last_completed_at).total_seconds() * 1000.0, 0.0) + elapsed = (agent_ended_at - last_completed_at).total_seconds() * 1000.0 + tail = max(elapsed - busy_ms(spans, last_completed_at, agent_ended_at), 0.0) return head, tail diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index 27263a4d0..fb8c324e4 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -137,12 +137,18 @@ def assert_timing_captured(record: dict[str, Any], *, expect_generation_window: **Unconditional, and keyed on the messages rather than on the flag.** A turn's head and tail (``harness_startup_ms`` / ``harness_teardown_ms``) are - set exactly when the turn produced an assistant message, because that is - what the collector measures them against — so both are non-``None`` when - one exists and both are ``None`` when none does. The flag is the wrong key - for this one: ``codex_e_orphan_tool`` streams a generation whose window - subtracts to zero, so it clears the flag while still having a head and a - tail to report. + set exactly when the turn produced an assistant message with a MEASURABLE + window, because that is what the collector measures them against — so both + are non-``None`` when one exists and both are ``None`` when none does. + + Both halves of that key are load-bearing. The flag is the wrong one: + ``codex_e_orphan_tool`` streams a generation whose window subtracts to + zero, so it clears the flag while still having a head and a tail to report. + And "any assistant message" is too weak: ``codex_g_items_rebuild`` rebuilds + its transcript from the rollout after the turn ended, with + ``generation_duration_ms=None`` and placeholder ``now()`` bounds, so there + is nothing there to measure an end against and the honest answer is + ``None`` for both. PRESENCE is all the fixtures can support, and it is the thing worth asserting: the replays run in ~0.3 ms of synthetic wall clock, so their @@ -171,18 +177,20 @@ def assert_timing_captured(record: dict[str, Any], *, expect_generation_window: ) assistant = [m for m in record.get("messages") or [] if m.get("role") == "assistant"] + measurable = [m for m in assistant if m.get("generation_duration_ms") is not None] for field in ("harness_startup_ms", "harness_teardown_ms"): value = record.get(field) - if assistant: + if measurable: assert value is not None, ( - f"{field} is None on a turn that produced {len(assistant)} assistant message(s): " - "the collector measures the head and tail against the first and last generation, " - "so a turn that generated has both — None here says the bucket was never measured" + f"{field} is None on a turn carrying {len(measurable)} measurable generation " + "window(s): the collector measures the head and tail against the earliest and " + "latest of those, so a turn that generated has both — None says never measured" ) else: assert value is None, ( - f"{field} is {value!r} on a turn that produced NO assistant message: there is no " - "generation window to measure against, and a number here claims a measurement " + f"{field} is {value!r} on a turn with no measurable generation window " + f"({len(assistant)} assistant message(s), none reporting a duration): there is " + "nothing to measure an end against, and a number here claims a measurement " "nobody could have taken" ) diff --git a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json index 1c88ea853..752850127 100644 --- a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json +++ b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json @@ -5,8 +5,8 @@ "crash_reason": null, "crashed": false, "duration_seconds": "", - "harness_startup_ms": "", - "harness_teardown_ms": "", + "harness_startup_ms": null, + "harness_teardown_ms": null, "iteration": 1, "max_turns_exhausted": false, "messages": [ diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index 54d7e98d5..df1df4121 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -293,8 +293,10 @@ def test_a_positive_window_passes(self): assert_timing_captured(self._record(windows=[12.5]), expect_generation_window=True) def test_a_none_window_raises_when_one_is_expected(self): + # overhead=(None, None) because a turn with no measurable window has no + # head or tail either; this isolates the generation-window assertion. with pytest.raises(AssertionError, match="positive generation window"): - assert_timing_captured(self._record(windows=[None]), expect_generation_window=True) + assert_timing_captured(self._record(windows=[None], overhead=(None, None)), expect_generation_window=True) def test_exactly_zero_raises_too(self): # The Antigravity defect's exact signature: a value that is present, @@ -312,7 +314,7 @@ def test_collapsed_bounds_raise_even_with_a_healthy_duration(self): assert_timing_captured(self._record(windows=[500.0], bounds_collapse=True), expect_generation_window=True) def test_a_none_window_passes_when_none_is_expected(self): - assert_timing_captured(self._record(windows=[None]), expect_generation_window=False) + assert_timing_captured(self._record(windows=[None], overhead=(None, None)), expect_generation_window=False) def test_one_positive_among_several_passes(self): # The FLOOR, not a per-entry rule. claude_d_subagent_terminal holds two @@ -384,14 +386,23 @@ def test_a_generating_turn_must_report_a_tail(self): def test_a_turn_with_no_generation_must_report_neither(self): # A number here claims a measurement nobody could have taken: the - # collector measures both against the first and last generation. + # collector measures both against the messages that report a window. with pytest.raises(AssertionError, match=r"harness_startup_ms is 0\.0"): assert_timing_captured(self._record(windows=[], overhead=(0.0, 3.5)), expect_generation_window=False) assert_timing_captured(self._record(windows=[], overhead=(None, None)), expect_generation_window=False) + def test_an_unmeasurable_window_is_not_something_to_measure_against(self): + # codex_g_items_rebuild's shape: an assistant message exists, but it was + # rebuilt after the turn ended with placeholder now() bounds and says so + # via generation_duration_ms=None. Those stamps are not window bounds, so + # the honest head and tail are None — keying on "any assistant message" + # would have demanded a number derived from a placeholder. + with pytest.raises(AssertionError, match=r"harness_startup_ms is 0\.0"): + assert_timing_captured(self._record(windows=[None], overhead=(0.0, 3.5)), expect_generation_window=False) + def test_the_buckets_are_checked_even_when_no_window_is_expected(self): # codex_e_orphan_tool clears the flag (its window subtracts to zero) # while still having a head and a tail — so the flag is the wrong key # for this half of the sensor, and the early return must not skip it. with pytest.raises(AssertionError, match="harness_teardown_ms is None"): - assert_timing_captured(self._record(windows=[None], overhead=(0.0, None)), expect_generation_window=False) + assert_timing_captured(self._record(windows=[5.0], overhead=(0.0, None)), expect_generation_window=False) diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 324f10fa8..8c1c0b790 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -485,15 +485,39 @@ class TestHarnessOverheadBuckets: """ @staticmethod - def _msg(started: datetime, completed: datetime) -> AssistantMessage: - return AssistantMessage(started_at=started, completed_at=completed, generation_duration_ms=1.0) + def _msg(started: datetime, completed: datetime, *, measurable: bool = True) -> AssistantMessage: + """A generation window. ``measurable=False`` is the placeholder shape + every fabricated-bounds producer writes — a rollout rebuild or a + sub-agent recovery — which stamps one instant on both bounds and says + so with ``generation_duration_ms=None``.""" + return AssistantMessage( + started_at=started, + completed_at=completed, + generation_duration_ms=1.0 if measurable else None, + ) + + @staticmethod + def _tool(started: datetime, completed: datetime, tool_id: str = "t1") -> ToolEndEvent: + return ToolEndEvent( + task_id=TASK_ID, + tool=CommandTelemetry( + tool_id=tool_id, + tool_name="Bash", + timestamp=started, + sequence_number=0, + execution_started_at=started, + execution_completed_at=completed, + result_status="success", + ), + ) - def _record(self, messages, *, start: datetime, end: datetime) -> TurnRecord: + def _record(self, messages, *, start: datetime, end: datetime, tools=()) -> TurnRecord: collector = EventCollector() _feed( collector, [ AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1, timestamp=start), + *tools, AgentEndEvent( task_id=TASK_ID, usage=TokenUsage(output_tokens=1), @@ -559,3 +583,98 @@ def test_a_snapshot_before_the_terminal_event_measures_nothing(self): rec = collector.build_turn_record() assert rec.harness_startup_ms is None assert rec.harness_teardown_ms is None + + def test_a_placeholder_message_does_not_supply_the_bounds(self): + """A Codex turn rebuilt from its rollout stamps every message at turn + END and marks them generation_duration_ms=None. Reading those stamps as + window bounds books the WHOLE TURN as harness startup.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [ + self._msg(t0.replace(second=2), t0.replace(second=5)), + self._msg(t0.replace(second=9), t0.replace(second=9), measurable=False), + ], + start=t0, + end=t0.replace(second=9), + ) + assert rec.harness_startup_ms == pytest.approx(2000.0) + # 9s - 5s, measured off the real window, not off the placeholder's stamp. + assert rec.harness_teardown_ms == pytest.approx(4000.0) + + def test_a_turn_of_only_placeholders_measures_nothing(self): + """codex_g_items_rebuild's shape: an assistant message exists, but + nothing in it was timed, so there is no end to measure against.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [self._msg(t0.replace(second=9), t0.replace(second=9), measurable=False)], + start=t0, + end=t0.replace(second=9), + ) + assert rec.harness_startup_ms is None + assert rec.harness_teardown_ms is None + + def test_the_bounds_do_not_depend_on_append_order(self): + """Codex appends recovered sub-agent messages after the parent's last + flush, so the list is not ordered by time.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [ + self._msg(t0.replace(second=6), t0.replace(second=8)), + self._msg(t0.replace(second=2), t0.replace(second=4)), + ], + start=t0, + end=t0.replace(second=9), + ) + assert rec.harness_startup_ms == pytest.approx(2000.0) + assert rec.harness_teardown_ms == pytest.approx(1000.0) + + def test_a_tool_running_past_the_last_window_is_not_counted_twice(self): + """Antigravity force-closes an orphan at finalization, stamping its + completion inside the tail, and backgrounds anything over ten seconds. + Such a span is already in the tool bucket, so leaving it in the tail + books it twice and drives the residual sharply negative.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [self._msg(t0.replace(second=1), t0.replace(second=4))], + start=t0, + end=t0.replace(second=9), + tools=[self._tool(t0.replace(second=3), t0.replace(second=7))], + ) + # Tail spans 4s->9s = 5s, of which 4s->7s = 3s was the tool still running. + assert rec.harness_teardown_ms == pytest.approx(2000.0) + + def test_a_tool_running_before_the_first_window_is_not_counted_twice(self): + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [self._msg(t0.replace(second=5), t0.replace(second=8))], + start=t0, + end=t0.replace(second=8), + tools=[self._tool(t0.replace(second=1), t0.replace(second=3))], + ) + # Head spans 0s->5s = 5s, of which 1s->3s = 2s was tool execution. + assert rec.harness_startup_ms == pytest.approx(3000.0) + + def test_a_new_turn_clears_the_previous_turn_terminal_event(self): + """EarlyStopWatcher keeps ONE collector across retries. Left stale, the + next attempt's start pairs with the last attempt's end and the clamped + inversion publishes as a measured 0.0.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + collector = EventCollector() + _feed( + collector, + [ + AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1, timestamp=t0), + AgentEndEvent( + task_id=TASK_ID, + usage=TokenUsage(output_tokens=1), + messages=[self._msg(t0.replace(second=1), t0.replace(second=2))], + timestamp=t0.replace(second=3), + crashed=True, + ), + # Retry, a minute later, with no terminal event of its own yet. + AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1, timestamp=t0.replace(minute=1)), + ], + ) + rec = collector.build_turn_record() + assert rec.harness_startup_ms is None + assert rec.harness_teardown_ms is None From bb05482c5c8fc3d9fcfe5133d2066ff7ddee7b3c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 10 Sep 2026 23:40:58 -0700 Subject: [PATCH 06/54] docs(harness): record three guards the head/tail review could not close The first is the valuable one: a golden-corpus assertion of the four-bucket identity would have caught this work's worst defect, and it is blocked only because 5 of 27 fixtures stamp generations on a clock that is not commensurable with their agent events. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 5eef1adce..de431a087 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -551,3 +551,35 @@ divergences, so the deferred-work record is one place. Measurements in candidate — a real bug needing its own change, with a decision about whether legacy records can be distinguished from current ones at all. Caught in: timing-capture final review (gpt-5.6-sol). + +- [ ] **No golden-corpus assertion of the four-bucket identity**, which is what + would have caught the worst defect of the head/tail work (head and tail were + not tool-subtracted, so an orphaned or window-straddling tool was booked + twice — `antigravity_d_orphaned_tool` reconciled at **-86% of wall clock** + and every one of the 72 golden tests passed). The check itself is three + lines in `assert_timing_captured`: `Σ generation + ∪ tool + head + tail` must + not exceed `duration_seconds`. It is blocked because **5 of 27 fixtures stamp + their generations on a clock that is not commensurable with their agent + events** — the codex scenarios hardcode `2027-01-15` while the agent events + are stamped `now()`, giving a head of ~126 days, and `opencode_b` and + `claude_i` are similar. Adding the assertion today means a 5-entry + suppression list, i.e. a guard that is off for the harnesses most likely to + break it. The real fix is to make the fixtures use one clock; then the + invariant costs three lines. Caught in: turn head/tail timing final review. + +- [ ] **No TypeScript counterpart to CE058.** `evalboard/lib/runs.ts` and + `_sections.tsx` carry the same None-vs-0 contract as the Python side, and + `sumMeasured` implements it correctly, but nothing stops the next author + writing `?? 0` where an unmeasured value must stay null. Not a simple lint + rule: the residual arithmetic in `_sections.tsx` uses `?? 0` *correctly* + (subtract only what was measured), so a blanket ban fires on right code and + the rule needs a way to tell "publishing a value" from "consuming one". + Caught in: turn head/tail timing final review. + +- [ ] **`timing.py::decompose_turn` raises an uncaught `TypeError` on a + naive/aware datetime mix**, straight out of `EventCollector.build_turn_record`, + killing the turn. Unreachable today — every stamp in `agents/` and + `streaming/` is a naive `datetime.now()` (verified by grep: zero hits for + `timezone.utc` / `utcnow` / `astimezone`) — but nothing pins that invariant, + so the first agent to record an aware stamp discovers it at runtime. + Caught in: turn head/tail timing final review. From ef53e8e528615d495aa7d59e5dbc537fe6039f33 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Thu, 10 Sep 2026 23:44:32 -0700 Subject: [PATCH 07/54] docs(harness): widen the measured head/tail figures to six turns per harness The post-fix re-verification doubled the sample. Figures move by 5-30% with CLI cache warmth, which is why the table already says to read their order of magnitude. Co-Authored-By: Claude Opus 5 (1M context) --- docs/agents/HARNESS_PARITY.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index d34b9921b..0a82f37c8 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -26,8 +26,8 @@ wall clock its numbers account for. |---|---|---|---|---|---| | `generation_duration_ms` source | harness clock: previous SDK event → this message | SDK item stamps, minus tool execution inside the window | harness clock: previous flush → this flush, minus tool execution inside the window | harness clock per CLI step, minus tool execution inside the step | harness clock per CLI turn, minus tool execution inside the turn | | what the **first** window covers | turn start → msg0, so dispatch + TTFT are INSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | turn start → first flush, so dispatch + TTFT are INSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | -| `harness_startup_ms` (turn head) | 0.0 — the window above already covers it | ~3.2 s — CLI boot fused with TTFT | 0.0 — the window above already covers it | ~2.5 s — CLI boot fused with TTFT | ~0.24 s — CLI boot fused with TTFT | -| `harness_teardown_ms` (turn tail) | ~1.4 s | ~12 ms | ~5 ms | ~28 ms | ~13 ms | +| `harness_startup_ms` (turn head) | 0.0 — the window above already covers it | ~3.1 s — CLI boot fused with TTFT | 0.0 — the window above already covers it | ~2.5 s — CLI boot fused with TTFT | ~0.23 s — CLI boot fused with TTFT | +| `harness_teardown_ms` (turn tail) | ~1.3 s | ~13 ms | ~7 ms | ~26 ms | ~19 ms | | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | @@ -69,14 +69,14 @@ is what keeps the four buckets disjoint: a tool is not confined to a generation window (Antigravity force-closes an orphan at finalization, inside the tail, and backgrounds anything over ten seconds), so a span that escapes one would otherwise be counted both as tool and as head or tail. With all -four buckets and the union, three live turns per harness reconcile to within -1.3 ms of `duration_seconds` (worst case 0.012% of wall clock; the residual is +four buckets and the union, six live turns per harness reconcile to within +1.7 ms of `duration_seconds` (worst case 0.014% of wall clock; the residual is clock skew, since head and tail are measured between wall-clock event stamps while `duration_seconds` is the agent's own monotonic span, and its sign flips between harnesses). `scripts/timing/decompose_run.py` reproduces the table. The head and tail -figures in the table above are means of three live `tasks/hello_date` turns -per harness and move with CLI cache warmth, so read their ORDER OF -MAGNITUDE, not the digits. +figures in the table above are means of six live `tasks/hello_date` turns per +harness and move with CLI cache warmth, so read their ORDER OF MAGNITUDE, not +the digits. What the head CONTAINS differs per harness and is deliberately **not** decomposed, because the divergence is real and unfixable in both directions: From d154d876dde0f1b9aa0271de48d895c16205eb19 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 06:46:28 -0700 Subject: [PATCH 08/54] test(timing): unify the fixture clocks and assert the four-bucket identity MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The golden corpus could not catch a DOUBLE-COUNT, only an absence. That is how the head/tail work shipped a defect where an orphaned tool was booked both in the tool union and in the tail: `antigravity_d_orphaned_tool` reconciled at -86% of its own wall clock while all 72 golden tests passed. Unify the clocks first, because the assertion is meaningless without it. Codex stamped its SDK items at a fixed 2027 epoch and OpenCode a month in the past, while both agents stamp their own lifecycle events with `now()` — so a codex replay recorded a `harness_startup_ms` of ~126 days and no presence-only check could see it. Both catalogues stay declarative with an absolute base; the runners now shift that base onto the replay's own clock, which keeps every derived duration exact (a 250 ms command stays 250 ms) and fixes only the era. No golden dump changes — these stamps are scrubbed. Then assert it: generation + UNION(tool) + head + tail cannot exceed `duration_seconds`, because the four are disjoint. The threshold is relative with an absolute floor, which is what makes it work at fixture scale — the defect reads +55% of wall but only +0.175 ms, so an absolute-only bound generous enough to survive scheduler jitter would have missed it. Mutation-verified: reintroducing the defect fails the antigravity fixture. 22 of 27 scenarios are checked. The other 5 inject SDK stamps in integer MILLISECONDS — 17 to 900 ms of declared item time against a replay that runs in well under one — so no rebasing makes them commensurable and they are exempt via `FICTIONAL_DURATIONS`, named individually with the reason. Closing that last gap needs the agent's own clock faked, not the fixtures' rebased. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 26 +++-- tests/_fixtures/golden_streams/_scrub.py | 79 ++++++++++++++- .../golden_streams/codex_fixtures.py | 50 +++++++++- .../golden_streams/opencode_fixtures.py | 41 +++++++- tests/test_agent_golden_master.py | 98 ++++++++++++++++++- 5 files changed, 270 insertions(+), 24 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index de431a087..7c4e31a97 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -552,20 +552,18 @@ divergences, so the deferred-work record is one place. Measurements in whether legacy records can be distinguished from current ones at all. Caught in: timing-capture final review (gpt-5.6-sol). -- [ ] **No golden-corpus assertion of the four-bucket identity**, which is what - would have caught the worst defect of the head/tail work (head and tail were - not tool-subtracted, so an orphaned or window-straddling tool was booked - twice — `antigravity_d_orphaned_tool` reconciled at **-86% of wall clock** - and every one of the 72 golden tests passed). The check itself is three - lines in `assert_timing_captured`: `Σ generation + ∪ tool + head + tail` must - not exceed `duration_seconds`. It is blocked because **5 of 27 fixtures stamp - their generations on a clock that is not commensurable with their agent - events** — the codex scenarios hardcode `2027-01-15` while the agent events - are stamped `now()`, giving a head of ~126 days, and `opencode_b` and - `claude_i` are similar. Adding the assertion today means a 5-entry - suppression list, i.e. a guard that is off for the harnesses most likely to - break it. The real fix is to make the fixtures use one clock; then the - invariant costs three lines. Caught in: turn head/tail timing final review. +- [x] ~~No golden-corpus assertion of the four-bucket identity.~~ **DONE.** The + fixture clocks were unified (`_rebase_notifications` / `_rebase_lines` shift + codex's 2027 base and opencode's month-old base onto the replay's own clock, + keeping every derived duration exact) and `assert_timing_captured` now + asserts `Σ generation + ∪ tool + head + tail` against `duration_seconds`. + Mutation-verified: reintroducing the defect fails + `test_antigravity_golden[d_orphaned_tool]`, which previously passed. + 22 of 27 scenarios are checked. The remaining 5 are exempt via + `FICTIONAL_DURATIONS` for a reason rebasing cannot fix: they inject SDK + stamps in integer MILLISECONDS (17-900 ms of declared item time) while the + replay runs in well under one, so closing that last gap needs the agent's + own clock faked, not the fixtures' rebased. - [ ] **No TypeScript counterpart to CE058.** `evalboard/lib/runs.ts` and `_sections.tsx` carry the same None-vs-0 contract as the Python side, and diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index fb8c324e4..d397b548c 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -2,8 +2,11 @@ from __future__ import annotations +from datetime import datetime from typing import Any +from coder_eval.timing import busy_ms + SCRUB_PLACEHOLDER = "" @@ -104,7 +107,41 @@ def assert_reconciliation(record: dict[str, Any]) -> None: assert cr_sum == usage["cache_read_input_tokens"], "cache_read bucket does not reconcile" -def assert_timing_captured(record: dict[str, Any], *, expect_generation_window: bool) -> None: +# The four buckets are disjoint by construction, so their sum cannot exceed the +# turn's own wall clock. Flag only an overshoot past BOTH bounds: the relative +# one is what catches the defect (an orphaned tool double-booked into the tail +# read +55% of wall on ``antigravity_d_orphaned_tool``), and the absolute floor +# keeps a replay whose whole turn is 40 microseconds from failing on scheduler +# jitter. Healthy fixtures overshoot by at most 0.003 ms / 2%. +_IDENTITY_FLOOR_MS = 0.1 +_IDENTITY_SHARE = 0.20 + + +def _tool_union_ms(record: dict[str, Any]) -> float: + """Wall ms this turn spent executing tools — the union, never the sum.""" + spans: list[tuple[datetime, datetime]] = [] + for command in record.get("commands") or []: + start = _parse_stamp(command.get("execution_started_at")) + end = _parse_stamp(command.get("execution_completed_at")) + if start is not None and end is not None and end >= start: + spans.append((start, end)) + if not spans: + return 0.0 + return busy_ms(spans, min(s for s, _ in spans), max(e for _, e in spans)) + + +def _parse_stamp(value: Any) -> datetime | None: + if not isinstance(value, str): + return None + try: + return datetime.fromisoformat(value) + except ValueError: + return None + + +def assert_timing_captured( + record: dict[str, Any], *, expect_generation_window: bool, check_identity: bool = True +) -> None: """Assert a TurnRecord dump actually recorded the timing it could measure. Run on the UNSCRUBBED dump. ``scrub()`` masks values but preserves ``None`` @@ -157,6 +194,20 @@ def assert_timing_captured(record: dict[str, Any], *, expect_generation_window: clamps with ``max(..., 0.0)``, so it would restate the implementation and could never fail. + **The four-bucket identity**, when ``check_identity``. Generation plus the + UNION of the tool intervals plus the head plus the tail cannot exceed the + turn's ``duration_seconds``, because the four are disjoint: the windows are + tool-subtracted and so are the head and tail. This is the one assertion + that catches a DOUBLE-COUNT rather than an absence — it is how an orphaned + tool force-closed inside the tail, booked both as tool and as teardown, was + found reconciling at -86% of wall clock while all 72 golden tests passed. + + ``check_identity`` is off for the scenarios that inject their own SDK + timestamps (see ``FICTIONAL_DURATIONS``): those declare integer-millisecond + item durations of 17-900 ms while the replay itself takes ~0.3 ms of real + wall clock, so no rebasing can make the two commensurable — the SDK's + stamps are milliseconds and the replay is faster than one. + Why a scenario-level floor rather than a per-entry rule: no per-entry form works against the real snapshots. ``claude_d_subagent_terminal`` holds two content-bearing assistant messages of which exactly one is legitimately @@ -194,6 +245,32 @@ def assert_timing_captured(record: dict[str, Any], *, expect_generation_window: "nobody could have taken" ) + if check_identity: + wall_ms = (record.get("duration_seconds") or 0.0) * 1000.0 + # Main thread only: a sub-agent's generations bubble into the same + # stream, and the spawning Agent call's own interval already spans them. + generation_ms = sum( + m.get("generation_duration_ms") or 0.0 + for m in record.get("messages") or [] + if m.get("role") == "assistant" and m.get("parent_tool_use_id") is None + ) + tool_ms = _tool_union_ms(record) + bucket_sum = ( + generation_ms + + tool_ms + + (record.get("harness_startup_ms") or 0.0) + + (record.get("harness_teardown_ms") or 0.0) + ) + overshoot = bucket_sum - wall_ms + assert overshoot <= max(_IDENTITY_FLOOR_MS, _IDENTITY_SHARE * wall_ms), ( + f"the four buckets sum to {bucket_sum:.4f} ms against a {wall_ms:.4f} ms turn " + f"(over by {overshoot:.4f} ms): generation={generation_ms:.4f}, tool_union={tool_ms:.4f}, " + f"startup={record.get('harness_startup_ms')!r}, teardown={record.get('harness_teardown_ms')!r}. " + "They are meant to be DISJOINT, so a sum this far over the turn means something is " + "booked twice — most likely a tool that ran outside every generation window and was " + "left in the head or tail as well as in the tool union" + ) + if not expect_generation_window: return windows = [(m.get("generation_duration_ms"), m.get("started_at"), m.get("completed_at")) for m in assistant] diff --git a/tests/_fixtures/golden_streams/codex_fixtures.py b/tests/_fixtures/golden_streams/codex_fixtures.py index 30008af70..a04def75d 100644 --- a/tests/_fixtures/golden_streams/codex_fixtures.py +++ b/tests/_fixtures/golden_streams/codex_fixtures.py @@ -13,6 +13,7 @@ import os from dataclasses import dataclass +from datetime import datetime from pathlib import Path from types import SimpleNamespace from typing import Any @@ -26,12 +27,23 @@ CODEX_MODEL = "gpt-5-codex" +# How far after the replay's start the rebased timeline begins. Small, but +# non-zero so the first generation window opens AFTER the AgentStartEvent and +# the head is a measured interval instead of a clamped inversion. +_REPLAY_LEAD_MS = 2 + # --- Notification factories (mirror test_codex_agent) ----------------------- # Fixed epoch milliseconds, so every derived duration is deterministic and the -# golden snapshots pin a real value rather than a scrubbed clock read. +# golden snapshots pin a real value rather than a scrubbed clock read. It is a +# BASE, not a wall-clock claim: ``_rebase_notifications`` shifts the whole +# timeline onto the replay's own clock before the scenario runs, so the SDK +# stamps and the agent's own event stamps are commensurable. Left absolute, +# a codex replay recorded a ``harness_startup_ms`` of ~126 DAYS — the agent +# events are stamped ``now()`` while these sat in 2027 — which is a number no +# presence-only assertion can catch. _T0_MS = 1_800_000_000_000 @@ -314,6 +326,40 @@ def turn(self, _user_input: str) -> _FakeTurnHandle: return _FakeTurnHandle(self._notifications) +def _rebase_notifications(notifications: list[Any]) -> list[Any]: + """Shift every SDK item stamp from ``_T0_MS`` onto the replay's own clock. + + The scenario catalogue is built once at import with an absolute base, which + keeps every DERIVED duration deterministic (a 250 ms command stays 250 ms). + But the agent stamps its own lifecycle events with ``datetime.now()``, so + left absolute the two clocks are months apart and the recorded head and + tail are nonsense. Rebasing keeps the deltas and fixes the era. + + The offset puts the first item a beat AFTER the replay starts, so the head + is a small positive interval rather than an inversion clamped to 0.0. + """ + offset = int(datetime.now().timestamp() * 1000) - _T0_MS + _REPLAY_LEAD_MS + rebased: list[Any] = [] + for note in notifications: + payload = getattr(note, "payload", None) + started = getattr(payload, "started_at_ms", None) + completed = getattr(payload, "completed_at_ms", None) + if payload is None or (started is None and completed is None): + rebased.append(note) + continue + rebased.append( + SimpleNamespace( + method=note.method, + payload=SimpleNamespace( + item=payload.item, + started_at_ms=None if started is None else started + offset, + completed_at_ms=None if completed is None else completed + offset, + ), + ) + ) + return rebased + + async def run_codex_scenario(scenario: CodexScenario, working_dir: str) -> dict[str, Any]: """Run ``scenario`` with fakes and return the TurnRecord/pending_turn dump.""" import pytest @@ -322,7 +368,7 @@ async def run_codex_scenario(scenario: CodexScenario, working_dir: str) -> dict[ agent = CodexAgent(config) agent.working_directory = Path(working_dir) agent.codex_client = SimpleNamespace(close=lambda: None) - agent.thread = _FakeThread(scenario.notifications) + agent.thread = _FakeThread(_rebase_notifications(scenario.notifications)) # Point CODEX_HOME at a sessions-less dir so sub-agent rollout recovery # short-circuits instead of polling the real ~/.codex. diff --git a/tests/_fixtures/golden_streams/opencode_fixtures.py b/tests/_fixtures/golden_streams/opencode_fixtures.py index f88c6a322..5a86908dd 100644 --- a/tests/_fixtures/golden_streams/opencode_fixtures.py +++ b/tests/_fixtures/golden_streams/opencode_fixtures.py @@ -25,6 +25,7 @@ import json import os from dataclasses import dataclass +from datetime import datetime from typing import Any from unittest.mock import patch @@ -35,13 +36,49 @@ SESSION = "ses_test123" +# Base epoch milliseconds for the recorded stream. A BASE, not a wall-clock +# claim: `_rebase_lines` shifts the whole timeline onto the replay's own clock +# before the scenario runs, so these stamps and the agent's own `datetime.now()` +# event stamps are commensurable. Left absolute they sit a month away from the +# replay, which puts the recorded tool interval outside every measured window. +_T0_MS = 1_786_663_016_802 + +# How far after the replay's start the rebased timeline begins — small, but +# non-zero so the first window opens after the AgentStartEvent. +_REPLAY_LEAD_MS = 2 + + def _evt(event_type: str, part: dict[str, Any]) -> str: """One CLI event line: payload under ``part``, sessionID on the envelope.""" return json.dumps( - {"type": event_type, "timestamp": 1786663016802, "sessionID": SESSION, "part": {"sessionID": SESSION, **part}} + {"type": event_type, "timestamp": _T0_MS, "sessionID": SESSION, "part": {"sessionID": SESSION, **part}} ) +def _rebase_lines(lines: list[str]) -> list[str]: + """Shift every recorded stamp from ``_T0_MS`` onto the replay's own clock. + + Keeps every DERIVED duration exact (a 17 ms tool stays 17 ms) and fixes + only the era, so the head and tail the collector records against the + agent's `datetime.now()` stamps are meaningful rather than a month wide. + """ + offset = int(datetime.now().timestamp() * 1000) - _T0_MS + _REPLAY_LEAD_MS + + def shift(node: Any) -> Any: + if isinstance(node, dict): + return {k: (v + offset if k in _STAMP_KEYS and isinstance(v, int) else shift(v)) for k, v in node.items()} + if isinstance(node, list): + return [shift(v) for v in node] + return node + + return [json.dumps(shift(json.loads(line))) for line in lines] + + +# Millisecond-epoch keys anywhere in an event payload: the envelope's own +# stamp, and a tool's `state.time` bounds. +_STAMP_KEYS = frozenset({"timestamp", "start", "end"}) + + def _tokens(inp: int, out: int, *, write: int = 0, read: int = 0, reasoning: int = 0) -> dict[str, Any]: """Token payload in the NESTED convention (total = input+output+reasoning, cache counted inside `input`); see TestTokenShapeIsObservable for the flat one.""" @@ -175,7 +212,7 @@ class OpenCodeScenario: async def run_opencode_scenario(scenario: OpenCodeScenario, working_dir: str) -> dict[str, Any]: """Replay one scenario and return the resulting record as a plain dump.""" - proc = _FakeProcess(scenario.lines) + proc = _FakeProcess(_rebase_lines(scenario.lines)) async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: proc.stderr = proc # type: ignore[assignment] diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index df1df4121..7655a5daa 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -70,6 +70,29 @@ def _expect_window(harness: str, scenario_name: str) -> bool: return f"{harness}_{scenario_name}" not in NO_GENERATION_WINDOW +# Scenarios that inject their own SDK timestamps, so their recorded durations +# are FICTIONAL and cannot be reconciled against the replay's real wall clock. +# `_rebase_notifications` / `_rebase_lines` put those stamps on the replay's +# clock, which fixes the era — but the SDK's stamps are integer MILLISECONDS +# and these scenarios declare 17-900 ms of item time, while the replay itself +# runs in well under one. No rebasing closes that; the agent's own clock would +# have to be faked too. Everything else — every claude, antigravity and pi +# scenario, and the codex/opencode ones that inject nothing — is checked. +FICTIONAL_DURATIONS: frozenset[str] = frozenset( + { + "codex_b_command_execution", # 250 ms command + 150 ms generation + "codex_d_cross_flush_is_error", # 400 ms command + "codex_e_orphan_tool", # command started, never completed + "codex_f_collab_fallback", # 900 ms collab wait + "opencode_b_tool_call_resolved", # 17 ms tool interval + } +) + + +def _check_identity(harness: str, scenario_name: str) -> bool: + return f"{harness}_{scenario_name}" not in FICTIONAL_DURATIONS + + _EXPECTED_DIR = Path(__file__).parent / "_fixtures" / "golden_streams" / "expected" _REGEN = os.environ.get("GOLDEN_REGEN", "").strip().lower() in {"1", "true", "yes", "on"} @@ -101,7 +124,11 @@ async def test_claude_golden(scenario, tmp_path): # Reconciliation is asserted on the UNscrubbed dump (token buckets are never # scrubbed, but cost/timestamps are — assert before masking to be explicit). assert_reconciliation(raw) - assert_timing_captured(raw, expect_generation_window=_expect_window("claude", scenario.name)) + assert_timing_captured( + raw, + expect_generation_window=_expect_window("claude", scenario.name), + check_identity=_check_identity("claude", scenario.name), + ) _compare_or_regen(f"claude_{scenario.name}", scrub(raw)) @@ -111,7 +138,11 @@ async def test_claude_golden(scenario, tmp_path): async def test_codex_golden(scenario, tmp_path): raw = await run_codex_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) - assert_timing_captured(raw, expect_generation_window=_expect_window("codex", scenario.name)) + assert_timing_captured( + raw, + expect_generation_window=_expect_window("codex", scenario.name), + check_identity=_check_identity("codex", scenario.name), + ) _compare_or_regen(f"codex_{scenario.name}", scrub(raw)) @@ -137,7 +168,11 @@ async def test_codex_reconciliation_invariant(scenario, tmp_path): async def test_antigravity_golden(scenario, tmp_path): raw = await run_antigravity_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) - assert_timing_captured(raw, expect_generation_window=_expect_window("antigravity", scenario.name)) + assert_timing_captured( + raw, + expect_generation_window=_expect_window("antigravity", scenario.name), + check_identity=_check_identity("antigravity", scenario.name), + ) _compare_or_regen(f"antigravity_{scenario.name}", scrub(raw)) @@ -153,7 +188,11 @@ async def test_antigravity_reconciliation_invariant(scenario, tmp_path): async def test_opencode_golden(scenario, tmp_path): raw = await run_opencode_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) - assert_timing_captured(raw, expect_generation_window=_expect_window("opencode", scenario.name)) + assert_timing_captured( + raw, + expect_generation_window=_expect_window("opencode", scenario.name), + check_identity=_check_identity("opencode", scenario.name), + ) _compare_or_regen(f"opencode_{scenario.name}", scrub(raw)) @@ -169,7 +208,11 @@ async def test_opencode_reconciliation_invariant(scenario, tmp_path): async def test_pi_golden(scenario, tmp_path): raw = await run_pi_scenario(scenario, str(tmp_path)) assert_reconciliation(raw) - assert_timing_captured(raw, expect_generation_window=_expect_window("pi", scenario.name)) + assert_timing_captured( + raw, + expect_generation_window=_expect_window("pi", scenario.name), + check_identity=_check_identity("pi", scenario.name), + ) _compare_or_regen(f"pi_{scenario.name}", scrub(raw)) @@ -266,6 +309,7 @@ def _record( commands: list[dict[str, Any]] = (), bounds_collapse: bool = False, overhead: tuple[float | None, float | None] = (0.0, 3.5), + duration_seconds: float = 10.0, ) -> dict[str, Any]: """A record whose bounds span each window, unless `bounds_collapse`. @@ -273,8 +317,13 @@ def _record( a 0.0 head is antigravity's real answer — because every record here carries an assistant message unless a test says otherwise, and the sensor requires both buckets on such a turn. + + `duration_seconds` defaults to a turn long enough that the four-bucket + identity is trivially satisfied, so these cases constrain only what + each is about; the identity has its own cases below. """ return { + "duration_seconds": duration_seconds, "messages": [ { "role": "assistant", @@ -400,6 +449,45 @@ def test_an_unmeasurable_window_is_not_something_to_measure_against(self): with pytest.raises(AssertionError, match=r"harness_startup_ms is 0\.0"): assert_timing_captured(self._record(windows=[None], overhead=(0.0, 3.5)), expect_generation_window=False) + # The four-bucket identity: generation + tool union + head + tail cannot + # exceed the turn, because the four are disjoint. + def test_buckets_summing_past_the_turn_raise(self): + # 4s generation + a 3.5ms tail on a 1s turn. + with pytest.raises(AssertionError, match="booked twice"): + assert_timing_captured(self._record(windows=[4000.0], duration_seconds=1.0), expect_generation_window=True) + + def test_a_tool_double_booked_into_the_tail_is_caught(self): + """The exact defect: an orphan force-closed inside the tail, counted + both in the tool union and in harness_teardown_ms.""" + record = self._record( + windows=[40.0], + duration_seconds=0.1, # 100 ms turn + overhead=(0.0, 50.0), + commands=[ + { + "tool_id": "orphan", + "result_status": "success", + "duration_ms": 50.0, + "execution_started_at": "2026-01-01T00:00:00.020000", + "execution_completed_at": "2026-01-01T00:00:00.070000", + } + ], + ) + with pytest.raises(AssertionError, match="booked twice"): + assert_timing_captured(record, expect_generation_window=True) + + def test_the_identity_can_be_waived_for_a_fictional_clock(self): + # codex/opencode scenarios declare integer-millisecond item durations + # that a sub-millisecond replay can never contain. + assert_timing_captured( + self._record(windows=[4000.0], duration_seconds=1.0), + expect_generation_window=True, + check_identity=False, + ) + + def test_buckets_well_inside_the_turn_pass(self): + assert_timing_captured(self._record(windows=[40.0], duration_seconds=1.0), expect_generation_window=True) + def test_the_buckets_are_checked_even_when_no_window_is_expected(self): # codex_e_orphan_tool clears the flag (its window subtracts to zero) # while still having a head and a tail — so the flag is the wrong key From 6ebfccf47090cfbb9d6f1adf4e55d18a20540127 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 06:49:53 -0700 Subject: [PATCH 09/54] test(harness): pin why claude-code's zero head is left as a clamp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The question was whether to emit `AgentStartEvent` before `_build_claude_query`, so the head became a measurement rather than a clamped negative. Measured first: the build is 0.03 ms, and 0.10 ms with four plugin roots — not the hundreds of milliseconds the review hypothesised, because the transport is constructed lazily and plugin resolution is path work. So: no. Moving the emit would not change the number anyway — `last_event_wall`, which becomes the first window's start, is stamped before the build too, so the build sits inside msg0's generation window either way. It would only convert a -0.03 ms clamp into a +0.03 ms measurement, and it would cost the event its `model=effective_model`, which the build resolves and the live renderers display. Surfacing the build cost would need the window re-seeded after it, which is the generation-window seeding change HARNESS_PARITY.md already rules out for an in-process SDK. Both rejections rest on the build being cheap, so guard that rather than leaving it as a claim in a commit message: `TestClaudeHeadIsStructurallyZero` holds it under 50 ms (~300x headroom, best-of-5 so a loaded runner cannot trip it) and its docstring carries the reasoning. The parity doc now states the measured figures instead of implying an unquantified gap. Co-Authored-By: Claude Opus 5 (1M context) --- docs/agents/HARNESS_PARITY.md | 11 ++++-- tests/test_agent_telemetry.py | 73 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 0a82f37c8..dfbc977cb 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -89,9 +89,14 @@ decomposed, because the divergence is real and unfixable in both directions: as a measured interval**: the window actually opens marginally BEFORE the `AgentStartEvent` stamp (claude-code builds its turn state, then `_build_claude_query`, and only then emits the event), so the raw figure is - negative and clamps. The practical consequence is that harness setup on - these two is booked as generation, and a regression in it would not show up - in the Startup cell. + negative and clamps. The setup between those two points is therefore booked + as generation — **measured at 0.03 ms, and 0.10 ms with four plugin roots**, + so it is the sub-millisecond skew the clamp exists for rather than hidden + overhead. Emitting the event earlier would make the `0.0` a measurement + instead of a clamp but would not change it, since the window's start stamp + also precedes the build; only re-seeding the window after the build would + surface that time, and that is the seeding change ruled out above. + `TestClaudeHeadIsStructurallyZero` pins the build cost so this stays true. - On a **subprocess harness** (codex, opencode, pi) the first window cannot start before the first event the CLI emits, so the head is one opaque interval fusing CLI boot, provider resolution, dispatch and TTFT. Measured on diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py index 12c8c1719..646c4c20e 100644 --- a/tests/test_agent_telemetry.py +++ b/tests/test_agent_telemetry.py @@ -1278,3 +1278,76 @@ async def mock_query(prompt, options): assert second.cache_read_tokens == 50 finally: agent_module.query = original_query + + +class TestClaudeHeadIsStructurallyZero: + """Why claude-code's `harness_startup_ms` is 0.0, and why that is left alone. + + `_ClaudeTurnState.__init__` stamps `last_event_wall`, which becomes the + FIRST generation window's `started_at`. `_build_claude_query` runs next, + and only then is `AgentStartEvent` emitted. So the head — agent start to + first window — is a small NEGATIVE that `decompose_turn` clamps to 0.0. + + Two changes were considered and rejected, and this class pins the facts + each rejection rests on, because both are the kind of thing that rots + silently: + + 1. *Emit `AgentStartEvent` before `_build_claude_query`.* It would turn the + clamp into a genuine measurement, but the value stays ~0 either way — + `last_event_wall` is stamped before the build too, so the build sits + inside msg0's window regardless. The cost is real: the event carries + `model=effective_model`, which the build resolves, so moving it means + the live renderers show the configured model rather than the effective + one. Not worth it for a sub-millisecond gain. + + 2. *Re-seed the first window after the build.* That WOULD surface the build + cost, and it is the generation-window seeding change ruled out in + docs/agents/HARNESS_PARITY.md — for an in-process SDK the interval from + turn entry to the first message is msg0's generation. + + Both rejections assume the build is cheap. This test is what keeps that + assumption honest. + """ + + # Measured at 0.03 ms bare and 0.10 ms with four plugin roots. The bound is + # ~300x that: generous enough that a loaded CI box cannot trip it, tight + # enough to catch a regression that would make the reasoning above wrong. + BUDGET_MS = 50.0 + + @staticmethod + def _build_ms(**config_kwargs) -> float: + from pathlib import Path + + from coder_eval.agents.claude_code_agent import ClaudeCodeAgent + + config = parse_agent_config(type=AgentKind.CLAUDE_CODE, model="claude-haiku-4-5-20251001", **config_kwargs) + agent = ClaudeCodeAgent(config) + agent.working_directory = Path(".") + # Best of N: the claim is about the work the call does, not about the + # worst scheduling slice a shared runner happens to hand it. + samples = [] + for _ in range(5): + started = time.perf_counter() + agent._build_claude_query("hi", 60, 10, lambda _line: None) + samples.append((time.perf_counter() - started) * 1000.0) + return min(samples) + + def test_the_query_build_is_cheap_enough_to_leave_inside_msg0(self): + elapsed = self._build_ms() + assert elapsed < self.BUDGET_MS, ( + f"_build_claude_query took {elapsed:.2f} ms, over the {self.BUDGET_MS} ms budget. It runs " + "BETWEEN the first generation window's start stamp and the AgentStartEvent, so this time " + "is booked as model generation and the clamped 0.0 head hides it. At a few hundred " + "microseconds that is the right trade; at this size it is not — revisit the two options " + "in this class's docstring." + ) + + def test_plugin_resolution_does_not_change_that(self, tmp_path): + """The rejected proposal's motivating case was a plugin-heavy task.""" + (tmp_path / "skills").mkdir() + roots = [{"type": "local", "path": str(tmp_path)} for _ in range(4)] + elapsed = self._build_ms(plugins=roots) + assert elapsed < self.BUDGET_MS, ( + f"_build_claude_query with 4 plugin roots took {elapsed:.2f} ms, over the " + f"{self.BUDGET_MS} ms budget — see the sibling test for why that matters." + ) From 67f9f7a978c44252494b4ca15b6aa65f27858c1d Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 07:21:59 -0700 Subject: [PATCH 10/54] docs(harness): claude-code's generation windows are not tool-subtracted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live verification on a task with concurrent tool calls — the earlier runs all used `hello_date`, which has none — found the four-bucket identity failing on claude-code alone, by 482 ms and 340 ms on two ~18-25 s turns. The residual equals the generation/tool overlap to within 1.4 ms on every claude-code turn measured, including the two whose overlap was under a millisecond and which reconciled to within 0.1 ms. Cause is a documented exemption whose premise does not hold: claude-code is the one harness that does not subtract tool time from its generation windows, on the reasoning that a tool's execution falls between two windows. A tool's timer starts at the EMISSION carrying its tool_use block, and one assistant turn spans several emissions, so a later emission's window runs concurrently with a tool already timing. The other four harnesses overlapped by ~2.0-2.3 s on the same task and reconciled to within 1.2 ms, because they subtract it. This predates the head/tail work — generation-vs-tool timing is older — but that work's identity is what made it visible, and the parity table was claiming "yes" for all five. Correct the table and the paragraph, state the measurement, and track the fix as a candidate: applying `busy_ms` here changes a published `generation_duration_ms` on the most-used harness, so it needs its own golden regeneration and live pass rather than a quiet amendment here. Also warn in the new golden identity assertion's failure text, so a future claude-code fixture that trips it is not misdiagnosed as a fresh double-count. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 22 ++++++++++++++++++++++ docs/agents/HARNESS_PARITY.md | 23 ++++++++++++++++++----- tests/_fixtures/golden_streams/_scrub.py | 6 +++++- 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 7c4e31a97..3a508d779 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -581,3 +581,25 @@ divergences, so the deferred-work record is one place. Measurements in `timezone.utc` / `utcnow` / `astimezone`) — but nothing pins that invariant, so the first agent to record an aware stamp discovers it at runtime. Caught in: turn head/tail timing final review. + +- [ ] **`claude-code` does not subtract tool execution from its generation + windows, and the premise for that is measurably wrong.** The other four + harnesses subtract the union (`timing.py::busy_ms`); claude-code is exempted + on the reasoning that it "marks the end of the previous SDK event and reads + again when the next message arrives, so a tool's execution falls between two + windows rather than inside one". But a tool's timer starts at the **emission** + carrying its `tool_use` block, and one assistant turn spans several emissions, + so a later emission's window runs concurrently with a tool already timing. + Measured live on a task with five parallel writes, five reads and two + concurrent `Bash` calls: the generation/tool overlap was **482 ms and 340 ms** + on two ~18-25 s turns, and the four-bucket residual came out at exactly + `-481 ms` / `-339 ms` — the overlap accounts for it to within 1.4 ms. The + other four harnesses overlapped by ~2.0-2.3 s on the same task and reconciled + to within 1.2 ms. Two claude-code turns with <1 ms of overlap reconciled to + within 0.1 ms, so the fault is precisely the missing subtraction. + Fix is to apply `busy_ms` in `on_assistant_message` as the other four do, but + it changes a PUBLISHED `generation_duration_ms` on the most-used harness, so + it needs its own golden regeneration and live pass. NOT introduced by the + head/tail work — generation-vs-tool timing predates it — but that work's + four-bucket identity is what made it visible. + Caught in: post-merge live verification of the head/tail buckets. diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index dfbc977cb..c4a13f537 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -31,7 +31,7 @@ wall clock its numbers account for. | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | -| `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes | yes | yes | yes | yes | +| `Σ generation + ∪ tool + head + tail ≈ turn duration` | within 0.1 ms when nothing overlaps; off by the generation/tool overlap when it does — see below | yes | yes | yes | yes | **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** Four of the five harnesses interleave tool execution into a single generation @@ -51,10 +51,23 @@ the result to a clamped zero. The consequence worth knowing: on an emission that carries *only* a tool call, the whole measured window was that tool running, so the recorded generation time is legitimately `0.0`. That is a measurement, not a placeholder — `None` -is what "never measured" looks like. Only `claude-code` does not need the -subtraction: it marks the end of the previous SDK event and reads again when -the next message arrives, so a tool's execution falls between two windows -rather than inside one. +is what "never measured" looks like. + +`claude-code` is the one harness that does **not** apply the subtraction: it +marks the end of the previous SDK event and reads again when the next message +arrives, on the premise that a tool's execution then falls between two windows +rather than inside one. **Measured, that premise does not always hold.** A tool's +timer starts at the emission carrying its `tool_use` block, and one assistant +turn can span several emissions, so a later emission's window runs concurrently +with a tool already timing. On a task issuing five parallel writes, five reads +and two concurrent `Bash` calls, the overlap was 482 ms and 340 ms on two ~18-25 s +turns — and the four-bucket residual came out at exactly `-481 ms` and `-339 ms`. +On the same task the other four harnesses overlapped by ~2.0-2.3 s and still +reconciled to within 1.2 ms, because they subtract it. Two claude-code turns in +the same batch that happened to overlap by <1 ms reconciled to within 0.1 ms. +Applying `busy_ms` here as the other four do is the obvious fix and is tracked +in `.claude/harness-candidates.md`; it is a change to a published +`generation_duration_ms`, so it needs its own verification pass. **The head and tail are measured, not normalized.** Generation and tool are only two of the four buckets. The turn's **head** (turn start → first diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index d397b548c..e15740538 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -268,7 +268,11 @@ def assert_timing_captured( f"startup={record.get('harness_startup_ms')!r}, teardown={record.get('harness_teardown_ms')!r}. " "They are meant to be DISJOINT, so a sum this far over the turn means something is " "booked twice — most likely a tool that ran outside every generation window and was " - "left in the head or tail as well as in the tool union" + "left in the head or tail as well as in the tool union. The one KNOWN exception is " + "claude-code, which does not subtract tool time from its generation windows, so a " + "trajectory where a tool runs concurrently with a later emission of the same turn " + "overlaps legitimately; see docs/agents/HARNESS_PARITY.md before concluding this is " + "a new bug" ) if not expect_generation_window: From 966fac7b37e8a0a50f70ef55a0e7d5c2b3c1e3ab Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 07:48:02 -0700 Subject: [PATCH 11/54] fix(claude-code): subtract tool execution from the generation windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit claude-code was the one harness that did not, and the reason it was exempt is measurably wrong. The premise was that because it marks the end of the previous SDK event and reads again when the next message arrives, a tool's execution falls BETWEEN two windows. But a tool's timer starts at the EMISSION carrying its `tool_use` block, and one assistant turn spans several emissions, so a later emission's window runs concurrently with a tool already timing. Measured on a task with five parallel writes, five reads and two concurrent `Bash` calls: 482 ms and 340 ms of overlap on two ~18-25 s turns, and the four-bucket residual came out at exactly -481 ms and -339 ms. The other four harnesses overlapped by ~2.0-2.3 s on the same task and still reconciled to within 1.2 ms, because they subtract it. Two claude-code turns in the same batch whose overlap happened to be under a millisecond reconciled to 0.1 ms, which is what isolated the cause to the missing subtraction rather than to anything about the head and tail. The subtraction cannot happen while flushing: a tool issued by an earlier emission is still running when the next window closes, so its interval does not exist yet. `_subtract_tool_time_from_windows` therefore runs once at finalization, when every span is known, and uses the same `busy_ms` union the other four use — the union and not the sum, because these tools overlap each other too. Sub-agent emissions are skipped: their own tools are not in this command list, and the Agent call that spawned them already spans their run. Re-verified live, same task: claude-code 481 ms / 2.691% -> 1.4 ms / 0.006% over four turns that all carried overlapping tool calls, and all five harnesses reconcile (worst 1.7 ms, 0.012%). `generation_duration_ms` now means the same thing on every harness, so the parity table's identity row is "yes" for all five without a caveat. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 11 +++++- docs/agents/HARNESS_PARITY.md | 44 +++++++++++---------- src/coder_eval/agents/claude_code_agent.py | 45 ++++++++++++++++++++++ src/coder_eval/models/telemetry.py | 5 ++- tests/_fixtures/golden_streams/_scrub.py | 8 ++-- 5 files changed, 84 insertions(+), 29 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 3a508d779..883562709 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -582,8 +582,15 @@ divergences, so the deferred-work record is one place. Measurements in so the first agent to record an aware stamp discovers it at runtime. Caught in: turn head/tail timing final review. -- [ ] **`claude-code` does not subtract tool execution from its generation - windows, and the premise for that is measurably wrong.** The other four +- [x] ~~**`claude-code` does not subtract tool execution from its generation + windows.**~~ **FIXED** in `_ClaudeTurnState._subtract_tool_time_from_windows`, + which runs at finalization (it cannot run at flush time — a tool issued by an + earlier emission is still running when the next window closes). Re-measured + on the same task: 481 ms / 2.691% -> **1.4 ms / 0.006%** over four turns that + all carried overlapping tool calls. Original report kept below for the + reasoning. + + ORIGINAL: The other four harnesses subtract the union (`timing.py::busy_ms`); claude-code is exempted on the reasoning that it "marks the end of the previous SDK event and reads again when the next message arrives, so a tool's execution falls between two diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index c4a13f537..ea99835a7 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -31,17 +31,18 @@ wall clock its numbers account for. | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | -| `Σ generation + ∪ tool + head + tail ≈ turn duration` | within 0.1 ms when nothing overlaps; off by the generation/tool overlap when it does — see below | yes | yes | yes | yes | +| `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes | yes | yes | yes | yes | **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** -Four of the five harnesses interleave tool execution into a single generation -window. Antigravity reports a `Step` for the tool and only a later +All five harnesses can have tool execution inside a generation window, and all +five subtract it. Four interleave it structurally: Antigravity reports a `Step` +for the tool and only a later `usage_metadata` `Step` cuts the message; Codex's message window is seeded from the first item's start and extended to the last item's completion; OpenCode opens its window at `step_start` and closes it at `step_finish`, and Pi at -`turn_start` / `turn_end`, with every tool call running inside. In all four the +`turn_start` / `turn_end`, with every tool call running inside. In each the span between the recorded bounds legitimately CONTAINS tool time that the model -did not spend generating, so all four subtract it — the **union** of the closed tool intervals +did not spend generating, so each subtracts it — the **union** of the closed tool intervals clipped to the window (`coder_eval/timing.py::busy_ms`), never the sum, because tool calls overlap: Antigravity resolves several from one `Step` and backgrounds anything over ten seconds, and Codex spawns collab agents concurrently. Summing @@ -53,21 +54,24 @@ the whole measured window was that tool running, so the recorded generation time is legitimately `0.0`. That is a measurement, not a placeholder — `None` is what "never measured" looks like. -`claude-code` is the one harness that does **not** apply the subtraction: it -marks the end of the previous SDK event and reads again when the next message -arrives, on the premise that a tool's execution then falls between two windows -rather than inside one. **Measured, that premise does not always hold.** A tool's -timer starts at the emission carrying its `tool_use` block, and one assistant -turn can span several emissions, so a later emission's window runs concurrently -with a tool already timing. On a task issuing five parallel writes, five reads -and two concurrent `Bash` calls, the overlap was 482 ms and 340 ms on two ~18-25 s -turns — and the four-bucket residual came out at exactly `-481 ms` and `-339 ms`. -On the same task the other four harnesses overlapped by ~2.0-2.3 s and still -reconciled to within 1.2 ms, because they subtract it. Two claude-code turns in -the same batch that happened to overlap by <1 ms reconciled to within 0.1 ms. -Applying `busy_ms` here as the other four do is the obvious fix and is tracked -in `.claude/harness-candidates.md`; it is a change to a published -`generation_duration_ms`, so it needs its own verification pass. +**`claude-code` subtracts at finalization, not as it flushes.** It was once +exempt entirely, on the premise that because it marks the end of the previous +SDK event and reads again when the next message arrives, a tool's execution +falls *between* two windows rather than inside one. Measured, that premise does +not hold: a tool's timer starts at the **emission** carrying its `tool_use` +block, and one assistant turn spans several emissions, so a later emission's +window runs concurrently with a tool already timing. On a task issuing five +parallel writes, five reads and two concurrent `Bash` calls the overlap was +482 ms and 340 ms on two ~18-25 s turns, and the four-bucket residual came out +at exactly `-481 ms` and `-339 ms`; the other four overlapped by ~2.0-2.3 s on +the same task and still reconciled to within 1.2 ms, because they subtract it. + +It cannot subtract while flushing, because a tool issued by an earlier emission +is still running when the next window closes and its interval does not exist +yet. `_ClaudeTurnState._subtract_tool_time_from_windows` therefore runs once at +finalization, when every span is known. After it, the same task reconciles to +**1.4 ms (0.006% of wall)** over four turns that all carried overlapping tool +calls. **The head and tail are measured, not normalized.** Generation and tool are only two of the four buckets. The turn's **head** (turn start → first diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 7175cb7f3..b49305e02 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -76,6 +76,7 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.timing import busy_ms from coder_eval.utils import dump_dataclass, process_plugins @@ -582,6 +583,49 @@ def _finalize_token_usage(self) -> TokenUsage: self._agent._reprice_for_litellm(usage, self.effective_model) return usage + def _subtract_tool_time_from_windows(self, commands: list[CommandTelemetry]) -> None: + """Take tool execution back out of the generation windows it overlapped. + + The other four harnesses do this as they flush, because their stream + interleaves tool calls into one window. claude-code was exempted on the + premise that a tool's execution falls BETWEEN two windows — but a tool's + timer starts at the emission carrying its ``tool_use`` block, and one + assistant turn spans several emissions, so a later emission's window + runs concurrently with a tool already timing. Measured on a task with + two concurrent ``Bash`` calls: 482 ms and 340 ms of a ~18-25 s turn + counted as both generation and tool, which is exactly the amount by + which the four-bucket identity missed. + + Deferred to finalization rather than done in ``on_assistant_message`` + because that is the first point where every span is known: a tool + issued by an earlier emission is still running when the next window + closes, so its interval does not exist yet. + + ``generation_duration_ms`` therefore means the same thing on all five + harnesses — wall time inside the window with no tool running. A window + entirely covered by tool execution legitimately reads ``0.0``; that is + a measurement, and ``None`` remains what "never measured" means. + """ + spans = [ + (c.execution_started_at, c.execution_completed_at) + for c in commands + if c.execution_started_at is not None and c.execution_completed_at is not None + ] + if not spans: + return + for emission in self.sdk_messages: + # A sub-agent's generation is not on this timeline: its own tools + # are not in `commands`, and the Agent call that spawned it already + # spans its whole run. A UserMessage / ReconciliationMessage has no + # window at all. + if not isinstance(emission, AssistantMessageTelemetry): + continue + if emission.generation_duration_ms is None or emission.parent_tool_use_id is not None: + continue + overlap = busy_ms(spans, emission.started_at, emission.completed_at) + if overlap > 0.0: + emission.generation_duration_ms = max(emission.generation_duration_ms - overlap, 0.0) + def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None: """Close orphaned tools + the open turn, emit the terminal AgentEndEvent, and on a crash build the partial TurnRecord. Idempotent.""" @@ -590,6 +634,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso self.finalized = True commands = self._agent._finalize_commands(self.pending_commands, self.messages) + self._subtract_tool_time_from_windows(commands) for cmd in commands: if cmd.tool_id in self.emitted_tool_ends: continue diff --git a/src/coder_eval/models/telemetry.py b/src/coder_eval/models/telemetry.py index 0bef609f9..65a8f5acb 100644 --- a/src/coder_eval/models/telemetry.py +++ b/src/coder_eval/models/telemetry.py @@ -228,8 +228,9 @@ class AssistantMessage(BaseModel): "Model-generation time for this emission, in milliseconds. None when the harness " "surfaced the message with no measurable window (a rollout rebuild, or a sub-agent " "generation delivered as a tool result). Equals completed_at - started_at only when " - "no tool execution closed inside the window; a harness whose stream interleaves tool " - "calls into one generation (Antigravity) subtracts those. The property this field exists " + "no tool execution closed inside the window; every harness subtracts tool time that " + "ran inside one (claude-code does it at finalization, the other four as they flush). " + "The property this field exists " "to make true — once every harness records a real window — is the FOUR-bucket identity: " "sum(generation_duration_ms) + UNION(command execution intervals) " "+ TurnRecord.harness_startup_ms + TurnRecord.harness_teardown_ms ~= turn duration_seconds. " diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index e15740538..9fb4705fd 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -268,11 +268,9 @@ def assert_timing_captured( f"startup={record.get('harness_startup_ms')!r}, teardown={record.get('harness_teardown_ms')!r}. " "They are meant to be DISJOINT, so a sum this far over the turn means something is " "booked twice — most likely a tool that ran outside every generation window and was " - "left in the head or tail as well as in the tool union. The one KNOWN exception is " - "claude-code, which does not subtract tool time from its generation windows, so a " - "trajectory where a tool runs concurrently with a later emission of the same turn " - "overlaps legitimately; see docs/agents/HARNESS_PARITY.md before concluding this is " - "a new bug" + "left in the head or tail as well as in the tool union, or a generation window that " + "kept tool time it should have subtracted (see docs/agents/HARNESS_PARITY.md — all " + "five harnesses subtract, claude-code at finalization rather than as it flushes)" ) if not expect_generation_window: From d28686ce610a61ad8c4bf340bed7bb35a874884a Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 11:04:49 -0700 Subject: [PATCH 12/54] =?UTF-8?q?fix(antigravity):=201/3=20=E2=80=94=20giv?= =?UTF-8?q?e=20every=20generation=20a=20message=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Step stream carries no message id, so every Antigravity `AssistantMessage` was recorded with `message_id: None`. The evalboard groups assistant emissions by that field and falls back to a wall-clock gap threshold when either side lacks one — and PR #164 made this harness's generation windows contiguous, so the gap is now exactly 0 ms and the fallback folds a whole turn's generations into one timeline row. Synthesize the id the way Codex does (`{turn_id}-msg-{gen_index}`), reusing the `_assistant_turns` counter that already counts appended generations, read before its increment so the first id is `-msg-0`. Totals are unaffected: the evalboard sums token buckets across a group, and the turn/generation counts come from `_assistant_turns` Python-side. Only display granularity was lost. The five regenerated goldens are the regression sensor (`message_id` is not scrubbed); the new unit assertion pins the exact id strings, so moving the increment above the append fails loudly instead of silently making the ids 1-based. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/agents/antigravity_agent.py | 4 ++++ .../expected/antigravity_a_single_text_turn.json | 2 +- .../expected/antigravity_b_tool_call_resolved.json | 2 +- .../antigravity_c_thinking_and_tool_same_generation.json | 2 +- .../expected/antigravity_d_orphaned_tool.json | 2 +- .../expected/antigravity_e_multi_generation.json | 6 +++--- tests/test_antigravity_agent.py | 9 ++++++++- 7 files changed, 19 insertions(+), 8 deletions(-) diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index f1a339144..51414eea8 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -1087,6 +1087,10 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: cache_read_tokens=gen.cache_read_input_tokens, reasoning_tokens=reasoning_tokens, model=self.model, + # The Step stream carries no message id, and the evalboard's + # SAME_EMISSION_GAP_MS fallback cannot split this harness's + # contiguous windows — see docs/agents/HARNESS_PARITY.md. + message_id=f"{self.turn_id}-msg-{self._assistant_turns}", ) ) self._assistant_turns += 1 diff --git a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json index fe601ee40..915390ff4 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json @@ -27,7 +27,7 @@ ], "generation_duration_ms": "", "input_tokens": 100, - "message_id": null, + "message_id": "antigravity-1-msg-0", "model": "gemini-3.5-flash", "output_tokens": 20, "parent_tool_use_id": null, diff --git a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json index 3729ef537..bda601266 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json @@ -56,7 +56,7 @@ ], "generation_duration_ms": "", "input_tokens": 120, - "message_id": null, + "message_id": "antigravity-1-msg-0", "model": "gemini-3.5-flash", "output_tokens": 15, "parent_tool_use_id": null, diff --git a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json index f75f40fbf..56138f54b 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json @@ -65,7 +65,7 @@ ], "generation_duration_ms": "", "input_tokens": 200, - "message_id": null, + "message_id": "antigravity-1-msg-0", "model": "gemini-3.5-flash", "output_tokens": 50, "parent_tool_use_id": null, diff --git a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json index ee177ff69..02f130642 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json @@ -47,7 +47,7 @@ ], "generation_duration_ms": "", "input_tokens": 90, - "message_id": null, + "message_id": "antigravity-1-msg-0", "model": "gemini-3.5-flash", "output_tokens": 10, "parent_tool_use_id": null, diff --git a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json index 1a47c81f1..1b69cc78a 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json @@ -27,7 +27,7 @@ ], "generation_duration_ms": "", "input_tokens": 100, - "message_id": null, + "message_id": "antigravity-1-msg-0", "model": "gemini-3.5-flash", "output_tokens": 15, "parent_tool_use_id": null, @@ -54,7 +54,7 @@ ], "generation_duration_ms": "", "input_tokens": 110, - "message_id": null, + "message_id": "antigravity-1-msg-1", "model": "gemini-3.5-flash", "output_tokens": 18, "parent_tool_use_id": null, @@ -81,7 +81,7 @@ ], "generation_duration_ms": "", "input_tokens": 120, - "message_id": null, + "message_id": "antigravity-1-msg-2", "model": "gemini-3.5-flash", "output_tokens": 14, "parent_tool_use_id": null, diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 0924de62a..725138717 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -24,7 +24,7 @@ _to_token_usage, ) from coder_eval.agents.registry import AgentRegistry -from coder_eval.models import AgentKind, AntigravityAgentConfig, parse_agent_config +from coder_eval.models import AgentKind, AntigravityAgentConfig, AssistantMessage, parse_agent_config from coder_eval.plugins import ensure_plugins_loaded from coder_eval.pricing import calculate_cost from tests._fixtures.golden_streams._scrub import assert_reconciliation @@ -335,6 +335,13 @@ async def test_communicate_maps_steps_to_turn_record(): assert sum(m.output_tokens for m in bucketed) == tr.token_usage.output_tokens assert sum(m.cache_creation_tokens for m in bucketed) == tr.token_usage.cache_creation_input_tokens assert sum(m.cache_read_tokens for m in bucketed) == tr.token_usage.cache_read_input_tokens + # Every generation carries its own identity. Filter explicitly: `tr.messages` + # is list[TranscriptMessage] and ReconciliationMessage has no `message_id`, + # so a bare comprehension would raise the moment a residual is booked. + # The literal strings pin the 0-based Codex-parity scheme, which mere + # distinctness (a uuid would pass) does not. + ids = [m.message_id for m in tr.messages if isinstance(m, AssistantMessage)] + assert ids == ["antigravity-1-msg-0", "antigravity-1-msg-1", "antigravity-1-msg-2"] assert agent.pending_turn is None # success path leaves no partial From 0f5d03358242297a84bd0f9d108a9ad5e207111c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 11:10:09 -0700 Subject: [PATCH 13/54] =?UTF-8?q?test(lint):=202/3=20=E2=80=94=20CE060,=20?= =?UTF-8?q?an=20AssistantMessage=20must=20declare=20its=20message=5Fid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Antigravity omitted the kwarg and nothing failed: the field defaulted to None on every message, the evalboard summed the collapsed group so the totals stayed right, and the golden snapshots had ratified the null the day they were written. A snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission — which is why the author-time rule is worth its cost and is the only one of the three sensors that would have failed on the day this shipped. Unlike CE058/CE059 it derives its constructor set from each module's own `coder_eval.models` imports rather than hardcoding the spelling. That closes the blind spot CE058's own docstring concedes: claude_code_agent binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening the other two the same way is recorded in .claude/harness-candidates.md — it changes two shipped rules and needs its own per-rule mutation check. Verified non-vacuous: stripping the Phase 1 kwarg yields exactly one violation, at the site it came from; the clean tree yields zero, with no suppression anywhere. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 20 ++++ CLAUDE.md | 2 +- pyproject.toml | 1 + tests/lint/rules/ce060_message_id_declared.py | 99 +++++++++++++++++++ tests/lint/runner.py | 2 + tests/test_custom_lint.py | 80 +++++++++++++++ 6 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 tests/lint/rules/ce060_message_id_declared.py diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 883562709..45a0fd778 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -610,3 +610,23 @@ divergences, so the deferred-work record is one place. Measurements in head/tail work — generation-vs-tool timing predates it — but that work's four-bucket identity is what made it visible. Caught in: post-merge live verification of the head/tail buckets. + +### Deferred lint-rule widenings + +- [ ] **CE058 and CE059 still match `AssistantMessage` by a hardcoded constructor + NAME LIST** (`_MESSAGE_CONSTRUCTORS`), where CE060 derives the set from each + module's own `coder_eval.models` imports. The weakness is live, not + theoretical: `claude_code_agent.py` binds *only* + `AssistantMessage as AssistantMessageTelemetry` and never the bare name, so + the two shipped rules guard that file's two construction sites purely because + somebody wrote the current alias into a different file's frozenset — rename + the alias and both go silently blind there — and an arbitrary + `AssistantMessage as Msg` is missed outright by both. Adopting CE060's + alias-resolving `check()` pre-pass is about ten lines per rule, but it widens + two SHIPPED rules whose firing sets are load-bearing (CE058's constructor set + is a different, wider one: `CommandTelemetry`, `SlowestCommandInfo`, + `TurnRecord`), so it needs its own mutation check per rule and a re-measured + firing set over all of `src/`, not a drive-by edit. If a fourth same-scope + kwarg rule ever lands, extract `tests/lint/rules/_message_calls.py` at that + point rather than sooner. + Caught in: the CE060 / antigravity `message_id` run. diff --git a/CLAUDE.md b/CLAUDE.md index 616945b80..d5a23e92d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,7 +236,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). +Recent additions, each traceable to a shipped defect: **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right and only granularity was lost, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/pyproject.toml b/pyproject.toml index 3e0964fa4..8669cbc46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -295,6 +295,7 @@ external = [ "CE057", "CE058", "CE059", + "CE060", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/tests/lint/rules/ce060_message_id_declared.py b/tests/lint/rules/ce060_message_id_declared.py new file mode 100644 index 000000000..c28d13749 --- /dev/null +++ b/tests/lint/rules/ce060_message_id_declared.py @@ -0,0 +1,99 @@ +"""CE060: an assistant message must declare its identity. + +``AssistantMessage.message_id`` is what lets a consumer tell two generations +apart. Antigravity simply omitted the kwarg, so the field defaulted to ``None`` +on every message it ever recorded, and the evalboard — which groups assistant +emissions by ``message_id`` and falls back to a wall-clock ``SAME_EMISSION_GAP_MS`` +threshold when either side lacks one — folded a whole turn's generations into a +single timeline row once the harness's windows became contiguous. Nothing +failed: the totals are summed across the group, so only granularity was lost, +and the golden snapshots had ratified the ``null`` the day they were written. +The mechanism lives in ``docs/agents/HARNESS_PARITY.md`` § Timing capture; +it is not restated here. + +Separate id from CE058 and CE059 deliberately: those two are about *timing* +(an unknown duration published as a literal, a window built from one clock +read), this one is about *identity*. One invariant per id is what makes a +``# noqa`` mean one thing. + +WHY IT RESOLVES ALIASES where CE058 and CE059 hardcode constructor names: +CE058's own docstring already concedes that spelling-based matching dies on a +rename, and the weakness is live — ``claude_code_agent.py`` binds *only* +``AssistantMessage as AssistantMessageTelemetry`` and never the bare name, so a +name list guards that file's two construction sites purely because somebody +wrote the current alias into a different rule. CE060 instead derives its +constructor set from each module's own ``coder_eval.models`` imports, which +removes the gap rather than documenting it and catches an arbitrary +``AssistantMessage as Msg`` besides. Widening the other two rules the same way +is recorded in ``.claude/harness-candidates.md``; it is a change to two shipped +rules and needs its own mutation checks. + +BLIND SPOT: the runtime ``None``. The rule requires the kwarg to be *present*, +not non-``None`` when it runs. ``opencode_agent.py`` passes +``str(part.get("messageID") or "") or None`` and ``pi_agent.py`` the same shape +for ``responseId``, and both evaluate to ``None`` whenever the CLI omits the id +— ``pi_a_single_text_turn.json`` records exactly that. No AST rule can see it, +and demanding a statically non-``None`` value would be wrong: passing a +fallback expression *is* deciding what the id is. The sensor for that case is +the golden corpus, and only partially — a snapshot is written from whatever the +code currently does, so it catches a later change, never an initial omission. + +A ``**``-expanded call fires: such a call has not declared the field at the +site. There is no carve-out because no site in ``src/coder_eval/agents/`` uses +``**`` expansion for these constructors; if one is ever added, pass +``message_id=`` explicitly beside it. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule +from tests.lint.violation import Violation + + +_AGENTS_ROOT = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]agents[/\\]") + +_MODELS_MODULE = "coder_eval.models" + + +def _is_none(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value is None + + +class MessageIdDeclared(BaseRule): + id = "CE060" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(_AGENTS_ROOT.search(filepath)) + # Local bindings of coder_eval.models.AssistantMessage in THIS module. + # Built per file in check(): caching it across files would leak one + # module's alias into another's matching. + self._names: set[str] = set() + + def check(self, tree: ast.AST) -> list[Violation]: + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and (node.module or "").startswith(_MODELS_MODULE): + self._names.update(a.asname or a.name for a in node.names if a.name == "AssistantMessage") + return super().check(tree) + + def visit_Call(self, node: ast.Call) -> None: + if self._in_scope: + func = node.func + name = func.id if isinstance(func, ast.Name) else func.attr if isinstance(func, ast.Attribute) else None + if name in self._names: + kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg is not None} + if "message_id" not in kwargs or _is_none(kwargs["message_id"]): + self.violation( + node, + f"{name}(...) leaves 'message_id' undeclared — absent, or an explicit None — " + "so every message it builds shares one empty identity. Pass the field " + "with a real value: the CLI's own " + "id where the stream carries one, else synthesize it the way Codex does " + "(f'{turn_id}-msg-{gen_index}'). Antigravity shipped without it: the " + "evalboard then falls back to its SAME_EMISSION_GAP_MS wall-clock gap to " + "group emissions, and a harness whose generation windows are contiguous " + "has every one of a turn's generations collapse into one row. See " + "docs/agents/HARNESS_PARITY.md.", + ) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 8800c424d..c14f9865d 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -37,6 +37,7 @@ from tests.lint.rules.ce057_sidecar_shim_stdlib_only import SidecarShimStdlibOnly from tests.lint.rules.ce058_no_timing_literal import NoTimingLiteral from tests.lint.rules.ce059_generation_window_is_two_reads import GenerationWindowIsTwoReads +from tests.lint.rules.ce060_message_id_declared import MessageIdDeclared from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -97,6 +98,7 @@ SidecarShimStdlibOnly, NoTimingLiteral, GenerationWindowIsTwoReads, + MessageIdDeclared, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 84a95d532..16f8b6e81 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4446,3 +4446,83 @@ def test_noqa_suppresses(self): path = SRC / "coder_eval/agents/antigravity_agent.py" assert path.is_file(), "the noqa fixture file must exist or this test passes vacuously" assert not [v for v in check_file(path) if v.rule_id == "CE059"] + + +class TestCE060MessageIdDeclared: + """CE060 flags an assistant message built without an identity. + + Every source string carries its own import line: the rule derives its + constructor set from the module's own `coder_eval.models` imports, so a + bare `AssistantMessage(...)` with no import is correctly invisible to it. + """ + + _IMPORT = "from coder_eval.models import AssistantMessage\n" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/agents/antigravity_agent.py"): + import ast + + from tests.lint.rules.ce060_message_id_declared import MessageIdDeclared + + return MessageIdDeclared(filepath).check(ast.parse(src)) + + def test_flags_an_omitted_message_id(self): + assert self._run(self._IMPORT + "m = AssistantMessage(model=model, output_tokens=3)") + + def test_flags_an_explicit_none(self): + # Passing None is a claim that no id exists, which is never true for a + # harness that can synthesize one. + assert self._run(self._IMPORT + "m = AssistantMessage(model=model, message_id=None)") + + def test_flags_the_in_tree_alias_spelling(self): + assert self._run( + "from coder_eval.models import AssistantMessage as AssistantMessageTelemetry\n" + "m = AssistantMessageTelemetry(model=model)" + ) + + def test_flags_an_arbitrary_alias(self): + # The case a hardcoded name list misses entirely — the whole reason + # CE060 resolves aliases instead. + assert self._run("from coder_eval.models import AssistantMessage as Msg\nm = Msg(model=model)") + + def test_flags_the_attribute_spelling(self): + assert self._run(self._IMPORT + "m = models.AssistantMessage(model=model)") + + def test_flags_a_star_expanded_call(self): + # `**fields` has not declared the field at the site. + assert self._run(self._IMPORT + "m = AssistantMessage(**fields)") + + def test_allows_a_literal_id(self): + assert not self._run(self._IMPORT + 'm = AssistantMessage(message_id="x")') + + def test_allows_an_fstring_id(self): + assert not self._run(self._IMPORT + 'm = AssistantMessage(message_id=f"{turn_id}-msg-{i}")') + + def test_allows_a_fallback_expression(self): + # The runtime-None blind spot, exempted deliberately: passing a + # fallback expression IS deciding what the id is. + assert not self._run(self._IMPORT + "m = AssistantMessage(message_id=str(x) or None)") + + def test_allows_a_star_expanded_call_that_also_passes_the_field(self): + assert not self._run(self._IMPORT + "m = AssistantMessage(**fields, message_id=mid)") + + def test_ignores_an_unrelated_constructor(self): + assert not self._run(self._IMPORT + "s = Span(model=model)") + + def test_ignores_a_module_with_no_matching_import(self): + # Nothing is bound, so the rule claims nothing here. A construction + # site has to import the class to reach it. + assert not self._run("m = AssistantMessage(model=model)") + + def test_is_out_of_scope_outside_agents(self): + assert not self._run( + self._IMPORT + "m = AssistantMessage(model=model)", + filepath="src/coder_eval/orchestrator.py", + ) + + def test_the_real_antigravity_flush_declares_its_id(self): + from tests.lint.runner import check_file + + path = SRC / "coder_eval/agents/antigravity_agent.py" + assert path.is_file(), "the fixture file must exist or this test passes vacuously" + assert not [v for v in check_file(path) if v.rule_id == "CE060"] From 876524898e2959a58480575fd47c4563cbaffd3c Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 11:13:22 -0700 Subject: [PATCH 14/54] =?UTF-8?q?docs(harness):=203/3=20=E2=80=94=20messag?= =?UTF-8?q?e=5Fid=20is=20what=20splits=20the=20timeline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the per-harness `message_id` source in the Timing-capture table and give the rationale one home: the evalboard groups assistant emissions by the field and falls back to a wall-clock gap when either side lacks one, which cannot split windows that are contiguous by construction. The source comment and the CE060 docstring point here rather than restating it, and this is the only place the 100 ms numeral is written outside runs.ts. The table row names both synthetic sub-agent forms, since a row titled "message_id source" that omits them reads as wrong the first time somebody greps it. Nothing goes in Known divergences — this is a fix. On the consumer side, tighten the existing message_id-splitting case from a 10 ms to a 0 ms gap so the fixture matches the shape this harness really emits. No second case: runs.ts short-circuits on the two ids before the gap is computed, so 10 ms and 0 ms take the identical branch and a parallel case would test nothing new. Co-Authored-By: Claude Opus 5 (1M context) --- docs/agents/HARNESS_PARITY.md | 19 +++++++++++++++++++ evalboard/lib/__tests__/parseMessages.test.ts | 8 +++++--- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index ea99835a7..5a60ca0a8 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -31,6 +31,7 @@ wall clock its numbers account for. | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | +| `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID` | CLI `responseId`; `None` when absent | | `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes | yes | yes | yes | yes | **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** @@ -141,6 +142,24 @@ generic tool items now carry a duration where they previously carried none, so `avg_command_time_ms` and `total_command_time_ms` for a Codex run describe every tool call rather than shell commands alone. +**`message_id` is what splits the timeline.** The evalboard groups assistant +emissions by `message_id`, and falls back to a wall-clock gap threshold +(`SAME_EMISSION_GAP_MS`, 100 ms, in `evalboard/lib/runs.ts`) when either side +lacks one. Antigravity's `Step` stream carries no message id, so the harness +synthesizes one — and it must, because this harness's generation windows are +*contiguous* by construction: each opens exactly where the previous one closed, +so the gap between two of them is always 0 ms and the fallback would fold a +whole turn's generations into a single row. Nothing about the numbers would +look wrong, because the consumer SUMS a group's token buckets and durations; +what is lost is per-generation thinking / text / tool attribution. CE060 makes +the kwarg mandatory in `src/coder_eval/agents/` for that reason. Note the two +synthetic schemes read differently on purpose: Codex deliberately REPEATS one +id across the sub-messages of a single generation — that is exactly the "the +CLI split one API response" signal the field exists to carry — while +Antigravity's are all distinct, because it emits one message per generation +with every block inside it. Runs recorded before a harness captured the field +still carry `null` and still depend on the gap fallback, which is why it stays. + ### Known divergences - **Delegate (`delegate-sdk`, out of tree)** records `duration_ms` but no diff --git a/evalboard/lib/__tests__/parseMessages.test.ts b/evalboard/lib/__tests__/parseMessages.test.ts index 04580ba15..3bffe27b9 100644 --- a/evalboard/lib/__tests__/parseMessages.test.ts +++ b/evalboard/lib/__tests__/parseMessages.test.ts @@ -228,7 +228,7 @@ describe("parseMessages — message_id collapsing", () => { expect(e.generationMs).toBe(5500); }); - test("splits when message_ids differ even with tight gap", () => { + test("splits differing message_ids across contiguous windows (0ms gap — the Antigravity shape)", () => { const turns: TurnEntry[] = [ { messages: [ @@ -242,9 +242,11 @@ describe("parseMessages — message_id collapsing", () => { }, { role: "assistant", - started_at: "2026-01-01T00:00:01.010Z", // 10ms gap + // Opens exactly where the previous one closed, the + // way Antigravity tiles its generation windows. + started_at: "2026-01-01T00:00:01.000Z", completed_at: "2026-01-01T00:00:02.000Z", - generation_duration_ms: 990, + generation_duration_ms: 1000, message_id: "msg_b", content_blocks: [{ block_type: "text", text: "hi" }], }, From 57257ccc9b85edd302cbf1db4a863c6240497107 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 11:37:56 -0700 Subject: [PATCH 15/54] fix: code review fixes for antigravity-message-id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings, each raised independently by both final reviewers. CE060's rename-safety was half delivered. Deriving the constructor set from the module's imports removes the local-BINDING spelling, but the class's own name was still a string literal here, so renaming the model — the likelier rename, since the alias exists only because two AssistantMessage types collide — would have disarmed the rule exactly as it disarms the name lists CE060 argues against. It now reads `AssistantMessage.__name__`, the way CE056 imports IN_CONTAINER_ENV. The import walk also traded the alias gap for an import-FORM gap that the docstring's "one remaining blind spot" did not mention: only an absolute `from coder_eval.models import ...` bound anything, so a relative import went silently blind for a whole file (and agents/ does use relative imports), as did every module-alias spelling. Both now fire, verified case by case; the attribute spelling is matched on the attribute alone, deliberately, because the module binding it arrives through is the part a class-binding walk cannot see. What remains — a re-export through an intermediate module — is now stated as such. The attribute test was retargeted at the module-alias form, since with a direct import beside it it had been passing for the wrong reason. The prose in all three surfaces claimed "only granularity was lost", which is measurably false: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose cache cascade is quadratic in that count, so a single-shot Antigravity run had every coefficient pinned at zero; the Messages count and the 10 s slow-generation bar were per-turn too. All three move toward the figure they were always meant to report, so this fix corrects them — but a trend compared across it is not comparing like with like, and the docs now say so. Also: the table gave OpenCode's `None` case where the CE060 docstring asserted it, so the two surfaces in one diff disagreed, and the remaining nulls are not legacy-only. Co-Authored-By: Claude Opus 5 (1M context) --- CLAUDE.md | 2 +- docs/agents/HARNESS_PARITY.md | 30 +++++-- tests/lint/rules/ce060_message_id_declared.py | 81 +++++++++++++++---- tests/test_custom_lint.py | 18 ++++- 4 files changed, 106 insertions(+), 25 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d5a23e92d..6bf3bc529 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,7 +236,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right and only granularity was lost, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). +Recent additions, each traceable to a shipped defect: **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. The damage was not confined to the timeline, which is why "only granularity is lost" was the wrong way to describe it: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose prompt-cache cascade is quadratic in that count, so a single-shot Antigravity run had every cascade coefficient pinned at zero; the `Messages` count and the 10 s slow-generation bar were per-turn too. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 5a60ca0a8..121c6d17b 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -31,7 +31,7 @@ wall clock its numbers account for. | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | -| `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID` | CLI `responseId`; `None` when absent | +| `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID`; `None` when absent | CLI `responseId`; `None` when absent | | `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes | yes | yes | yes | yes | **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** @@ -149,16 +149,32 @@ lacks one. Antigravity's `Step` stream carries no message id, so the harness synthesizes one — and it must, because this harness's generation windows are *contiguous* by construction: each opens exactly where the previous one closed, so the gap between two of them is always 0 ms and the fallback would fold a -whole turn's generations into a single row. Nothing about the numbers would -look wrong, because the consumer SUMS a group's token buckets and durations; -what is lost is per-generation thinking / text / tool attribution. CE060 makes -the kwarg mandatory in `src/coder_eval/agents/` for that reason. Note the two -synthetic schemes read differently on purpose: Codex deliberately REPEATS one +whole turn's generations into a single row. CE060 makes the kwarg mandatory in +`src/coder_eval/agents/` for that reason. + +The collapse is a *display* defect, not an accounting one — the consumer SUMS a +group's token buckets and durations, so every total, percentage and cost is +identical either way, as is the reconciliation residual. But it is not +cosmetic, and three displayed figures do move when a turn stops collapsing: +the thinking-cost simulator's per-call cache cascade (`calls` in +`evalboard/lib/thinkingSim.ts` is the number of grouped emissions, and the +cascade is quadratic in it — on a single-shot run it was pinned at one call, +so every coefficient was zero), the `Messages` count and timeline heading, and +the "slow generation" count, whose 10 s bar was being applied to a whole turn's +summed generation time. All three move toward the figure they were always +meant to report, so the fix corrects them rather than breaking them — but a +trend compared across this change is not comparing like with like. + +The two synthetic schemes read differently on purpose: Codex deliberately REPEATS one id across the sub-messages of a single generation — that is exactly the "the CLI split one API response" signal the field exists to carry — while Antigravity's are all distinct, because it emits one message per generation with every block inside it. Runs recorded before a harness captured the field -still carry `null` and still depend on the gap fallback, which is why it stays. +still carry `null` and still depend on the gap fallback, which is why it stays +— and so does a current OpenCode or Pi message whose payload omitted the id, +which is the case CE060 cannot see (it requires the kwarg to be present, not +non-`None` at runtime). OpenCode tiles its windows contiguously too, so it is +the other harness where a missing id can still collapse a turn. ### Known divergences diff --git a/tests/lint/rules/ce060_message_id_declared.py b/tests/lint/rules/ce060_message_id_declared.py index c28d13749..aeb31fdd7 100644 --- a/tests/lint/rules/ce060_message_id_declared.py +++ b/tests/lint/rules/ce060_message_id_declared.py @@ -6,10 +6,12 @@ emissions by ``message_id`` and falls back to a wall-clock ``SAME_EMISSION_GAP_MS`` threshold when either side lacks one — folded a whole turn's generations into a single timeline row once the harness's windows became contiguous. Nothing -failed: the totals are summed across the group, so only granularity was lost, -and the golden snapshots had ratified the ``null`` the day they were written. -The mechanism lives in ``docs/agents/HARNESS_PARITY.md`` § Timing capture; -it is not restated here. +failed: the consumer sums a group, so every total came out right, and the +golden snapshots had ratified the ``null`` the day they were written. It was +not confined to the timeline either — a grouped emission is one API call to the +thinking-cost simulator, so its whole cache cascade was computed from one call +per turn. The mechanism and the blast radius live in +``docs/agents/HARNESS_PARITY.md`` § Timing capture; neither is restated here. Separate id from CE058 and CE059 deliberately: those two are about *timing* (an unknown duration published as a literal, a window built from one clock @@ -28,15 +30,33 @@ is recorded in ``.claude/harness-candidates.md``; it is a change to two shipped rules and needs its own mutation checks. -BLIND SPOT: the runtime ``None``. The rule requires the kwarg to be *present*, -not non-``None`` when it runs. ``opencode_agent.py`` passes +What it removes is the *local binding* spelling, not every rename: the class's +own name still has to be known, so it is taken from the model itself +(``AssistantMessage.__name__``) rather than written here as a string, the way +CE056 imports ``IN_CONTAINER_ENV`` and CE057 derives its target set from +``SIDECAR_MODULES``. Renaming the model therefore moves this rule with it. + +BLIND SPOT 1: the runtime ``None``. The rule requires the kwarg to be +*present*, not non-``None`` when it runs. ``opencode_agent.py`` passes ``str(part.get("messageID") or "") or None`` and ``pi_agent.py`` the same shape -for ``responseId``, and both evaluate to ``None`` whenever the CLI omits the id -— ``pi_a_single_text_turn.json`` records exactly that. No AST rule can see it, -and demanding a statically non-``None`` value would be wrong: passing a -fallback expression *is* deciding what the id is. The sensor for that case is -the golden corpus, and only partially — a snapshot is written from whatever the -code currently does, so it catches a later change, never an initial omission. +for ``responseId``, so either records ``None`` whenever the id is missing from +the payload (`pi_a_single_text_turn.json` is a snapshot of that shape, though +its null comes from a fixture that emits no ``responseId`` rather than from a +live CLI omission). No AST rule can see it, and demanding a statically +non-``None`` value would be wrong: passing a fallback expression *is* deciding +what the id is. The sensor for that case is the golden corpus, and only +partially — a snapshot is written from whatever the code currently does, so it +catches a later change, never an initial omission. + +BLIND SPOT 2: a binding this file cannot resolve. ``check()`` reads one +module's own imports, so it sees the direct forms — absolute or relative +``from ... import AssistantMessage``, under any alias — and the attribute +spelling ``.AssistantMessage(...)``, which is matched on the attribute +alone precisely because the module binding it comes through (``import +coder_eval.models as models``, ``from coder_eval import models``) is the part a +class-binding walk misses. What remains invisible is a re-export through an +intermediate module (``from .sibling import AssistantMessage``): resolving that +means following imports across files, which no rule in this package does. A ``**``-expanded call fires: such a call has not declared the field at the site. There is no carve-out because no site in ``src/coder_eval/agents/`` uses @@ -47,6 +67,7 @@ import ast import re +from coder_eval.models import AssistantMessage from tests.lint.rules.base import BaseRule from tests.lint.violation import Violation @@ -54,6 +75,24 @@ _AGENTS_ROOT = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]agents[/\\]") _MODELS_MODULE = "coder_eval.models" +_MODELS_TAIL = _MODELS_MODULE.rpartition(".")[2] + +# Taken from the model, never spelled here: a rename then moves the rule too. +_CLASS = AssistantMessage.__name__ + + +def _binds_the_model(node: ast.ImportFrom) -> bool: + """True if this `from ... import` reaches `coder_eval.models`. + + A relative import inside `agents/` (`from ..models import ...`) carries only + the tail in `node.module`, so testing the absolute path alone would leave the + rule silently blind for a whole file — and `agents/` does use relative + imports. + """ + module = node.module or "" + if module.startswith(_MODELS_MODULE): + return True + return bool(node.level) and (module == _MODELS_TAIL or module.startswith(f"{_MODELS_TAIL}.")) def _is_none(node: ast.expr | None) -> bool: @@ -73,15 +112,25 @@ def __init__(self, filepath: str) -> None: def check(self, tree: ast.AST) -> list[Violation]: for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom) and (node.module or "").startswith(_MODELS_MODULE): - self._names.update(a.asname or a.name for a in node.names if a.name == "AssistantMessage") + if isinstance(node, ast.ImportFrom) and _binds_the_model(node): + self._names.update(a.asname or a.name for a in node.names if a.name == _CLASS) return super().check(tree) def visit_Call(self, node: ast.Call) -> None: if self._in_scope: func = node.func - name = func.id if isinstance(func, ast.Name) else func.attr if isinstance(func, ast.Attribute) else None - if name in self._names: + # A bare name has to be bound in this module to be ours; the + # attribute spelling is matched on the attribute alone, since the + # module binding it arrives through is what an import walk over one + # file's class bindings cannot see (see BLIND SPOT 2). + name = ( + func.id + if isinstance(func, ast.Name) and func.id in self._names + else func.attr + if isinstance(func, ast.Attribute) and func.attr == _CLASS + else None + ) + if name is not None: kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg is not None} if "message_id" not in kwargs or _is_none(kwargs["message_id"]): self.violation( diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 16f8b6e81..05aff5d58 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4485,9 +4485,25 @@ def test_flags_an_arbitrary_alias(self): # CE060 resolves aliases instead. assert self._run("from coder_eval.models import AssistantMessage as Msg\nm = Msg(model=model)") - def test_flags_the_attribute_spelling(self): + def test_flags_the_module_alias_spelling(self): + # The realistic way to write `models.AssistantMessage(...)`: the class + # itself is never bound, so only the attribute is left to match on. + assert self._run("import coder_eval.models as models\nm = models.AssistantMessage(model=model)") + + def test_flags_the_attribute_spelling_beside_a_direct_import(self): assert self._run(self._IMPORT + "m = models.AssistantMessage(model=model)") + def test_flags_a_relative_import(self): + # `agents/` does use relative imports, and the absolute path test alone + # left the rule silently blind for a whole file. + assert self._run("from ..models import AssistantMessage\nm = AssistantMessage(model=model)") + + def test_keys_on_the_model_name_rather_than_a_literal(self): + from coder_eval.models import AssistantMessage as _Model + from tests.lint.rules import ce060_message_id_declared as rule_mod + + assert _Model.__name__ == rule_mod._CLASS + def test_flags_a_star_expanded_call(self): # `**fields` has not declared the field at the site. assert self._run(self._IMPORT + "m = AssistantMessage(**fields)") From cb7d6f0ed359b0c48131f39c0c433bc6e6712503 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 11:39:23 -0700 Subject: [PATCH 16/54] docs(harness): register the message_id gaps the final review surfaced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three candidates, all deferred with the reason stated rather than the work done: the within-turn-only nature of a synthetic message_id (a negative property over two languages, and the obvious assertion would pass today while catching nothing), the absence of any evalboard test fed by a Python golden (needs a loader and a scrub-aware timestamp story), and the model field's claude-only description (the plan scoped out model changes; no mechanical guard is obvious). A fourth was attempted and dropped: a vitest case asserting that two null-id messages at a 0 ms gap collapse. Its mutation check showed it takes the identical `gap <= SAME_EMISSION_GAP_MS` branch as the existing 50 ms legacy case, so it could not fail for the reason it claimed — which is what the plan's own argument against a parallel case said. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 44 +++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 45a0fd778..2bc1fbbd2 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -630,3 +630,47 @@ divergences, so the deferred-work record is one place. Measurements in kwarg rule ever lands, extract `tests/lint/rules/_message_calls.py` at that point rather than sooner. Caught in: the CE060 / antigravity `message_id` run. + +- [ ] **Nothing pins that `message_id` is only ever a WITHIN-TURN identity.** Ids + repeat across retry attempts of one turn on every synthetic-id harness — + `Agent.discard_pending_turn` rolls the iteration counter back, so a crashed + partial and its retry both emit `-1-msg-0` (antigravity, codex, and + the out-of-tree delegate agent alike). Harmless today, and verified so: the + evalboard declares its grouping list INSIDE the per-turn loop + (`runs.ts:1822`, flushed at `:2217`) and only ever compares adjacent raws, and + no Python consumer reads the field at all. It stops being harmless the moment + anything joins on the id run-wide (a React key across turns, a cost join, a + dedup) — which is a natural thing to reach for once every harness populates + it. No cheap guard exists: the property to assert is "no consumer treats this + as run-unique", which is a negative over two languages, and asserting + within-turn uniqueness instead would pass today and catch nothing. Cheapest + real option is a comment on the model field; the durable one is a run-level + id if a consumer ever needs one. + Caught in: the CE060 / antigravity `message_id` final review. + +- [ ] **No evalboard test is fed by a Python golden snapshot.** The two halves of + a capture fix are pinned by two hand-written fixtures that never meet: the + golden (`tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json`) + pins what the reducer emits, and `evalboard/lib/__tests__/parseMessages.test.ts` + pins what the consumer does with a fixture an author typed from the same + understanding. Nothing feeds a real recorded shape through `parseMessages`, so + a reducer change that makes the TS fixture unrepresentative breaks no test on + either side. Deferred as architectural: it needs a loader, a scrub-aware + timestamp story (the goldens mask exactly the stamps the grouping reads), and + a convention for which snapshots the JS suite owns — well over 30 min, and + wider than any one capture fix. + Caught in: the CE060 / antigravity `message_id` final review. + +- [ ] **`AssistantMessage.message_id`'s field description names one harness of + five** (`models/telemetry.py:283`: "Anthropic API message_id … when the Claude + Code CLI splits one API response"). Five backends now write the field and four + synthesize it, so `docs/agents/HARNESS_PARITY.md`'s new row is the real SSOT + while the model — which this project's DRY principle designates as + authoritative — describes claude-code only. Not fixed here because the plan + scoped out every model change (the field already existed, so touching it would + have put a schema file in a golden-regeneration diff for prose). No mechanical + guard is obvious either: "a field description must not name a single harness + when the union has five writers" needs a writer census per field, which is + CE054-shaped but over a `str` description rather than a key. The cheap version + is to fix the sentence in the next change that touches the model. + Caught in: the CE060 / antigravity `message_id` final review. From 517e376066d4450b8e92ed506df0c296728b9426 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 16:45:09 -0700 Subject: [PATCH 17/54] fix(timing): bracket the head and tail on the main thread only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on #165. `EventCollector._overhead_ms` bracketed the turn's generation span with every `AssistantMessage`, sub-agent emissions included — unlike its two sibling call sites (`codex_agent._token_usage_from_messages` and `scripts/timing/decompose_run.py`), which both filter on `parent_tool_use_id` for the same reason. A sub-agent's generations sit inside the spawning Agent call's own interval, and the identity the head and tail complete sums generation over the main thread ONLY. Letting a sub-agent message bracket the span shrinks the head or the tail by time no bucket then claims; Codex's recovered child messages carry the CHILD's clock, so it can move either end. Mutation-verified: dropping the filter fails both new cases. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/streaming/collector.py | 17 +++++++++- tests/test_event_collector.py | 45 +++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 2a4e1e173..4a08d54e0 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -141,8 +141,23 @@ def _overhead_ms(self, messages: list[TranscriptMessage]) -> tuple[float | None, the list is not ordered by time — Codex appends recovered sub-agent messages after the parent's last flush. Positional access made the result depend on append order, which nothing enforces. + + MAIN THREAD ONLY, the third restriction and the same rule its two + sibling call sites already apply (``codex_agent._token_usage_from_messages`` + and ``scripts/timing/decompose_run.py``). A sub-agent's generations + carry the spawning Agent call's ``parent_tool_use_id``, and the identity + these two values complete sums generation over the main thread ONLY — + the parent tool call's own interval already spans the sub-agent's whole + run. Bracketing the span with a sub-agent message therefore shrinks the + head or the tail by time no other bucket claims, and Codex's recovered + child messages carry the CHILD's clock, so the bracket can move either + way. Excluding them keeps all four buckets measuring one thread. """ - generations = [m for m in messages if isinstance(m, AssistantMessage) and m.generation_duration_ms is not None] + generations = [ + m + for m in messages + if isinstance(m, AssistantMessage) and m.generation_duration_ms is not None and m.parent_tool_use_id is None + ] if not generations: return None, None return decompose_turn( diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 8c1c0b790..f60dd4647 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -496,6 +496,17 @@ def _msg(started: datetime, completed: datetime, *, measurable: bool = True) -> generation_duration_ms=1.0 if measurable else None, ) + @staticmethod + def _subagent_msg(started: datetime, completed: datetime) -> AssistantMessage: + """A sub-agent generation: same shape, tagged with the spawning Agent + call's tool_use_id. Its time is already inside that call's interval.""" + return AssistantMessage( + started_at=started, + completed_at=completed, + generation_duration_ms=1.0, + parent_tool_use_id="toolu_agent", + ) + @staticmethod def _tool(started: datetime, completed: datetime, tool_id: str = "t1") -> ToolEndEvent: return ToolEndEvent( @@ -538,6 +549,40 @@ def test_head_and_tail_are_measured_from_the_agent_event_stamps(self): assert rec.harness_startup_ms == pytest.approx(2000.0) assert rec.harness_teardown_ms == pytest.approx(4000.0) + def test_a_sub_agent_generation_does_not_move_the_bracket(self): + """MAIN THREAD ONLY, the rule the two sibling call sites already apply. + + The identity these buckets complete sums generation over the main + thread only — a sub-agent's run is already inside its parent Agent + call's interval. Letting a sub-agent message bracket the span shrinks + the head or the tail by time no bucket then claims, and Codex's + recovered child messages carry the CHILD's clock, so the bracket can + move either way. + """ + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [ + self._msg(t0.replace(second=2), t0.replace(second=5)), + # Stamps outside the main thread's own span, in both directions. + self._subagent_msg(t0.replace(second=1), t0.replace(second=8)), + ], + start=t0, + end=t0.replace(second=9), + ) + assert rec.harness_startup_ms == pytest.approx(2000.0) + assert rec.harness_teardown_ms == pytest.approx(4000.0) + + def test_a_turn_whose_only_generations_are_sub_agent_reports_no_overhead(self): + """No main-thread window means nothing was measured — None, not 0.0.""" + t0 = datetime(2026, 1, 1, 12, 0, 0) + rec = self._record( + [self._subagent_msg(t0.replace(second=2), t0.replace(second=5))], + start=t0, + end=t0.replace(second=9), + ) + assert rec.harness_startup_ms is None + assert rec.harness_teardown_ms is None + def test_a_turn_with_no_generation_says_so_rather_than_claiming_zero(self): """None means never measured; 0.0 would mean measured-and-instant (CE058).""" t0 = datetime(2026, 1, 1, 12, 0, 0) From 462a204501969fe511db63f0424a8697ed95a4aa Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 17:41:03 -0700 Subject: [PATCH 18/54] docs(timing): every harness subtracts tool time now, not two The module docstring named Antigravity and Codex as the only harnesses that interleave tool execution into a generation window. That stopped being true in the same release: #164 gave OpenCode and Pi tiled windows (so a call open at a boundary runs inside two of them), and this branch gives claude-code tool subtraction. All five now subtract, and all five subtract the union. Also names the TypeScript twin and the corpus that holds the two in step, which the docstring did not mention at all. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/timing.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 66f4a1a1f..ba3b65cdb 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -4,13 +4,21 @@ ``agents/`` because ``EventCollector`` consumes it, and importing anything under ``agents/`` pulls in every agent, which imports ``streaming/``. -Two harnesses interleave tool execution into a single generation window — -Antigravity (the Step for the tool arrives and only a later ``usage_metadata`` -Step cuts the message) and Codex (``_flush_message``'s window is extended to -the last item's ``completed_at_ms``). Both must therefore subtract the tool -time from the window before publishing ``generation_duration_ms``, and both -must subtract the same thing: the UNION of the closed intervals, clipped to -the window. +EVERY harness now subtracts tool execution from its generation windows before +publishing ``generation_duration_ms``, and all of them subtract the same +thing: the UNION of the intervals, clipped to the window. Two interleave a +tool into a single window outright — Antigravity (the Step for the tool +arrives and only a later ``usage_metadata`` Step cuts the message) and Codex +(``_flush_message``'s window is extended to the last item's +``completed_at_ms``). The other three reach the same place from the opposite +direction: their windows tile the turn contiguously, so a call open at a +window boundary runs inside two of them. + +There is a TypeScript twin, ``evalboard/lib/timing.ts::busyMs``, which +subtracts tool time from a task's WALL CLOCK to produce the Unaccounted +residual. It answers the same question about the same ``task.json``, so the +two must agree — neither owns the numbers: ``tests/_fixtures/timing_union_cases.json`` +does, and both suites replay it. """ from datetime import datetime From 712526fbb3f4286f0812f06b311ad644e54f74e1 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 18:33:23 -0700 Subject: [PATCH 19/54] =?UTF-8?q?feat(timing):=201/6=20=E2=80=94=20a=20two?= =?UTF-8?q?-sided=20residual=20gate=20for=20the=20four-bucket=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The only sensor for `Σ generation + ∪ tool + head + tail ≈ duration` is one-sided: `_scrub.py` asserts `overshoot <= ...`, which catches a bucket claiming MORE time than the turn contains and says nothing at all about one claiming less. An unmeasured bucket — the defect the next four phases move numbers to fix — passes every test in the suite today. `--max-residual-pct` gates on `abs(share)` per turn, so both signs count. It skips a turn on the turn's OWN `crashed` flag and head/tail pair, never on the record's `final_status`: the orchestrator preserves a crashed partial across a retry, so a SUCCESS record can hold a crashed turn, and an `execute` corpus finalizes every row as NOT_GRADED, which is not a statement about timing. Both skips are counted independently — short-circuiting left the no-window tally reading 0 on the one corpus that contains it. An empty gateable set exits non-zero when a threshold was asked for. A gate that passes because it measured nothing is the failure this file exists to remove. Report-only on landing: nothing passes the flag. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/timing/decompose_run.py | 145 ++++++++++++++++++++++++++++++-- 1 file changed, 136 insertions(+), 9 deletions(-) diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index ba6e3c410..42a634c52 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -9,15 +9,26 @@ uv run python scripts/timing/decompose_run.py runs//default/*/00/task.json +Pass `--max-residual-pct` to turn the report into a GATE: a non-zero exit when +any single turn's |residual| exceeds that share of its own wall clock. The gate +is deliberately TWO-SIDED, because the only other sensor for this identity is +not. `tests/_fixtures/golden_streams/_scrub.py` asserts `overshoot <= ...`, +which catches a bucket that claims MORE time than the turn contains and says +nothing at all about a bucket that claims less — so an unmeasured bucket, the +exact defect this file exists to find, passes every test in the suite. Gating +on `abs(share)` covers both signs. + Not wired into `make`: it needs live runs, not fixtures. NOTE `scripts/` is outside the Makefile's LINT_PATHS, so this file is neither formatted nor -ruff-checked — keep it small and dependency-free. +ruff-checked — keep it small and dependency-free (stdlib plus the one shared +`busy_ms` import, so the union rule has a single definition). """ from __future__ import annotations import argparse import json +import statistics import sys from collections import defaultdict from datetime import datetime @@ -94,12 +105,52 @@ def _turn_buckets(turn: dict) -> tuple[float, float, float, float, float] | None ) +def _residual_ms(buckets: tuple[float, float, float, float, float]) -> float: + wall, gen, tool, up, down = buckets + return wall - gen - tool - up - down + + +def _never_measured(turn: dict) -> bool: + """True when the collector recorded NO generation window for this turn. + + Both head and tail `None` is how `EventCollector` says a turn had nothing + measurable — `_scrub.py` asserts exactly that pairing. There is nothing to + reconcile against, so a 100% residual here is an artifact of the absence, + not a bucket the harness failed to fill. One of the two set and the other + `None` is the opposite case and is NOT skipped: that IS a real unmeasured + bucket, and `_turn_buckets` counts it as 0.0 so it surfaces as residual. + """ + return turn.get("harness_startup_ms") is None and turn.get("harness_teardown_ms") is None + + def main(argv: list[str]) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("task_json", nargs="+", type=Path, help="task.json files to decompose") + parser.add_argument( + "--max-residual-pct", + type=float, + default=None, + help="fail (exit 1) if any gateable turn's |residual| exceeds this share of its own wall clock", + ) + parser.add_argument( + "--min-turn-ms", + type=float, + default=1000.0, + help="turns shorter than this are excluded from the share columns and the gate (default 1000)", + ) + parser.add_argument( + "--include-crashed", + action="store_true", + help="keep crashed and never-measured turns instead of skipping them", + ) args = parser.parse_args(argv) - by_harness: dict[str, list[tuple[float, float, float, float, float]]] = defaultdict(list) + # (path, turn_index, buckets) rather than bare buckets: a breach that cannot + # name the file it came from is a gate nobody can act on, and a gate nobody + # can act on gets muted. + by_harness: dict[str, list[tuple[Path, int, tuple[float, float, float, float, float]]]] = defaultdict(list) + skipped_crashed = 0 + skipped_no_window = 0 for path in args.task_json: try: record = json.loads(path.read_text(encoding="utf-8")) @@ -107,10 +158,29 @@ def main(argv: list[str]) -> int: print(f"skipping {path}: {exc}", file=sys.stderr) continue harness = record.get("agent_type") or "unknown" - for turn in record.get("iterations") or []: + for index, turn in enumerate(record.get("iterations") or []): + # Filter on the TURN, never on the record's `final_status`. The + # orchestrator preserves a crashed partial TurnRecord across a + # retry, so a SUCCESS record can hold a crashed turn; and a + # `coder-eval execute` corpus finalizes EVERY row as NOT_GRADED, + # which says nothing about timing — a status allowlist would skip + # all of it and then report a clean gate over nothing measured. + # + # The two conditions are counted INDEPENDENTLY and a turn matching + # both is counted under each. Short-circuiting on the first would + # leave the second's tally reading 0 on the only corpus that + # contains it — the measured case: all three excluded turns are + # crashed and two of them are also never-measured — which reads as + # a filter that never fires rather than one with no evidence. + crashed = turn.get("crashed") is True + no_window = _never_measured(turn) + if not args.include_crashed and (crashed or no_window): + skipped_crashed += int(crashed) + skipped_no_window += int(no_window) + continue buckets = _turn_buckets(turn) if buckets is not None: - by_harness[harness].append(buckets) + by_harness[harness].append((path, index, buckets)) if not by_harness: print("no timed turns found", file=sys.stderr) @@ -118,14 +188,19 @@ def main(argv: list[str]) -> int: header = ( f"{'harness':<14} {'n':>3} {'wall':>10} {'generation':>11} {'tool':>9} " - f"{'startup':>9} {'teardown':>9} {'residual':>10} {'%':>7} {'worst turn':>11}" + f"{'startup':>9} {'teardown':>9} {'residual':>10} {'%':>7} {'worst turn':>11} " + f"{'gated':>6} {'med|%|':>7} {'worst|%|':>9}" ) print(header) print("-" * len(header)) worst_share = 0.0 worst_turn = 0.0 + skipped_short = 0 + breaches: list[tuple[str, float, float, float, Path, int]] = [] + gateable_total = 0 for harness in sorted(by_harness): - turns = by_harness[harness] + rows = by_harness[harness] + turns = [buckets for _, _, buckets in rows] n = len(turns) wall, gen, tool, up, down = (sum(col) / n for col in zip(*turns, strict=True)) residual = wall - gen - tool - up - down @@ -134,16 +209,68 @@ def main(argv: list[str]) -> int: # flips between harnesses because head/tail are measured between event # stamps while duration_seconds is the agent's own monotonic span. So # report the worst single turn beside it; that is the real bound. - per_turn = max(abs(w - g - t - u - d) for w, g, t, u, d in turns) + per_turn = max(abs(_residual_ms(buckets)) for buckets in turns) worst_share = max(worst_share, abs(share)) worst_turn = max(worst_turn, per_turn) + + # Per-turn |residual| as a share of that turn's OWN wall clock. A 30 ms + # turn with a 5 ms residual is not a 17% defect, so short turns are out + # of the share columns and out of the gate — but they stay in the means + # above, where their absolute contribution is honest and tiny. + # `wall_ms <= 0` is guarded here and not in `_turn_buckets`, which + # returns a real tuple for a literal 0 duration. + shares: list[tuple[float, Path, int, tuple[float, float, float, float, float]]] = [] + for path, index, buckets in rows: + turn_wall = buckets[0] + if turn_wall <= 0 or turn_wall < args.min_turn_ms: + skipped_short += 1 + continue + shares.append((abs(_residual_ms(buckets)) / turn_wall * 100.0, path, index, buckets)) + gateable_total += len(shares) + if shares: + median_share = statistics.median(s for s, _, _, _ in shares) + worst_row = max(shares, key=lambda row: row[0]) + med_col = f"{median_share:>6.3f}%" + worst_col = f"{worst_row[0]:>8.3f}%" + else: + med_col = f"{'—':>7}" + worst_col = f"{'—':>9}" + if args.max_residual_pct is not None: + for turn_share, path, index, buckets in shares: + if turn_share > args.max_residual_pct: + breaches.append((harness, turn_share, _residual_ms(buckets), buckets[0], path, index)) + print( f"{harness:<14} {n:>3} {wall:>9.1f}ms {gen:>10.1f}ms {tool:>8.1f}ms " - f"{up:>8.1f}ms {down:>8.1f}ms {residual:>9.3f}ms {share:>6.2f}% {per_turn:>9.3f}ms" + f"{up:>8.1f}ms {down:>8.1f}ms {residual:>9.3f}ms {share:>6.2f}% {per_turn:>9.3f}ms " + f"{len(shares):>6} {med_col} {worst_col}" ) print(f"\nworst mean |residual| = {worst_share:.2f}% of wall clock") print(f"worst single-turn |residual| = {worst_turn:.3f}ms") - return 0 + print( + f"skipped: {skipped_crashed} crashed, {skipped_no_window} no-window " + f"(a turn can be both), {skipped_short} short (< {args.min_turn_ms:.0f}ms)" + ) + + if not gateable_total: + # A gate that passes because it measured nothing is the exact failure + # this script exists to remove, so it only passes when none was asked for. + print("no gateable turns", file=sys.stderr) + return 1 if args.max_residual_pct is not None else 0 + + if args.max_residual_pct is None: + return 0 + if not breaches: + print(f"\ngate OK: every one of {gateable_total} gateable turns is within {args.max_residual_pct}%") + return 0 + print(f"\ngate FAILED: {len(breaches)} turn(s) over {args.max_residual_pct}% of wall clock", file=sys.stderr) + for harness, turn_share, residual_ms, turn_wall, path, index in sorted(breaches, key=lambda b: -b[1]): + print( + f" {harness:<14} {turn_share:>8.3f}% residual {residual_ms:>10.3f}ms " + f"wall {turn_wall:>10.1f}ms {path} turn {index}", + file=sys.stderr, + ) + return 1 if __name__ == "__main__": From fd451fe5f1cbad95d7faaf88cd9941de3b7dfe07 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 18:49:18 -0700 Subject: [PATCH 20/54] =?UTF-8?q?refactor(timing):=202/6=20=E2=80=94=20one?= =?UTF-8?q?=20close=5Fwindow()=20for=20the=20tiling=20reducers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex, opencode and pi each carried their own copy of the same window arithmetic — tile from the mark, defend the start with min(), bound the still-open calls at the boundary, subtract the UNION, clamp at zero — plus three near-identical paragraphs explaining why subtracting an open call here does not double-subtract it later. One helper, one docstring. A pure refactor: the golden master passes with NO regeneration, and the three call sites were checked argument by argument against the formulas they replace. Codex's min() moves from the epoch-millisecond domain into the datetime domain, which is safe because `_ms_to_dt` is strictly monotone over ms-spaced inputs, and its `item_start` stays guarded so `_ms_to_dt(None)` cannot fire a third `datetime.now()`. `mark` is keyword-only with no default: a reducer cannot open a window without stating what it tiles from. That constrains the call shape, not the value — pi still passes its own turn start, and the docstring says so rather than claiming the defect is already gone. Antigravity is NOT migrated here. Its span is monotonic while its tool spans are wall, so this signature cannot express it without either dead code or a moved number; it migrates in 5/6, with the deletion of that split. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/agents/codex_agent.py | 43 +++----- src/coder_eval/agents/opencode_agent.py | 31 +++--- src/coder_eval/agents/pi_agent.py | 27 +++-- src/coder_eval/timing.py | 56 ++++++++++ tests/test_codex_agent.py | 124 ++++++++++++++++++++++ tests/test_opencode_agent.py | 52 ++++++++++ tests/test_pi_agent.py | 25 +++++ tests/test_timing_close_window.py | 130 ++++++++++++++++++++++++ 8 files changed, 428 insertions(+), 60 deletions(-) create mode 100644 tests/test_timing_close_window.py diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 0b5f39b33..6202f218b 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -53,7 +53,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.timing import busy_ms +from coder_eval.timing import close_window from coder_eval.utils import expand_env_vars @@ -473,14 +473,9 @@ def _flush_message(self, last: Any) -> None: # attributed to nothing. Across that turn only 15.8% of the 17 s wall # clock was accounted for. Tiling matches Antigravity and claude-code, # and is what lets Sum(generation) + Sum(tool) reconcile to the turn. - # - # min() is defensive: a stamp that goes backwards must never push the - # window start PAST the first item and invert the span. - window_start_ms = self.gen_mark_ms if self.gen_mark_ms is not None else self.open_start_ms - if window_start_ms is not None and self.open_start_ms is not None: - window_start_ms = min(window_start_ms, self.open_start_ms) + mark_ms = self.gen_mark_ms if self.gen_mark_ms is not None else self.open_start_ms window_end_ms = self.open_end_ms if self.open_end_ms is not None else self.open_start_ms - started = _ms_to_dt(window_start_ms) + mark = _ms_to_dt(mark_ms) completed = _ms_to_dt(window_end_ms) # The window is extended to the LAST item's completion, so any # generation containing a tool call already CONTAINS that tool's @@ -490,30 +485,24 @@ def _flush_message(self, last: Any) -> None: # Generation + Tool exec then exceeded the wall clock they must # reconcile to. # - # Same treatment, and the same shared helper, as Antigravity: subtract - # the UNION of the tool intervals clipped to this window. A sum would - # over-subtract wherever they overlap, which Codex produces natively - # via concurrent collab agents. - # - # Closed intervals, plus any call still OPEN at this flush bounded at - # the window end. Excluding the open ones publishes the part of a - # straddling call that ran inside this window as generation while the - # call's own duration_ms counts it again — harmless while the windows - # were too narrow to overlap a tool, and a live double-count now that - # they tile. Antigravity hit exactly that and broke the invariant by - # 0.26 ms; the fix travels with the tiling that makes it reachable. + # Same shared helper as the other tiling harnesses: the UNION of the + # tool intervals clipped to this window, the open calls bounded at its + # end, and the double-subtraction rule they rest on — all in + # `close_window`'s docstring rather than restated here. tool_spans = [ (c.execution_started_at, c.execution_completed_at) for c in self.commands if c.execution_started_at is not None and c.execution_completed_at is not None ] - tool_spans += [ - (t.execution_started_at, completed) - for t in self.open_tools.values() - if t.execution_started_at is not None and t.execution_started_at < completed - ] - span_ms = max((completed - started).total_seconds() * 1000.0, 0.0) - gen_ms = max(0.0, span_ms - busy_ms(tool_spans, started, completed)) + started, gen_ms = close_window( + mark=mark, + now=completed, + item_start=_ms_to_dt(self.open_start_ms) if self.open_start_ms is not None else None, + closed_spans=tool_spans, + open_started_ats=[ + t.execution_started_at for t in self.open_tools.values() if t.execution_started_at is not None + ], + ) message_id = f"{self.turn_id}-msg-{self.gen_index}" # Output split: reasoning portion to the thinking row, the remainder to diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index be6920117..8bb95d51f 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -76,7 +76,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.timing import busy_ms +from coder_eval.timing import close_window from ._skills import _plugin_skill_dirs from .registry import AgentRegistry @@ -701,9 +701,6 @@ def on_step_finish(self, part: dict[str, Any]) -> None: completed = datetime.now() step_start = self.step_started_at or completed - # Tile from the previous step's finish; min() keeps a clock that went - # backwards from inverting the span. - started = min(self.gen_mark, step_start) if self.gen_mark is not None else step_start blocks: list[ContentBlock] = [] step_text = "".join(self.step_text_parts) if step_text: @@ -711,24 +708,22 @@ def on_step_finish(self, part: dict[str, Any]) -> None: for i, tool_id in enumerate(self.step_tool_ids, start=len(blocks)): blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) - # A call still OPEN at this boundary counts too, bounded at `completed`. - # Subtracting only CLOSED intervals publishes the part of a straddling - # call that ran inside this window as generation, while the call's own - # duration_ms counts it again — a live double-count now that the windows - # tile contiguously from `gen_mark`. No double subtraction: when the call - # later closes, `_finish_tool` appends its full interval to the NEXT - # window's list, where busy_ms clips it to the post-boundary remainder. - spans = self.step_tool_spans + [ - (t.execution_started_at, completed) for t in self.open_tools.values() if t.execution_started_at is not None - ] + # Tile from the previous step's finish. The open calls and the double- + # subtraction rule they rest on live in `close_window`'s docstring. + started, generation_ms = close_window( + mark=self.gen_mark if self.gen_mark is not None else step_start, + now=completed, + item_start=step_start, + closed_spans=self.step_tool_spans, + open_started_ats=[ + t.execution_started_at for t in self.open_tools.values() if t.execution_started_at is not None + ], + ) self.messages.append( AssistantMessage( started_at=started, completed_at=completed, - generation_duration_ms=max( - 0.0, - (completed - started).total_seconds() * 1000 - busy_ms(spans, started, completed), - ), + generation_duration_ms=generation_ms, content_blocks=blocks, tool_use_ids=list(self.step_tool_ids), input_tokens=step_in, diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index e327e66a1..ea160b838 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -109,7 +109,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.timing import busy_ms +from coder_eval.timing import close_window from .registry import AgentRegistry @@ -573,7 +573,6 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: else: self.error_message = None - started = self.turn_started_at or datetime.now() completed = datetime.now() blocks: list[ContentBlock] = [] turn_text = "".join(self.turn_text_parts) @@ -582,23 +581,21 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: for i, tool_id in enumerate(self.turn_tool_ids, start=len(blocks)): blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) - # A call still OPEN at this boundary counts too, bounded at `completed`. - # Subtracting only CLOSED intervals publishes the part of a straddling - # call that ran inside this window as generation, while the call's own - # duration_ms counts it again. No double subtraction: when the call later - # closes, `_finish_tool` appends its full interval to the NEXT turn's - # list, where busy_ms clips it to the post-boundary remainder. - spans = self.turn_tool_spans + [ - (t.execution_started_at, completed) for t in self.open_tools.values() if t.execution_started_at is not None - ] + # The open calls and the double-subtraction rule they rest on live in + # `close_window`'s docstring. + started, generation_ms = close_window( + mark=self.turn_started_at if self.turn_started_at is not None else completed, + now=completed, + closed_spans=self.turn_tool_spans, + open_started_ats=[ + t.execution_started_at for t in self.open_tools.values() if t.execution_started_at is not None + ], + ) self.messages.append( AssistantMessage( started_at=started, completed_at=completed, - generation_duration_ms=max( - 0.0, - (completed - started).total_seconds() * 1000 - busy_ms(spans, started, completed), - ), + generation_duration_ms=generation_ms, content_blocks=blocks, tool_use_ids=list(self.turn_tool_ids), input_tokens=step_in, diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index ba3b65cdb..5ea529431 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -54,6 +54,62 @@ def busy_ms(spans: list[tuple[datetime, datetime]], lo: datetime, hi: datetime) return total + (open_end - open_start).total_seconds() * 1000.0 +def close_window( + *, + mark: datetime, + now: datetime, + item_start: datetime | None = None, + closed_spans: list[tuple[datetime, datetime]], + open_started_ats: list[datetime], +) -> tuple[datetime, float]: + """Close one generation window at ``now``: its ``(started, generation_ms)``. + + The shape the tiling harnesses had copy-pasted; codex, opencode and pi call + it today. Antigravity is not merely unmigrated — it derives its span from + the MONOTONIC clock while unioning WALL-clock tool spans, which this + signature cannot express — and claude-code subtracts once at finalization + across every emission instead. + + ``mark`` is where the window opens — normally the previous flush's close, + which is what makes the windows TILE the turn contiguously instead of + leaving the model time that PRODUCED an item attributed to nothing. It is + keyword-only and has NO default so that no reducer can open a window + without stating what it tiles from. That constrains the call SHAPE, not the + VALUE: pi still passes its own turn start, so its inter-turn gaps are still + in no bucket until it grows a mark of its own. The signature makes the + omission visible; it does not fix it. + + ``item_start`` is this emission's own first stamp, when the harness has + one. The ``min()`` against ``mark`` is the tiling defense and nothing else: + a stamp that went backwards must never push the window start PAST the first + item and invert the span. + + A call still OPEN at this boundary counts against the window too, bounded + at ``now``. Subtracting only CLOSED intervals publishes the part of a + straddling call that ran inside this window as generation, while the call's + own ``duration_ms`` counts it again. + + NO DOUBLE SUBTRACTION, and this is the rationale that used to sit copy- + pasted at four call sites: when that open call later closes, the reducer + appends its FULL interval to the next window's ``closed_spans``, where + ``busy_ms`` clips it to the post-boundary remainder. Each millisecond of + tool time is therefore subtracted from exactly one window. + + The UNION is subtracted, never the sum (see ``busy_ms``), and the result is + clamped at ``0.0`` — an inverted window (``now`` before ``mark``, two + clocks disagreeing) is a measured zero, not a negative generation. + + It deliberately does NOT return ``completed``. The window always ends at + ``now``, which the caller passed in, so handing it back would be an + argument returned unchanged — redundancy dressed as symmetry. Call sites + write ``completed_at=now`` directly. + """ + started = min(mark, item_start) if item_start is not None else mark + bounded = [(s, now) for s in open_started_ats if s < now] + span_ms = (now - started).total_seconds() * 1000.0 + return started, max(0.0, span_ms - busy_ms(closed_spans + bounded, started, now)) + + def decompose_turn( first_started_at: datetime | None, last_completed_at: datetime | None, diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index 342f72979..e25db8637 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -2301,6 +2301,47 @@ async def test_generation_plus_tool_exec_does_not_exceed_the_window(self): assert gen_ms == pytest.approx(10.0) assert gen_ms + tool_ms == pytest.approx(window_ms) + async def test_the_published_window_reconciles_to_its_own_bounds(self): + """The reducer subtracted exactly the spans the record carries. + + `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell + both recompute the tool UNION from the recorded command spans and + subtract it from the recorded window bounds. The cases above pin + arithmetic results against known fixture constants; this one asserts + the reducer fed the window the same span set the record publishes. + + The narrow half by design: `expected` comes from the PUBLISHED bounds, + so it cannot see a wrong mark (TestFlushMessageWindowBounds does), and + both sides call `busy_ms`, so it cannot see a union bug. + """ + from coder_eval.timing import busy_ms + + first = _bounds_command_item("cmd_a") + second = _bounds_command_item("cmd_b") + notifications = [ + _item_notification("item/started", first, started_at_ms=_BOUNDS_EPOCH_MS), + _item_notification("item/completed", first, completed_at_ms=_BOUNDS_EPOCH_MS + 120), + _item_notification("item/started", second, started_at_ms=_BOUNDS_EPOCH_MS + 130), + _item_notification("item/completed", second, completed_at_ms=_BOUNDS_EPOCH_MS + 900), + _token_usage(inp=10, out=5, cached=0), + _turn_completed(), + ] + agent = _started_agent(parse_agent_config(type=AgentKind.CODEX), notifications) + record = await agent.communicate("go") + + assistant = [m for m in record.messages if m.role == "assistant"] + spans = [ + (c.execution_started_at, c.execution_completed_at) + for c in record.commands + if c.execution_started_at is not None and c.execution_completed_at is not None + ] + # Codex splits one window's gen_ms across its sub-messages by output + # share, so the reconciliation is against their SUM, not any one row. + lo = min(m.started_at for m in assistant) + hi = max(m.completed_at for m in assistant) + expected = (hi - lo).total_seconds() * 1000.0 - busy_ms(spans, lo, hi) + assert sum(m.generation_duration_ms or 0.0 for m in assistant) == pytest.approx(expected) + class TestGenerationWindowsTileTheTurn: """Each generation window runs from the PREVIOUS one's end, not its own first item. @@ -2364,6 +2405,89 @@ async def test_tool_time_is_still_excluded_from_a_tiled_window(self): assert gen_ms + tool_ms == pytest.approx(2050.0) +class TestFlushMessageWindowBounds: + """Where `_flush_message`'s window OPENS, driven at the reducer. + + The end-to-end cases above all describe a stream whose stamps advance, so + they cannot reach the two arguments the reducer hands `close_window` for + the awkward cases: the emission's own first stamp (`item_start`) and the + calls still open at the flush. Both moved from inline code into the shared + helper, so without these they are pinned only in the helper's own unit + tests — the wiring between the two would be free to rot. + """ + + @staticmethod + def _flush(*, gen_mark_ms, open_start_ms, open_end_ms, open_tool_started_ms=None): + from coder_eval.agents.codex_agent import _CodexTurnState, _ms_to_dt + from coder_eval.models import CommandTelemetry, ContentBlock + from coder_eval.streaming.callbacks import CompositeStreamCallback + from coder_eval.streaming.collector import EventCollector + + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5.5")) + collector = EventCollector() + st = _CodexTurnState( + agent, + emit=CompositeStreamCallback([collector]), + task_id="codex", + turn_id="codex-1", + collector=collector, + commands=[], + messages=[], + user_input="go", + iteration=1, + turn_start_time=0.0, + ) + st.open_blocks = [ContentBlock(block_type="text", sequence=0, text="answer")] + st.gen_mark_ms = gen_mark_ms + st.open_start_ms = open_start_ms + st.open_end_ms = open_end_ms + if open_tool_started_ms is not None: + st.open_tools["open-1"] = CommandTelemetry( + tool_name="bash", + tool_id="open-1", + timestamp=_ms_to_dt(open_tool_started_ms), + execution_started_at=_ms_to_dt(open_tool_started_ms), + ) + st._flush_message(SimpleNamespace(input_tokens=10, cached_input_tokens=0, output_tokens=5)) + return st.messages[0] + + def test_a_mark_later_than_the_first_item_does_not_invert_the_window(self): + # A backwards SDK stamp: the previous flush closed at +2000 while this + # emission's first item claims +500. The window must cover the item. + # Without `item_start` it opens at +2000, past its own end, and clamps + # to a fabricated instant generation. + message = self._flush( + gen_mark_ms=_BOUNDS_EPOCH_MS + 2000, + open_start_ms=_BOUNDS_EPOCH_MS + 500, + open_end_ms=_BOUNDS_EPOCH_MS + 1100, + ) + from coder_eval.agents.codex_agent import _ms_to_dt + + assert message.started_at == _ms_to_dt(_BOUNDS_EPOCH_MS + 500) + assert message.generation_duration_ms == pytest.approx(600.0) + + def test_a_call_still_open_at_the_flush_is_subtracted_bounded_at_the_end(self): + # It has no completion yet, so only [start, window end] is not model + # time. Its full interval joins the NEXT window's closed spans, where + # busy_ms clips it to the remainder — subtracted once, not twice. + message = self._flush( + gen_mark_ms=_BOUNDS_EPOCH_MS, + open_start_ms=_BOUNDS_EPOCH_MS, + open_end_ms=_BOUNDS_EPOCH_MS + 1000, + open_tool_started_ms=_BOUNDS_EPOCH_MS + 700, + ) + assert message.generation_duration_ms == pytest.approx(700.0) + + def test_a_call_opening_after_the_window_closes_is_ignored(self): + message = self._flush( + gen_mark_ms=_BOUNDS_EPOCH_MS, + open_start_ms=_BOUNDS_EPOCH_MS, + open_end_ms=_BOUNDS_EPOCH_MS + 1000, + open_tool_started_ms=_BOUNDS_EPOCH_MS + 1500, + ) + assert message.generation_duration_ms == pytest.approx(1000.0) + + class TestFlushMessageGenTimeSplit: """`gen_ms` is apportioned across sub-messages by their output share. diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 6d62bbcd4..17f1b7e66 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -1879,6 +1879,58 @@ def test_an_open_tool_overlapping_a_closed_one_is_counted_once(self, monkeypatch ) assert message.generation_duration_ms == pytest.approx(200.0) + def test_a_mark_later_than_the_step_start_does_not_invert_the_window(self, monkeypatch): + """The backwards-clock defence, pinned at the reducer, not in isolation. + + `close_window`'s `min()` only fires if the reducer actually passes the + step's own start as `item_start`. Drop that argument and the window + opens at the (later) mark instead, so the span shrinks — or inverts and + clamps to 0.0, publishing a fabricated instant generation. Nothing else + in this file fails when it is dropped. + """ + state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="do it", model="m") + state.step_started_at = self.WINDOW_START + # A mark 400ms AFTER this step began: the CLI's step_finish for the + # previous step landed late, or the clock stepped. + state.gen_mark = self.WINDOW_START + timedelta(milliseconds=400) + + class _Clock(datetime): + @staticmethod + def now(tz=None): + return TestGenerationWindowExcludesToolExecution.WINDOW_END + + monkeypatch.setattr(agent_module, "datetime", _Clock) + state.on_step_finish({"reason": "stop", "tokens": {"input": 100, "output": 20}}) + + message = next(m for m in state.messages if m.role == "assistant") + assert message.started_at == self.WINDOW_START + assert message.generation_duration_ms == pytest.approx(1000.0) + + def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): + """The reducer subtracted exactly the spans the record carries. + + `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell + both recompute the tool UNION from the recorded command spans and + subtract it from the recorded window bounds. This asserts the reducer + fed the window the same set, so a span silently added or dropped on + the way in shows up here. + + It is deliberately the narrow half: `expected` is derived from the + PUBLISHED bounds, so it cannot see a wrong mark, and both sides call + `busy_ms`, so it cannot see a union bug. Those are pinned by the cases + above and by tests/test_timing_close_window.py. + """ + from coder_eval.timing import busy_ms + + closed = [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))] + open_start = self.WINDOW_START + timedelta(milliseconds=500) + message = self._finish_step(monkeypatch, closed, open_starts=[open_start]) + + spans = [*closed, (open_start, message.completed_at)] + span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 + expected = span_ms - busy_ms(spans, message.started_at, message.completed_at) + assert message.generation_duration_ms == pytest.approx(expected) + class TestGenerationWindowsTileTheTurn: """Each step's window runs from the PREVIOUS step's finish, not its own `step_start`. diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index dcc1c3f78..d756febe0 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -1185,3 +1185,28 @@ def test_an_open_tool_overlapping_a_closed_one_is_counted_once(self, monkeypatch open_starts=[self.WINDOW_START + timedelta(milliseconds=500)], ) assert message.generation_duration_ms == pytest.approx(200.0) + + def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): + """The reducer subtracted exactly the spans the record carries. + + `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell + both recompute the tool UNION from the recorded command spans and + subtract it from the recorded window bounds. This asserts the reducer + fed the window the same set, so a span silently added or dropped on + the way in shows up here. + + It is deliberately the narrow half: `expected` is derived from the + PUBLISHED bounds, so it cannot see a wrong mark, and both sides call + `busy_ms`, so it cannot see a union bug. Those are pinned by the cases + above and by tests/test_timing_close_window.py. + """ + from coder_eval.timing import busy_ms + + closed = [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))] + open_start = self.WINDOW_START + timedelta(milliseconds=500) + message = self._finish_turn(monkeypatch, closed, open_starts=[open_start]) + + spans = [*closed, (open_start, message.completed_at)] + span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 + expected = span_ms - busy_ms(spans, message.started_at, message.completed_at) + assert message.generation_duration_ms == pytest.approx(expected) diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py new file mode 100644 index 000000000..a55143cb4 --- /dev/null +++ b/tests/test_timing_close_window.py @@ -0,0 +1,130 @@ +"""`close_window` — the one generation-window arithmetic the tiling reducers share. + +Each of codex, opencode and pi had to get these cases right independently while +the body was copy-pasted; this file proves them once, against the helper. It is +the sensor for the arithmetic ITSELF, as distinct from the per-reducer tests, +which pin that a given reducer feeds it the right bounds and spans. +""" + +from datetime import datetime, timedelta + +import pytest + +from coder_eval.timing import close_window + + +MARK = datetime(2026, 9, 11, 12, 0, 0) + + +def _at(ms: int) -> datetime: + return MARK + timedelta(milliseconds=ms) + + +class TestCloseWindow: + def test_no_tools_keeps_the_whole_window(self): + started, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[]) + assert started == MARK + assert generation_ms == pytest.approx(1000.0) + + def test_a_contained_closed_tool_is_subtracted_once(self): + _, generation_ms = close_window( + mark=MARK, + now=_at(1000), + closed_spans=[(_at(200), _at(700))], + open_started_ats=[], + ) + assert generation_ms == pytest.approx(500.0) + + def test_overlapping_closed_tools_subtract_their_union_not_their_sum(self): + # Two 500 ms calls overlapping by 400 ms occupy 600 ms of wall clock. + # Summing them would leave 0 generation for a window that generated 400. + _, generation_ms = close_window( + mark=MARK, + now=_at(1000), + closed_spans=[(_at(100), _at(600)), (_at(200), _at(700))], + open_started_ats=[], + ) + assert generation_ms == pytest.approx(400.0) + + def test_an_open_tool_is_bounded_at_now(self): + # Still running when the window closes: it owns [300, 1000], not nothing. + _, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[_at(300)]) + assert generation_ms == pytest.approx(300.0) + + def test_a_tool_straddling_the_mark_is_clipped_to_the_post_mark_part(self): + # The pre-mark half belongs to the PREVIOUS window, which already + # subtracted it. Counting it again here would over-subtract. + _, generation_ms = close_window( + mark=MARK, + now=_at(1000), + closed_spans=[(_at(-400), _at(300))], + open_started_ats=[], + ) + assert generation_ms == pytest.approx(700.0) + + def test_item_start_before_the_mark_wins(self): + # A stamp that went backwards: the window must cover the item, so the + # min() moves the start back rather than inverting the span. + started, generation_ms = close_window( + mark=MARK, + now=_at(1000), + item_start=_at(-200), + closed_spans=[], + open_started_ats=[], + ) + assert started == _at(-200) + assert generation_ms == pytest.approx(1200.0) + + def test_item_start_after_the_mark_keeps_the_mark(self): + # The normal tiling case: the gap between the previous close and this + # item's first stamp IS model time and belongs inside the window. + started, generation_ms = close_window( + mark=MARK, + now=_at(1000), + item_start=_at(400), + closed_spans=[], + open_started_ats=[], + ) + assert started == MARK + assert generation_ms == pytest.approx(1000.0) + + def test_an_inverted_window_clamps_to_zero_rather_than_going_negative(self): + started, generation_ms = close_window(mark=_at(1000), now=MARK, closed_spans=[], open_started_ats=[]) + assert started == _at(1000) + assert generation_ms == 0.0 + + def test_an_open_tool_starting_after_now_is_ignored(self): + _, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[_at(1500)]) + assert generation_ms == pytest.approx(1000.0) + + def test_an_open_tool_starting_exactly_at_now_is_ignored(self): + _, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[_at(1000)]) + assert generation_ms == pytest.approx(1000.0) + + def test_closed_and_open_spans_are_unioned_together(self): + # A closed [100, 400] and an open from 300 bounded at 1000 union to + # [100, 1000] — 900 ms busy, 100 ms of generation. + _, generation_ms = close_window( + mark=MARK, + now=_at(1000), + closed_spans=[(_at(100), _at(400))], + open_started_ats=[_at(300)], + ) + assert generation_ms == pytest.approx(100.0) + + def test_tools_covering_the_whole_window_leave_zero_not_a_negative(self): + _, generation_ms = close_window( + mark=MARK, + now=_at(1000), + closed_spans=[(_at(-500), _at(1500))], + open_started_ats=[], + ) + assert generation_ms == 0.0 + + def test_mark_is_keyword_only_and_has_no_default(self): + # A reducer cannot open a window without STATING what it tiles from. + # The value is still the caller's to get right — see the docstring. + with pytest.raises(TypeError): + close_window(MARK, _at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[misc] + with pytest.raises(TypeError): + close_window(now=_at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[call-arg] From f0782829775705a6a837853addc29e29a9f08bd7 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 19:06:51 -0700 Subject: [PATCH 21/54] =?UTF-8?q?fix(timing):=203/6=20=E2=80=94=20a=20tool?= =?UTF-8?q?=20that=20closes=20between=20two=20windows=20is=20not=20model?= =?UTF-8?q?=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both reducers cleared their tool-span list at turn/step START, which is after the window that list feeds has already opened at the mark. A call closing in the gap therefore had its span wiped before the next flush could subtract it, and the window published that call's execution as model time while the call's own duration_ms counted the same milliseconds again. Reproduced against the real state objects, not argued: a call opening at 100, still running when the step finishes at 1000, closing at 1500, with the next window tiling 1000 -> 2000. OpenCode published 1000.0 for a window whose model time was 500.0 — a 100% overstatement, and it needs the non-terminal tool path, which is why the CLI's usual one-shot `completed` event hides it and the measured corpus reads 0.00%. Pi gets the same reset move AND a `gen_mark`, in one commit and in that order. It was the last harness measuring from its own turn start, so every inter-turn gap fell in no bucket — but it was protected from the span-reset defect BY not tiling, so tiling it without moving the reset first would take a correct harness and introduce the 500 ms double-count. The reset is the value here; Pi's tiling gap measures 0.25 ms median over 25 real window pairs. The golden corpus cannot see any of this: `_scrub.py` masks every timing value to a placeholder, and its identity assertion is an upper bound, so under-accounting passes it silently. So both harnesses gain an ms-exact `generation + UNION(tool) == span` test across the boundary, and the reset move is mutation-pinned on each. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/agents/opencode_agent.py | 13 +- src/coder_eval/agents/pi_agent.py | 27 ++++- tests/test_opencode_agent.py | 130 ++++++++++++++++++++ tests/test_pi_agent.py | 153 ++++++++++++++++++++++++ 4 files changed, 319 insertions(+), 4 deletions(-) diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index 8bb95d51f..eb2f16cb6 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -370,7 +370,14 @@ def on_step_start(self, part: dict[str, Any]) -> None: self.step_started_at = datetime.now() self.step_text_parts = [] self.step_tool_ids = [] - self.step_tool_spans = [] + # `step_tool_spans` is deliberately NOT reset here. The window this + # list feeds opened at `gen_mark` — the PREVIOUS step's finish — so a + # call closing in the gap before this `step_start` belongs to it, and + # clearing the list now wipes the span before `step_finish` can + # subtract it. Reproduced: the window then published the call's + # execution as model time while the call's own `duration_ms` counted + # the same milliseconds again — a 100% overstatement of that window. + # It is cleared at the flush instead, right after the mark advances. self.emit( TurnStartEvent( task_id=self.task_id, @@ -739,8 +746,10 @@ def on_step_finish(self, part: dict[str, Any]) -> None: # A message was appended, so the next window starts where this one # ended. Only `step_finish` advances the mark: a step that never # finished published nothing, so tiling past it would attribute its - # time to whichever step finishes next. + # time to whichever step finishes next. The span list is cleared with + # it, and only with it — see `on_step_start`. self.gen_mark = completed + self.step_tool_spans = [] self.emit( TurnEndEvent( task_id=self.task_id, diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index ea160b838..2b26902b0 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -296,6 +296,16 @@ def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str # whenever the harness runs tools concurrently, and only their # union may be subtracted (timing.py::busy_ms). self.turn_tool_spans: list[tuple[datetime, datetime]] = [] + # Where the NEXT generation window starts: the previous turn's end. + # Pi was the only harness measuring from its own `turn_start`, so the + # wall clock between one `turn_end` and the next `turn_start` — the + # model time that PRODUCED that turn — fell into no bucket at all. + # + # None until the first turn finishes, and deliberately so: the first + # window keeps its own `turn_start`, because everything before it is + # CLI process spawn, not model time. Same shape as OpenCode's + # `gen_mark` and Codex's `gen_mark_ms`. + self.gen_mark: datetime | None = None # toolCallId -> telemetry for tools awaiting a result. self.open_tools: dict[str, CommandTelemetry] = {} @@ -356,7 +366,11 @@ def on_turn_start(self) -> None: self.turn_started_at = datetime.now() self.turn_text_parts = [] self.turn_tool_ids = [] - self.turn_tool_spans = [] + # `turn_tool_spans` is deliberately NOT reset here — see the identical + # note in `opencode_agent.on_step_start`. Now that the window opens at + # `gen_mark` rather than at this `turn_start`, a call closing in the + # gap between them belongs to it, and clearing the list here would + # publish that call's execution as the next window's model time. self.emit( TurnStartEvent( task_id=self.task_id, @@ -583,9 +597,11 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: # The open calls and the double-subtraction rule they rest on live in # `close_window`'s docstring. + turn_start = self.turn_started_at if self.turn_started_at is not None else completed started, generation_ms = close_window( - mark=self.turn_started_at if self.turn_started_at is not None else completed, + mark=self.gen_mark if self.gen_mark is not None else turn_start, now=completed, + item_start=turn_start, closed_spans=self.turn_tool_spans, open_started_ats=[ t.execution_started_at for t in self.open_tools.values() if t.execution_started_at is not None @@ -608,6 +624,13 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: message_id=str(message.get("responseId") or "") or None, ) ) + # A message was appended, so the next window starts where this one + # ended. Only a finished turn advances the mark: one that never + # finished published nothing, so tiling past it would attribute its + # time to whichever turn finishes next. The span list is cleared with + # it, and only with it — see `on_turn_start`. + self.gen_mark = completed + self.turn_tool_spans = [] self.emit( TurnEndEvent( task_id=self.task_id, diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 17f1b7e66..458a51bf8 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -1996,3 +1996,133 @@ def test_the_steps_leave_no_gap_between_them(self, monkeypatch): covered = (second.completed_at - first.started_at).total_seconds() * 1000.0 gen = sum(m.generation_duration_ms or 0.0 for m in (first, second)) assert gen == pytest.approx(covered) + + +_SPAN_EPOCH_MS = 1_800_000_000_000 +_SPAN_BASE = datetime.fromtimestamp(_SPAN_EPOCH_MS / 1000) + + +class _SteppedClock(datetime): + """A clock the test moves by hand, in ms from `_SPAN_BASE`. + + Subclasses `datetime` rather than stubbing it, because `_epoch_ms_to_dt` + calls `datetime.fromtimestamp` through the same module global and must keep + resolving to the real implementation — the CLI's epoch stamps and the + reducer's own `now()` reads have to land on ONE timeline for the span + arithmetic under test to mean anything. + """ + + at_ms = 0.0 + + @staticmethod + def now(tz=None): + return _SPAN_BASE + timedelta(milliseconds=_SteppedClock.at_ms) + + +class TestToolSpansSurviveTheStepBoundary: + """A tool that closes BETWEEN two steps still belongs to the next window. + + `step_tool_spans` used to be cleared at `step_start`, which is after the + window it feeds has already opened at `gen_mark`. A call closing in that + gap had its span wiped before the next `step_finish` could subtract it, so + the window published the call's execution as model time while the call's + own `duration_ms` counted the same milliseconds again. + + It needs the NON-TERMINAL tool path to reach: the CLI normally emits one + already-`completed` event per call, which closes inside the step that + opened it. That is why the measured corpus reads 0.00% and a reproduction + has to drive the state object. + """ + + def _run(self, monkeypatch): + monkeypatch.setattr(agent_module, "datetime", _SteppedClock) + state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") + # The resolved telemetry leaves the state via ToolEnd; the identity + # case below reconciles against what was RECORDED, not against the + # clock the test scripted. + resolved: list[Any] = [] + state.bind(lambda e: resolved.append(e.tool) if isinstance(e, ToolEndEvent) else None) + + def tool(status, *, end_ms=None): + times = {"start": _SPAN_EPOCH_MS + 100} + if end_ms is not None: + times["end"] = _SPAN_EPOCH_MS + end_ms + state.on_tool_use({"callID": "c1", "tool": "bash", "state": {"status": status, "time": times}}) + + _SteppedClock.at_ms = 0 + state.on_step_start({"messageID": "m1"}) + _SteppedClock.at_ms = 100 + tool("running") # non-terminal: stays open across the boundary + _SteppedClock.at_ms = 1000 + state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) + _SteppedClock.at_ms = 1500 + tool("completed", end_ms=1500) # closes in the GAP between the steps + _SteppedClock.at_ms = 1600 + state.on_step_start({"messageID": "m2"}) + _SteppedClock.at_ms = 2000 + state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) + return resolved, [m for m in state.messages if m.role == "assistant"] + + def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self, monkeypatch): + _, messages = self._run(monkeypatch) + assert len(messages) == 2 + # Window 2 tiles 1000 -> 2000. c1 ran for 1000 -> 1500 of it, so 500ms + # is model time. Before the reset moved, this published 1000.0 — a 100% + # overstatement, with c1's own duration_ms counting the same 500ms. + assert messages[1].generation_duration_ms == pytest.approx(500.0) + + def test_the_call_is_subtracted_from_exactly_one_window(self, monkeypatch): + # Window 1 owns c1's 100 -> 1000 slice (it was open at that boundary + # and bounded there); window 2 owns 1000 -> 1500. Neither owns both. + _, messages = self._run(monkeypatch) + assert messages[0].generation_duration_ms == pytest.approx(100.0) + assert messages[1].generation_duration_ms == pytest.approx(500.0) + + def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self, monkeypatch): + """generation + UNION(tool) accounts for the whole span, to the ms. + + The assertion the golden corpus CANNOT make: `_scrub.py` masks + `generation_duration_ms` and both bounds to a placeholder, so a + snapshot records that a window was measured and never what it + measured, and its identity check is an upper bound besides — so + under-accounting, the defect this phase fixes, passes it silently. + """ + from coder_eval.timing import busy_ms + + resolved, messages = self._run(monkeypatch) + lo, hi = messages[0].started_at, messages[1].completed_at + generation_ms = sum(m.generation_duration_ms or 0.0 for m in messages) + command = next(c for c in resolved if c.tool_id == "c1") + tool_ms = busy_ms([(command.execution_started_at, command.execution_completed_at)], lo, hi) + + assert generation_ms + tool_ms == pytest.approx((hi - lo).total_seconds() * 1000.0) + + def test_a_step_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self, monkeypatch): + monkeypatch.setattr(agent_module, "datetime", _SteppedClock) + state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") + _SteppedClock.at_ms = 0 + state.on_step_start({"messageID": "m1"}) + _SteppedClock.at_ms = 1000 + state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) + mark_after_flush = state.gen_mark + + _SteppedClock.at_ms = 1600 + state.on_step_start({"messageID": "m2"}) + _SteppedClock.at_ms = 1700 + state.on_tool_use( + { + "callID": "c2", + "tool": "bash", + "state": {"status": "running", "time": {"start": _SPAN_EPOCH_MS + 1700}}, + } + ) + _SteppedClock.at_ms = 1900 + state.close_open_tools() # crash/timeout orphan sweep — no message appended + + # Published nothing, so tiling past it would hand its time to whichever + # step finishes next, and wiping the spans would publish c2's execution + # as that step's model time. + assert state.gen_mark == mark_after_flush + assert state.step_tool_spans == [ + (_SPAN_BASE + timedelta(milliseconds=1700), _SPAN_BASE + timedelta(milliseconds=1900)) + ] diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index d756febe0..e7bd8558f 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -1210,3 +1210,156 @@ def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 expected = span_ms - busy_ms(spans, message.started_at, message.completed_at) assert message.generation_duration_ms == pytest.approx(expected) + + +_SPAN_BASE = datetime(2026, 3, 1, 9, 0, 0) + + +class _SteppedClock(datetime): + """A clock the test moves by hand, in ms from `_SPAN_BASE`. + + Pi self-stamps its tool spans with `datetime.now()`, so the tool intervals + and the window bounds come from this one source; scripting it is what makes + the span arithmetic deterministic. + """ + + at_ms = 0.0 + + @staticmethod + def now(tz=None): + return _SPAN_BASE + timedelta(milliseconds=_SteppedClock.at_ms) + + +def _turn_end_payload(): + return {"message": {"role": "assistant", "usage": {"input": 10, "output": 5}, "stopReason": "stop"}} + + +class TestGenerationWindowsTileTheTurn: + """Each window runs from the PREVIOUS `turn_end`, not from its own `turn_start`. + + Pi was the only harness measuring from its own turn start, so the wall + clock between one `turn_end` and the next `turn_start` — the model time + that PRODUCED the next turn — fell into no bucket at all. The four-bucket + identity is asserted only as an upper bound, so nothing failed. + + The gap is small in practice (measured across 25 real window pairs: median + 0.25 ms, max 0.75 ms). The value here is that it closes, and that the tool + spans keep working once it does — see TestToolSpansSurviveTheTurnBoundary, + which is the half that carries the weight. + """ + + def _two_turns(self, monkeypatch): + monkeypatch.setattr(agent_module, "datetime", _SteppedClock) + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m") + _SteppedClock.at_ms = 0 + state.on_turn_start() + _SteppedClock.at_ms = 1000 + state.on_turn_end(_turn_end_payload()) + _SteppedClock.at_ms = 1600 + state.on_turn_start() + _SteppedClock.at_ms = 2000 + state.on_turn_end(_turn_end_payload()) + return [m for m in state.messages if m.role == "assistant"] + + def test_the_second_window_abuts_the_first(self, monkeypatch): + messages = self._two_turns(monkeypatch) + assert len(messages) == 2 + assert messages[1].started_at == messages[0].completed_at + + def test_the_inter_turn_gap_is_inside_a_window_rather_than_unaccounted(self, monkeypatch): + messages = self._two_turns(monkeypatch) + # 1000 -> 2000, which includes the 600ms between `turn_end` and the + # next `turn_start`. Untiled this reported 400ms and lost the 600. + assert messages[1].generation_duration_ms == pytest.approx(1000.0) + + +class TestToolSpansSurviveTheTurnBoundary: + """A tool that closes BETWEEN two turns still belongs to the next window. + + `turn_tool_spans` used to be cleared at `turn_start`, which is after the + window it feeds has opened at the mark. Pi was protected from that only by + NOT tiling: its window opened at `turn_start`, so a call that ended before + then fell outside it anyway. Tiling without moving the reset therefore + takes a correct harness and introduces the double-count — which is why both + changes land in one commit, reset first. + """ + + def _run(self, monkeypatch): + monkeypatch.setattr(agent_module, "datetime", _SteppedClock) + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m") + # The resolved telemetry leaves the state via ToolEnd; the identity + # case below reconciles against what was RECORDED, not against the + # clock the test scripted. + resolved: list[Any] = [] + state.bind(lambda e: resolved.append(e.tool) if isinstance(e, ToolEndEvent) else None) + _SteppedClock.at_ms = 0 + state.on_turn_start() + _SteppedClock.at_ms = 100 + state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) + _SteppedClock.at_ms = 1000 + state.on_turn_end(_turn_end_payload()) + _SteppedClock.at_ms = 1500 + state.on_tool_execution_end({"toolCallId": "c1", "result": "ok"}) # closes in the GAP + _SteppedClock.at_ms = 1600 + state.on_turn_start() + _SteppedClock.at_ms = 2000 + state.on_turn_end(_turn_end_payload()) + return resolved, [m for m in state.messages if m.role == "assistant"] + + def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self, monkeypatch): + _, messages = self._run(monkeypatch) + # Window 2 tiles 1000 -> 2000. c1 ran for 1000 -> 1500 of it, so 500ms + # is model time. With the reset left at `turn_start` this reads 1000.0. + assert messages[1].generation_duration_ms == pytest.approx(500.0) + + def test_the_call_is_subtracted_from_exactly_one_window(self, monkeypatch): + _, messages = self._run(monkeypatch) + # Window 1 bounded c1 at its own close (100 -> 1000); window 2 takes + # only the remainder. + assert messages[0].generation_duration_ms == pytest.approx(100.0) + assert messages[1].generation_duration_ms == pytest.approx(500.0) + + def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self, monkeypatch): + """generation + UNION(tool) accounts for the whole span, to the ms. + + This is the assertion the golden corpus CANNOT make: `_scrub.py` masks + `generation_duration_ms` and both bounds to a placeholder, so a + snapshot records that a window was measured and never what it measured. + Its identity check (`_scrub.py`) is an upper bound besides, so + under-accounting — the defect this phase fixes — passes it silently. + `scripts/timing/decompose_run.py --max-residual-pct` is the two-sided + check on live runs; this is the committed one. + """ + from coder_eval.timing import busy_ms + + resolved, messages = self._run(monkeypatch) + lo, hi = messages[0].started_at, messages[1].completed_at + generation_ms = sum(m.generation_duration_ms or 0.0 for m in messages) + command = next(c for c in resolved if c.tool_id == "c1") + tool_ms = busy_ms([(command.execution_started_at, command.execution_completed_at)], lo, hi) + + assert generation_ms + tool_ms == pytest.approx((hi - lo).total_seconds() * 1000.0) + + def test_a_turn_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self, monkeypatch): + monkeypatch.setattr(agent_module, "datetime", _SteppedClock) + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m") + _SteppedClock.at_ms = 0 + state.on_turn_start() + _SteppedClock.at_ms = 1000 + state.on_turn_end(_turn_end_payload()) + mark_after_flush = state.gen_mark + + _SteppedClock.at_ms = 1600 + state.on_turn_start() + _SteppedClock.at_ms = 1700 + state.on_tool_execution_start({"toolCallId": "c2", "toolName": "bash", "args": {}}) + _SteppedClock.at_ms = 1900 + state.close_open_tools() # crash/timeout orphan sweep — no message appended + + # Published nothing, so tiling past it would hand its time to whichever + # turn finishes next, and wiping the spans would publish c2's execution + # as that turn's model time. + assert state.gen_mark == mark_after_flush + assert [(s, e) for s, e in state.turn_tool_spans] == [ + (_SPAN_BASE + timedelta(milliseconds=1700), _SPAN_BASE + timedelta(milliseconds=1900)) + ] From 125b3886f83983b0f77cdb191cfb8efc15c7eb5d Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 19:15:09 -0700 Subject: [PATCH 22/54] =?UTF-8?q?test(lint):=204/6=20=E2=80=94=20CE061,=20?= =?UTF-8?q?a=20window=20must=20come=20from=20the=20shared=20helper?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pi shipped measuring its generation window from its own turn_start while four sibling reducers tiled from a mark, so every inter-turn gap fell in no bucket. Nothing caught it: the parity doc asserted the four-bucket identity, the only sensor for that identity checks one side, and Pi's own tests were written against Pi's own arithmetic. A sixth harness rolling its own window would arrive the same way — with a green suite by construction. So the rule is about PROVENANCE, not values: a module in agents/ that publishes a measured `generation_duration_ms` must import `close_window`. Separate id from CE058/CE059/CE060, which are about the values a message carries — one invariant per id is what makes a noqa mean one thing. Its weakness is stated in its own docstring rather than left to be discovered: it proves the helper is imported, never that a given call used it. The value is always a local, so no AST rule can trace it. The sensors for the arithmetic are tests/test_timing_close_window.py and the per-reducer window tests. Two suppressions, not the one the plan predicted. claude-code's is permanent — it subtracts tool time once at finalization across every emission, a shape `close_window` cannot take without a mode flag. Antigravity's is marked TEMPORARY and comes out in 5/6 with its clock conversion. A test pins that exactly these two files need suppressing, so a noqa cannot outlive its reason. CE060 already owned the alias resolution both rules need, so it moves to a shared `_model_ctor.py` rather than being copied: a new import spelling now needs one fix, not two. Every behavioural CE060 test is unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 1 + src/coder_eval/agents/antigravity_agent.py | 7 +- src/coder_eval/agents/claude_code_agent.py | 9 +- tests/lint/rules/_model_ctor.py | 89 ++++++++++++ tests/lint/rules/ce060_message_id_declared.py | 82 +++-------- .../rules/ce061_window_via_close_window.py | 131 ++++++++++++++++++ tests/lint/runner.py | 2 + tests/test_custom_lint.py | 122 +++++++++++++++- 8 files changed, 380 insertions(+), 63 deletions(-) create mode 100644 tests/lint/rules/_model_ctor.py create mode 100644 tests/lint/rules/ce061_window_via_close_window.py diff --git a/pyproject.toml b/pyproject.toml index 8669cbc46..4159a0365 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -296,6 +296,7 @@ external = [ "CE058", "CE059", "CE060", + "CE061", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 51414eea8..708485778 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -1075,7 +1075,12 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: for i, block in enumerate(self._blocks): block.sequence = i self.messages.append( - AssistantMessage( + # CE061 suppressed TEMPORARILY, removed in 5/6. This window cannot be + # expressed by `close_window` yet: its span is monotonic while its + # tool spans are wall, so the helper (which derives the span from + # `now - started`, both wall) would change the published number. + # The clock conversion and this migration land together. + AssistantMessage( # noqa: CE061 started_at=self._gen_mark_wall, completed_at=now_wall, generation_duration_ms=max(0.0, generation_ms), diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index b49305e02..f5e9fb349 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -425,7 +425,14 @@ def on_assistant_message(self, message: Message) -> None: out_tok = int(msg_usage.get("output_tokens", 0) or 0) self.pending_delta_output_tokens = None - assistant_telemetry = AssistantMessageTelemetry( + # CE061's one permanent exception. This harness measures its + # window as a monotonic delta and subtracts tool time ONCE at + # finalization across every emission (`_subtract_tool_time_from_windows`), + # because a call issued by an earlier emission is still running when the + # next window closes. `close_window` subtracts per flush; forcing both + # shapes into it means a mode flag on a helper whose whole value is + # having one shape. It already uses the shared `busy_ms`. + assistant_telemetry = AssistantMessageTelemetry( # noqa: CE061 started_at=generation_started_wall, completed_at=message_arrival_wall, generation_duration_ms=max(0.0, generation_duration_ms), diff --git a/tests/lint/rules/_model_ctor.py b/tests/lint/rules/_model_ctor.py new file mode 100644 index 000000000..b5074cd63 --- /dev/null +++ b/tests/lint/rules/_model_ctor.py @@ -0,0 +1,89 @@ +"""Resolve `coder_eval.models` constructor calls inside one module's AST. + +CE060 and CE061 ask the same first question — *is this call building an +`AssistantMessage`?* — and answering it takes more than matching a name: a +module may bind the class under any alias, reach it through a relative import, +or never bind it at all and spell it `models.AssistantMessage(...)`. CE060 +worked that out once; duplicating it into CE061 would mean a model rename or a +new import spelling needs two fixes in two rules, and the second one is the one +that gets missed. So it lives here and both rules consume it. + +The class name is taken from the model itself rather than written as a string, +the way CE056 imports `IN_CONTAINER_ENV` and CE057 derives its target set from +`SIDECAR_MODULES`: renaming the model moves both rules with it. + +BLIND SPOT, inherited by every consumer: a re-export through an intermediate +module (`from .sibling import AssistantMessage`) is invisible, because +resolving it means following imports across files and no rule in this package +does that. +""" + +import ast +import re + +from coder_eval.models import AssistantMessage + + +# Reducers live here; nothing outside it builds a generation window. +AGENTS_ROOT = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]agents[/\\]") + +_MODELS_MODULE = "coder_eval.models" +_MODELS_TAIL = _MODELS_MODULE.rpartition(".")[2] + +# Taken from the model, never spelled here: a rename then moves the rules too. +ASSISTANT_MESSAGE = AssistantMessage.__name__ + + +def reaches_models_module(node: ast.ImportFrom) -> bool: + """True if this `from ... import` reaches `coder_eval.models`. + + A relative import inside `agents/` (`from ..models import ...`) carries only + the tail in `node.module`, so testing the absolute path alone would leave a + rule silently blind for a whole file — and `agents/` does use relative + imports. + """ + module = node.module or "" + if module.startswith(_MODELS_MODULE): + return True + return bool(node.level) and (module == _MODELS_TAIL or module.startswith(f"{_MODELS_TAIL}.")) + + +def local_bindings(tree: ast.AST, class_name: str) -> set[str]: + """Every local name this module binds `coder_eval.models.` to. + + Built per file: caching it across files would leak one module's alias into + another's matching. + """ + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and reaches_models_module(node): + names.update(a.asname or a.name for a in node.names if a.name == class_name) + return names + + +def constructor_name(func: ast.expr, names: set[str], class_name: str) -> str | None: + """The spelling this call used to name the model, or None if it did not. + + A bare name has to be bound in this module to be ours; the attribute + spelling is matched on the attribute alone, since the module binding it + arrives through (`import coder_eval.models as models`, `from coder_eval + import models`) is what a walk over one file's CLASS bindings cannot see. + """ + if isinstance(func, ast.Name) and func.id in names: + return func.id + if isinstance(func, ast.Attribute) and func.attr == class_name: + return func.attr + return None + + +def keywords_of(node: ast.Call) -> dict[str, ast.expr]: + """The call's named arguments. A `**`-expansion contributes nothing. + + That is deliberate rather than an oversight: such a call has not declared + the field AT THE SITE, which is what these rules are about. + """ + return {kw.arg: kw.value for kw in node.keywords if kw.arg is not None} + + +def is_none(node: ast.expr | None) -> bool: + return isinstance(node, ast.Constant) and node.value is None diff --git a/tests/lint/rules/ce060_message_id_declared.py b/tests/lint/rules/ce060_message_id_declared.py index aeb31fdd7..fbdb48200 100644 --- a/tests/lint/rules/ce060_message_id_declared.py +++ b/tests/lint/rules/ce060_message_id_declared.py @@ -36,6 +36,10 @@ CE056 imports ``IN_CONTAINER_ENV`` and CE057 derives its target set from ``SIDECAR_MODULES``. Renaming the model therefore moves this rule with it. +That resolution lives in ``_model_ctor.py`` and is shared with CE061, which +needs the identical answer to a different question. Keeping two copies would +mean a new import spelling needs two fixes in two rules. + BLIND SPOT 1: the runtime ``None``. The rule requires the kwarg to be *present*, not non-``None`` when it runs. ``opencode_agent.py`` passes ``str(part.get("messageID") or "") or None`` and ``pi_agent.py`` the same shape @@ -48,15 +52,12 @@ partially — a snapshot is written from whatever the code currently does, so it catches a later change, never an initial omission. -BLIND SPOT 2: a binding this file cannot resolve. ``check()`` reads one -module's own imports, so it sees the direct forms — absolute or relative -``from ... import AssistantMessage``, under any alias — and the attribute -spelling ``.AssistantMessage(...)``, which is matched on the attribute -alone precisely because the module binding it comes through (``import -coder_eval.models as models``, ``from coder_eval import models``) is the part a -class-binding walk misses. What remains invisible is a re-export through an -intermediate module (``from .sibling import AssistantMessage``): resolving that -means following imports across files, which no rule in this package does. +BLIND SPOT 2: a binding the resolver cannot follow. It reads one module's own +imports, so it sees the direct forms — absolute or relative ``from ... import +AssistantMessage``, under any alias — and the attribute spelling +``.AssistantMessage(...)``. What remains invisible is a re-export +through an intermediate module (``from .sibling import AssistantMessage``); see +``_model_ctor.py``. A ``**``-expanded call fires: such a call has not declared the field at the site. There is no carve-out because no site in ``src/coder_eval/agents/`` uses @@ -65,74 +66,37 @@ """ import ast -import re -from coder_eval.models import AssistantMessage +from tests.lint.rules._model_ctor import ( + AGENTS_ROOT, + ASSISTANT_MESSAGE, + constructor_name, + is_none, + keywords_of, + local_bindings, +) from tests.lint.rules.base import BaseRule from tests.lint.violation import Violation -_AGENTS_ROOT = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]agents[/\\]") - -_MODELS_MODULE = "coder_eval.models" -_MODELS_TAIL = _MODELS_MODULE.rpartition(".")[2] - -# Taken from the model, never spelled here: a rename then moves the rule too. -_CLASS = AssistantMessage.__name__ - - -def _binds_the_model(node: ast.ImportFrom) -> bool: - """True if this `from ... import` reaches `coder_eval.models`. - - A relative import inside `agents/` (`from ..models import ...`) carries only - the tail in `node.module`, so testing the absolute path alone would leave the - rule silently blind for a whole file — and `agents/` does use relative - imports. - """ - module = node.module or "" - if module.startswith(_MODELS_MODULE): - return True - return bool(node.level) and (module == _MODELS_TAIL or module.startswith(f"{_MODELS_TAIL}.")) - - -def _is_none(node: ast.expr | None) -> bool: - return isinstance(node, ast.Constant) and node.value is None - - class MessageIdDeclared(BaseRule): id = "CE060" def __init__(self, filepath: str) -> None: super().__init__(filepath) - self._in_scope = bool(_AGENTS_ROOT.search(filepath)) - # Local bindings of coder_eval.models.AssistantMessage in THIS module. - # Built per file in check(): caching it across files would leak one - # module's alias into another's matching. + self._in_scope = bool(AGENTS_ROOT.search(filepath)) self._names: set[str] = set() def check(self, tree: ast.AST) -> list[Violation]: - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom) and _binds_the_model(node): - self._names.update(a.asname or a.name for a in node.names if a.name == _CLASS) + self._names = local_bindings(tree, ASSISTANT_MESSAGE) return super().check(tree) def visit_Call(self, node: ast.Call) -> None: if self._in_scope: - func = node.func - # A bare name has to be bound in this module to be ours; the - # attribute spelling is matched on the attribute alone, since the - # module binding it arrives through is what an import walk over one - # file's class bindings cannot see (see BLIND SPOT 2). - name = ( - func.id - if isinstance(func, ast.Name) and func.id in self._names - else func.attr - if isinstance(func, ast.Attribute) and func.attr == _CLASS - else None - ) + name = constructor_name(node.func, self._names, ASSISTANT_MESSAGE) if name is not None: - kwargs = {kw.arg: kw.value for kw in node.keywords if kw.arg is not None} - if "message_id" not in kwargs or _is_none(kwargs["message_id"]): + kwargs = keywords_of(node) + if "message_id" not in kwargs or is_none(kwargs["message_id"]): self.violation( node, f"{name}(...) leaves 'message_id' undeclared — absent, or an explicit None — " diff --git a/tests/lint/rules/ce061_window_via_close_window.py b/tests/lint/rules/ce061_window_via_close_window.py new file mode 100644 index 000000000..4a27d5cce --- /dev/null +++ b/tests/lint/rules/ce061_window_via_close_window.py @@ -0,0 +1,131 @@ +"""CE061: a generation window must come from the shared helper. + +Pi shipped measuring its window from its own ``turn_start`` while four sibling +reducers tiled from a mark, so the wall clock between one turn's end and the +next turn's start — the model time that PRODUCED that turn — fell into no +bucket at all. Nothing failed. ``docs/agents/HARNESS_PARITY.md`` asserted the +four-bucket identity, and the only sensor for it +(``tests/_fixtures/golden_streams/_scrub.py``) checks ONE side: it catches a +bucket claiming more time than the turn contains and says nothing about one +claiming less. Pi's own tests passed because they were written against Pi's +own arithmetic. + +That is the shape this rule guards against: not a reducer that computes the +window wrongly, but a reducer that computes it AT ALL instead of asking +``coder_eval.timing.close_window``. A new harness whose author reimplements the +arithmetic inline arrives with a green test suite by construction. + +Separate id from CE058, CE059 and CE060 deliberately. Those three are about the +VALUES a message carries — an unknown duration published as a literal, a window +built from one clock read, a missing identity. This one is about PROVENANCE: +where the arithmetic came from. One invariant per id is what makes a ``# noqa`` +mean one thing. + +BLIND SPOT, and it is the whole weakness of the chosen shape: this proves the +module IMPORTS the helper, never that any particular call used it. The value +passed to ``generation_duration_ms=`` is always a local (``generation_ms``, +``gen_parts[idx]``), so no AST rule can trace it back to a call. The sensors for +the arithmetic itself are ``tests/test_timing_close_window.py`` and the +per-reducer window tests; this rule adds only the cheap structural half that +neither can reach — a sixth harness rolling its own. + +It costs exactly one permanent suppression. ``claude_code_agent.py`` computes +its window from a monotonic delta and subtracts tool time ONCE at finalization +across every emission, because a call issued by an earlier emission is still +running when the next window closes. Forcing that into ``close_window`` means a +mode flag on a helper whose whole value is having one shape. + +EXEMPT, because both are honest claims that no window was measured: an explicit +``generation_duration_ms=None`` (codex's rollout rebuild, claude-code's +sub-agent synthesis) and the kwarg absent altogether, which defaults to +``None``. Not matched: ``**``-expansion and ``model_copy(update={...})`` — CE058 +already covers the ``model_copy`` dict shape for timing literals. + +Alias resolution, and its blind spot, live in ``_model_ctor.py``, shared with +CE060. The helper's own name is taken from the function object rather than +written here as a string, so renaming it moves this rule too. +""" + +import ast + +from coder_eval.timing import close_window +from tests.lint.rules._model_ctor import ( + AGENTS_ROOT, + ASSISTANT_MESSAGE, + constructor_name, + is_none, + keywords_of, + local_bindings, +) +from tests.lint.rules.base import BaseRule +from tests.lint.violation import Violation + + +_TIMING_MODULE = "coder_eval.timing" +_TIMING_TAIL = _TIMING_MODULE.rpartition(".")[2] + +# Taken from the function, never spelled here: a rename then moves the rule too. +_HELPER = close_window.__name__ + + +def _imports_the_helper(tree: ast.AST) -> bool: + """True if this module can reach `close_window` under any spelling. + + Both the `from`-import (under any alias) and the module import that makes + `timing.close_window(...)` possible count — a rule that recognized only the + first would tell an author to change a working call site. + """ + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + module = node.module or "" + reaches = module.startswith(_TIMING_MODULE) or ( + bool(node.level) and (module == _TIMING_TAIL or module.startswith(f"{_TIMING_TAIL}.")) + ) + if reaches and any(a.name == _HELPER for a in node.names): + return True + # `from coder_eval import timing` / `from .. import timing`. The + # package is checked too: `from anywhere import timing` is not this + # module, and accepting it would let an unrelated name disarm the + # rule for a whole file. + package = module == _TIMING_MODULE.rpartition(".")[0] or (bool(node.level) and not module) + if package and any(a.name == _TIMING_TAIL for a in node.names): + return True + elif isinstance(node, ast.Import): + if any(a.name == _TIMING_MODULE for a in node.names): + return True + return False + + +class WindowViaCloseWindow(BaseRule): + id = "CE061" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(AGENTS_ROOT.search(filepath)) + self._names: set[str] = set() + self._has_helper = False + + def check(self, tree: ast.AST) -> list[Violation]: + self._names = local_bindings(tree, ASSISTANT_MESSAGE) + self._has_helper = _imports_the_helper(tree) + return super().check(tree) + + def visit_Call(self, node: ast.Call) -> None: + if self._in_scope and not self._has_helper: + name = constructor_name(node.func, self._names, ASSISTANT_MESSAGE) + duration = keywords_of(node).get("generation_duration_ms") + if name is not None and duration is not None and not is_none(duration): + self.violation( + node, + f"{name}(...) publishes a measured 'generation_duration_ms' but this module " + f"never imports {_TIMING_MODULE}.{_HELPER} — so it is computing a generation " + "window of its own. Every window is the same arithmetic: tile from the mark, " + "keep a backwards stamp from inverting the span, bound the calls still open at " + "the boundary, subtract the UNION of the tool intervals clipped to the window, " + "clamp at zero. Pi got that wrong by measuring from its own turn start, and " + "nothing caught it because the four-bucket identity is only asserted as an " + f"upper bound. Call {_HELPER} instead. If this harness genuinely cannot use it " + "— claude-code subtracts once at finalization across every emission — add " + "'# noqa: CE061' with a comment saying why.", + ) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index c14f9865d..791275091 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -38,6 +38,7 @@ from tests.lint.rules.ce058_no_timing_literal import NoTimingLiteral from tests.lint.rules.ce059_generation_window_is_two_reads import GenerationWindowIsTwoReads from tests.lint.rules.ce060_message_id_declared import MessageIdDeclared +from tests.lint.rules.ce061_window_via_close_window import WindowViaCloseWindow from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -99,6 +100,7 @@ NoTimingLiteral, GenerationWindowIsTwoReads, MessageIdDeclared, + WindowViaCloseWindow, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 05aff5d58..ea2a62d5f 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -3935,6 +3935,122 @@ def test_the_real_module_is_clean(self): assert "_os._exit(137)" in source, "the guarded call must still exist" +class TestCE061WindowViaCloseWindow: + """CE061 flags a reducer that computes a generation window of its own. + + Every source string carries its own import line: the rule derives its + constructor set from the module's own `coder_eval.models` imports (shared + with CE060 via `_model_ctor`), so a bare `AssistantMessage(...)` with no + import is correctly invisible to it. + """ + + _IMPORT = "from coder_eval.models import AssistantMessage\n" + _HELPER = "from coder_eval.timing import close_window\n" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/agents/pi_agent.py"): + import ast + + from tests.lint.rules.ce061_window_via_close_window import WindowViaCloseWindow + + return WindowViaCloseWindow(filepath).check(ast.parse(src)) + + def test_flags_a_measured_window_without_the_helper(self): + assert len(self._run(self._IMPORT + "m = AssistantMessage(generation_duration_ms=x)")) == 1 + + def test_allows_a_measured_window_when_the_helper_is_imported(self): + assert not self._run(self._IMPORT + self._HELPER + "m = AssistantMessage(generation_duration_ms=x)") + + def test_allows_an_explicit_none(self): + # "Never measured" is an honest claim and needs no window arithmetic — + # codex's rollout rebuild and claude-code's sub-agent synthesis. + assert not self._run(self._IMPORT + "m = AssistantMessage(generation_duration_ms=None)") + + def test_allows_the_kwarg_absent(self): + # Defaults to None, which is the same honest claim. + assert not self._run(self._IMPORT + "m = AssistantMessage(model=model)") + + def test_flags_an_arbitrary_alias(self): + # The gap CE058 concedes: a name list guards the in-tree spelling by + # coincidence and misses `as Msg` outright. + assert ( + len(self._run("from coder_eval.models import AssistantMessage as Msg\nm = Msg(generation_duration_ms=x)")) + == 1 + ) + + def test_flags_the_module_attribute_spelling(self): + assert ( + len(self._run("import coder_eval.models as models\nm = models.AssistantMessage(generation_duration_ms=x)")) + == 1 + ) + + def test_accepts_a_relative_helper_import(self): + # `agents/` uses relative imports; matching only the absolute path + # would leave the rule blind for a whole file. + assert not self._run( + self._IMPORT + "from ..timing import close_window\nm = AssistantMessage(generation_duration_ms=x)" + ) + + def test_accepts_the_module_import_spelling_of_the_helper(self): + # `timing.close_window(...)` is a working call site; a rule that saw + # only the from-import would tell its author to change it. + assert not self._run( + self._IMPORT + "from coder_eval import timing\nm = AssistantMessage(generation_duration_ms=x)" + ) + + def test_an_unrelated_timing_import_does_not_disarm_the_rule(self): + # `from somewhere.else import timing` is not this module; accepting any + # name spelled `timing` would switch the rule off for a whole file. + assert ( + len( + self._run( + self._IMPORT + "from vendor.sdk import timing\nm = AssistantMessage(generation_duration_ms=x)" + ) + ) + == 1 + ) + + def test_ignores_a_file_outside_agents(self): + assert not self._run( + self._IMPORT + "m = AssistantMessage(generation_duration_ms=x)", + filepath="src/coder_eval/streaming/collector.py", + ) + + def test_keys_on_the_helper_name_rather_than_a_literal(self): + from coder_eval.timing import close_window as _helper + from tests.lint.rules import ce061_window_via_close_window as rule_mod + + assert _helper.__name__ == rule_mod._HELPER + + def test_the_real_agents_tree_is_clean(self): + # After the two suppressions: claude-code's permanent one, and + # antigravity's temporary one pending its 5/6 migration. + import pathlib + + from tests.lint.rules.ce061_window_via_close_window import WindowViaCloseWindow + from tests.lint.runner import check_file + + root = pathlib.Path(__file__).resolve().parent.parent / "src" / "coder_eval" / "agents" + found = [v for path in sorted(root.glob("*.py")) for v in check_file(path, [WindowViaCloseWindow])] + assert not found, found + + def test_each_suppression_is_load_bearing(self): + # A noqa nobody needs is a noqa that outlives its reason. Both of these + # must correspond to a violation the rule actually raises. + import ast + import pathlib + + from tests.lint.rules.ce061_window_via_close_window import WindowViaCloseWindow + + root = pathlib.Path(__file__).resolve().parent.parent / "src" / "coder_eval" / "agents" + suppressed = { + path.name + for path in sorted(root.glob("*.py")) + if WindowViaCloseWindow(str(path)).check(ast.parse(path.read_text(encoding="utf-8"))) + } + assert suppressed == {"claude_code_agent.py", "antigravity_agent.py"} + + class TestRuffExternalCoversEveryRule: """Every CE rule's documented `# noqa` must be accepted by ruff. @@ -4499,10 +4615,12 @@ def test_flags_a_relative_import(self): assert self._run("from ..models import AssistantMessage\nm = AssistantMessage(model=model)") def test_keys_on_the_model_name_rather_than_a_literal(self): + # The constant moved into the shared resolver when CE061 was added; it + # is still derived from the model, which is the property under test. from coder_eval.models import AssistantMessage as _Model - from tests.lint.rules import ce060_message_id_declared as rule_mod + from tests.lint.rules import _model_ctor - assert _Model.__name__ == rule_mod._CLASS + assert _Model.__name__ == _model_ctor.ASSISTANT_MESSAGE def test_flags_a_star_expanded_call(self): # `**fields` has not declared the field at the site. From 1347a8cfaf5a8dde4115bd6b43c5150c0bcc8e78 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 19:32:46 -0700 Subject: [PATCH 23/54] =?UTF-8?q?fix(timing):=205/6=20=E2=80=94=20one=20cl?= =?UTF-8?q?ock=20basis=20per=20turn=20on=20antigravity=20and=20pi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Antigravity read its window span off time.monotonic() while unioning wall-clock tool intervals and subtracting one from the other. That is the only reason the window could go negative at all, and the clamp underneath it published a 0.0 indistinguishable from a real instant generation, with a debug line as the only trace. One basis makes the disagreement unrepresentable, so the branch and the clamp are deleted rather than left unreachable — a test greps the source to say so. It moves onto close_window in the same commit, which is the only point the two could be exchanged without either dead code or a moved number, and its temporary CE061 suppression comes out with it. Pi's stamps were naive-LOCAL datetime.now(). A DST transition or an NTP step inside a turn lands directly in a generation window — an hour in a field measured in milliseconds, on nightly runs that start at 04:18 and run for hours. A monotonic-derived stamp cannot express it. Codex and OpenCode keep theirs: their tool spans are the CLI's own epoch stamps, unreachable from the host, so converting only the window bounds would put two bases inside one busy_ms subtraction — relocating the defect instead of removing it. This narrows the hazard from five harnesses to two; the parity doc says so rather than implying it is solved. The clock is INJECTED into the turn-state constructors, not read from a module global, and that is the phase's largest blast radius rather than a style choice: a derived stamp does not read datetime.now(), so the four existing monkeypatches would have stopped reaching the reducer and those tests would have quietly measured the real clock and passed. Verified by hand on both harnesses that deleting the injected fake now FAILS. Deadlines stay on raw time.monotonic(), commented at one site per harness: a deadline must not move when the wall clock steps. Co-Authored-By: Claude Opus 5 (1M context) --- src/coder_eval/agents/antigravity_agent.py | 97 +++++------ src/coder_eval/agents/pi_agent.py | 33 +++- src/coder_eval/timing.py | 54 +++++- tests/test_antigravity_agent.py | 161 ++++++++++++++++-- tests/test_custom_lint.py | 8 +- tests/test_pi_agent.py | 189 +++++++++++++-------- tests/test_timing_close_window.py | 49 ++++++ 7 files changed, 441 insertions(+), 150 deletions(-) diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 708485778..542ab00e0 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -67,7 +67,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.timing import busy_ms +from coder_eval.timing import TurnClock, close_window from coder_eval.utils import expand_env_vars @@ -554,8 +554,15 @@ async def communicate( assert self.config.type is not None, "AntigravityAgent requires AgentConfig.type before communicate()" self._begin_turn() + # Raw monotonic, and deliberately not the turn clock: this seeds the + # poll deadline below and `duration_seconds`, neither of which may move + # when the wall clock steps. `TurnClock` is for the RECORDED stamps. turn_start_time = time.monotonic() - turn_start_wall = datetime.now() + # ONE clock per turn. This is the (monotonic, wall) pair the reducer + # already captured here and then failed to use for its later stamps — + # which is why its window span was monotonic while its tool intervals + # were wall, and why the two could disagree. + clock = TurnClock() task_id = str(self.config.type) model = self._effective_model() collector = EventCollector() @@ -572,7 +579,7 @@ async def communicate( iteration=self._iteration, model=model, turn_start_time=turn_start_time, - turn_start_wall=turn_start_wall, + clock=clock, max_turns=max_turns, ) @@ -806,7 +813,7 @@ def __init__( iteration: int, model: str, turn_start_time: float, - turn_start_wall: datetime, + clock: TurnClock, max_turns: int | None = None, ) -> None: self._agent = agent @@ -818,6 +825,11 @@ def __init__( self.iteration = iteration self.model = model self.turn_start_time = turn_start_time + # Every wall stamp below derives from this, so the tool spans and the + # window bounds they are subtracted from share one basis. Injected, not + # read from a module global, so a test supplies a fake instead of + # monkeypatching `datetime` out from under the reducer. + self.clock = clock self.max_turns = max_turns self.timeout_hit = False @@ -853,8 +865,7 @@ def __init__( # come from the SAME instant, captured by communicate(), so the # recorded bounds and the measured duration describe one span. # Advanced only by a flush that actually emitted a message. - self._gen_mark_monotonic: float = turn_start_time - self._gen_mark_wall: datetime = turn_start_wall + self._gen_mark_wall: datetime = clock.now() # Execution intervals of tools that CLOSED since the mark. This harness # interleaves tool calls into one generation — the Step for the tool # arrives and only a later usage_metadata Step cuts the message — so a @@ -939,7 +950,7 @@ def _handle_tool_call(self, call: Any, step: Any, done: bool, sstatus: Any, call self._next_seq += 1 tool_name = _ANTIGRAVITY_TO_CLAUDE_TOOL_MAP.get(raw_name, str(raw_name)) self._tool_input_keys[cid] = set(call.args) - now = datetime.now() + now = self.clock.now() tel = CommandTelemetry( tool_name=tool_name, tool_id=cid, @@ -965,7 +976,7 @@ def _handle_tool_call(self, call: Any, step: Any, done: bool, sstatus: Any, call or step.content or None ) - completed = datetime.now() + completed = self.clock.now() started = start_tel.execution_started_at or completed tool_ms = max((completed - started).total_seconds() * 1000.0, 0.0) end_tel = start_tel.model_copy( @@ -1024,11 +1035,7 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: """ if not self._blocks and gen.is_empty(): return - now_monotonic = time.monotonic() - now_wall = datetime.now() - # Model-generation time = the whole window MINUS the tool execution - # that happened inside it. - # + now_wall = self.clock.now() # Do NOT "simplify" this to resetting the mark when a tool ends. That # loses real model time: measured on run 2026-09-09_04-18-50, task # skill-rpa-uia-google-search, a harness-local Read closed 8 ms after @@ -1038,52 +1045,35 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: # 43 s Bash, where the model time really is the flush-to-DONE # remainder). # - # A tool that is still OPEN at flush time counts too, bounded at - # `now_wall`. Subtracting only CLOSED intervals published the portion - # of a straddling call that ran before the boundary as generation, - # while the call's own duration_ms counted it again — the one - # double-count that this harness's contiguous windows have no slack to - # absorb. Measured on tasks/hello_date: a Bash opening 1.7 ms before + # The window arithmetic itself, and why a call still open at this + # boundary counts against it, live in `close_window`'s docstring. + # Measured here before the helper existed: a Bash opening 1.7 ms before # the flush drove Sum(generation) + Sum(command) 0.26 ms PAST the turn - # wall, on a turn whose whole headroom was 1.4 ms. The four sibling - # runs passed by 1.2-8.7 ms out of ~12 s, so this was a coin flip, not - # a rounding artifact. + # wall, on a turn whose whole headroom was 1.4 ms. # - # No double subtraction: when the call later closes, the DONE path - # appends its full interval to the NEXT window's list, where busy_ms - # clips it to the post-flush remainder. - span_ms = (now_monotonic - self._gen_mark_monotonic) * 1000.0 - still_open = [ - (tel.execution_started_at, now_wall) - for cid, tel in self._open_tools.items() - if cid not in self._closed_tools and tel.execution_started_at is not None - ] - tool_ms = busy_ms(self._tool_spans_since_mark + still_open, self._gen_mark_wall, now_wall) - generation_ms = span_ms - tool_ms - if generation_ms < 0: - # busy_ms clips to this window and unions overlaps, so it cannot - # exceed the window's own wall span. Reaching here means the two - # clocks disagree (the span is monotonic, the tool intervals are - # wall), i.e. jitter — worth a line in the task log, because the - # clamped 0.0 below is otherwise indistinguishable from a real - # instant generation. Numbers only: no agent output is logged. - self._agent._log.debug( - "Generation window went negative (span=%.1fms tool=%.1fms); clamping to 0.", - span_ms, - tool_ms, - ) + # The span used to be read off `time.monotonic()` while these intervals + # were wall, and subtracting one from the other is the only reason this + # window could go negative — a clamp that was indistinguishable from a + # real instant generation. Both bounds now derive from `self.clock`, so + # the disagreement is unrepresentable and the branch that hid it is + # gone. + _, generation_ms = close_window( + mark=self._gen_mark_wall, + now=now_wall, + closed_spans=self._tool_spans_since_mark, + open_started_ats=[ + tel.execution_started_at + for cid, tel in self._open_tools.items() + if cid not in self._closed_tools and tel.execution_started_at is not None + ], + ) for i, block in enumerate(self._blocks): block.sequence = i self.messages.append( - # CE061 suppressed TEMPORARILY, removed in 5/6. This window cannot be - # expressed by `close_window` yet: its span is monotonic while its - # tool spans are wall, so the helper (which derives the span from - # `now - started`, both wall) would change the published number. - # The clock conversion and this migration land together. - AssistantMessage( # noqa: CE061 + AssistantMessage( started_at=self._gen_mark_wall, completed_at=now_wall, - generation_duration_ms=max(0.0, generation_ms), + generation_duration_ms=generation_ms, content_blocks=list(self._blocks), tool_use_ids=[b.tool_use_id for b in self._blocks if b.block_type == "tool_use" and b.tool_use_id], input_tokens=gen.uncached_input_tokens, @@ -1103,7 +1093,6 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: # Advance the mark ONLY after a message was actually appended. The # early return above means a no-op flush leaves the window open, so a # later real generation still measures from where it began. - self._gen_mark_monotonic = now_monotonic self._gen_mark_wall = now_wall self._tool_spans_since_mark = [] @@ -1147,7 +1136,7 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso for cid, tel in self._open_tools.items(): if cid in self._closed_tools: continue - orphan = tel.model_copy(update={"result_status": "unknown", "execution_completed_at": datetime.now()}) + orphan = tel.model_copy(update={"result_status": "unknown", "execution_completed_at": self.clock.now()}) self.emit.on_event( ToolEndEvent( task_id=self.task_id, diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 2b26902b0..51d089a00 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -109,7 +109,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.timing import close_window +from coder_eval.timing import TurnClock, close_window from .registry import AgentRegistry @@ -262,12 +262,27 @@ class _PiTurnState: to force-close orphans when a turn dies mid-flight. """ - def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str | None) -> None: + def __init__( + self, + *, + task_id: str, + iteration: int, + user_input: str, + model: str | None, + clock: TurnClock | None = None, + ) -> None: self.task_id = task_id self.iteration = iteration self.user_input = user_input self.model = model + # ONE clock per turn, and every wall stamp below derives from it, so + # the tool spans and the window bounds they are subtracted from cannot + # end up on different bases. Injectable so a test can supply a fake + # rather than monkeypatching this module's `datetime` global — which a + # derived stamp would silently escape, leaving the test passing against + # the real clock instead of failing. + self.clock = clock or TurnClock() self.started_at = time.monotonic() self.thread_id: str | None = None @@ -363,7 +378,7 @@ def on_turn_start(self) -> None: self.turn_count += 1 self.turn_open = True self.turn_id = f"turn_{self.turn_count}" - self.turn_started_at = datetime.now() + self.turn_started_at = self.clock.now() self.turn_text_parts = [] self.turn_tool_ids = [] # `turn_tool_spans` is deliberately NOT reset here — see the identical @@ -401,7 +416,7 @@ def on_tool_execution_start(self, obj: dict[str, Any]) -> None: tool_name = _TOOL_NAME_MAP.get(raw_tool.lower(), raw_tool) args = obj.get("args") params = args if isinstance(args, dict) else {} - started = datetime.now() + started = self.clock.now() telemetry = CommandTelemetry( tool_name=tool_name, tool_id=call_id, @@ -448,10 +463,10 @@ def _close_tool( tool_name="unknown", tool_id=call_id, assistant_turn_index=self.turn_count, - timestamp=datetime.now(), + timestamp=self.clock.now(), sequence_number=self.sequence, ) - completed = datetime.now() + completed = self.clock.now() telemetry.execution_completed_at = completed if telemetry.execution_started_at is not None: telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 @@ -587,7 +602,7 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: else: self.error_message = None - completed = datetime.now() + completed = self.clock.now() blocks: list[ContentBlock] = [] turn_text = "".join(self.turn_text_parts) if turn_text: @@ -1011,6 +1026,10 @@ def emit(event: StreamEvent) -> None: ) ) + # Deadlines stay on `time.monotonic()` and are deliberately NOT routed + # through the turn clock: a deadline must not move when the wall clock + # steps. `TurnClock` exists to give the RECORDED stamps one basis; this + # is the one place a raw monotonic reading is the right answer. deadline = None if timeout is None else time.monotonic() + timeout stopped_early = False stderr_drain: asyncio.Future[bytes] | None = None diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 5ea529431..173c591fb 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -21,7 +21,59 @@ does, and both suites replay it. """ -from datetime import datetime +import time +from datetime import datetime, timedelta + + +class TurnClock: + """One (wall, monotonic) pair per turn; every later stamp derives from it. + + A turn's bounds and its durations have to share a basis or they can + disagree, and the disagreement lands in a field measured in milliseconds. + Two concrete failures this removes: + + * Antigravity computed its window span on the MONOTONIC clock while + unioning WALL-clock tool intervals and subtracting one from the other. + That is the only reason its window could go negative at all, and the + clamp that hid it was indistinguishable from a real instant generation. + * Pi stamped with naive-LOCAL ``datetime.now()``. A DST transition or an + NTP step inside a turn lands directly in a generation window — an + hour-long jump in a millisecond field. Nightly runs start at 04:18 and + run for hours, so it is reachable rather than theoretical. A + monotonic-derived stamp cannot express it. + + It is an EXTRACTION, not an invention: antigravity already captured this + exact pair at the top of ``communicate`` and simply did not use it for + later stamps. + + Stamps stay NAIVE LOCAL, matching what the rest of the telemetry and the + persisted ``execution_started_at`` already are, so no consumer changes. + + Within a turn the derived stamp is monotonic-accurate and may drift from + real wall time; each turn re-anchors. That is intended — do not "fix" it by + re-reading the wall clock, which is the property being removed. + + ONE PER TURN, never module-level and never reused across turns: a long run + would accumulate drift between the pair and real wall time. The turn-state + constructors take it as an argument so the lifetime is visible in the + signature, and so tests can inject a fake instead of monkeypatching a + module global out from under the reducer. + + NOT for deadlines. Those stay on ``time.monotonic()`` directly: a deadline + must not move when the wall clock steps. + + Codex and OpenCode deliberately do NOT use it. Their tool spans are the + CLI's own epoch-millisecond stamps, unreachable from the host, so + converting only the window bounds would put two bases inside one + ``busy_ms`` subtraction — relocating the defect instead of removing it. + """ + + def __init__(self) -> None: + self._wall0 = datetime.now() + self._mono0 = time.monotonic() + + def now(self) -> datetime: + return self._wall0 + timedelta(seconds=time.monotonic() - self._mono0) def busy_ms(spans: list[tuple[datetime, datetime]], lo: datetime, hi: datetime) -> float: diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 725138717..bbbf47bad 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -5,6 +5,7 @@ """ import asyncio +import inspect import os import sys from collections.abc import Callable @@ -1500,16 +1501,26 @@ async def test_cap_reached_on_a_poll_redrain_stops_polling(monkeypatch): class _Clock: - """Controlled stand-in for the two clocks the reducer reads. - - ONE monotonically advancing counter, read by both clocks: every read — - `time.monotonic()` or `datetime.now()` — costs TICK_MS. So the fixture's - timeline is driven by read ORDER, not by elapsed time, and the two clocks - are deliberately coupled rather than independent. That is enough to pin - the arithmetic exactly; it is NOT a cross-check that the reducer keeps the - two clocks in their proper roles (a variant deriving the span from the - wall stamps would pass every test here). The module's only clock uses are - `time.monotonic` and `datetime.now`, so patching these two covers it. + """Controlled stand-in for the reducer's clocks — a `TurnClock` and `time`. + + ONE monotonically advancing counter, read by both: every read — the turn + clock's `now()` or `time.monotonic()` — costs TICK_MS. So the fixture's + timeline is driven by read ORDER, not by elapsed time, and the two are + deliberately coupled rather than independent. That is enough to pin the + arithmetic exactly. + + Every WALL stamp the reducer records now derives from its per-turn + `TurnClock`, so this stands in for that object rather than for the + module's `datetime`. That distinction is load-bearing, not cosmetic: a + derived stamp does not read `datetime.now()`, so the old patch would no + longer reach it and these tests would quietly measure the real clock and + pass by accident. `time` is still patched because `duration_seconds` and + the poll deadlines read `time.monotonic()` directly, and must — a deadline + may not move when the wall clock steps. + + What it still does NOT prove is that the reducer keeps the two in their + proper roles; with one basis for every wall stamp there is no longer a + second role to confuse it with. """ TICK_MS = 100.0 @@ -1529,8 +1540,15 @@ def now(self) -> datetime: def _install_clock(monkeypatch, clock: _Clock) -> None: + """Hand the reducer this clock for the turn it is about to build. + + `TurnClock` is replaced by a factory rather than the fake being passed + positionally, because the state — and therefore its clock — is built + inside `communicate()`, out of the caller's reach. One typed seam, and the + stand-in has to satisfy `now()`. + """ monkeypatch.setattr(agent_module, "time", SimpleNamespace(monotonic=clock.monotonic)) - monkeypatch.setattr(agent_module, "datetime", SimpleNamespace(now=clock.now)) + monkeypatch.setattr(agent_module, "TurnClock", lambda: clock) def _assistant(record): @@ -1704,14 +1722,22 @@ async def test_tool_execution_is_subtracted_from_the_window(monkeypatch): second = messages[1] bash = next(c for c in record.commands if c.tool_name == "Bash") - # The window spans 400ms of wall clock and contains a 100ms tool call, so - # 300ms of it was the model generating. Cross-checked against the recorded - # bounds, which come from the OTHER clock the reducer reads. + # The window contains a 100ms tool call, so what is left of it was the + # model generating. That relation is the assertion that matters, and it is + # independent of the fixture's tick size. span_ms = (second.completed_at - second.started_at).total_seconds() * 1000.0 assert bash.duration_ms == pytest.approx(100.0) - assert span_ms == pytest.approx(400.0) assert second.generation_duration_ms == pytest.approx(span_ms - bash.duration_ms) - assert second.generation_duration_ms == pytest.approx(300.0) + + # The absolute figures are artifacts of `_Clock`, which charges one TICK_MS + # per clock READ. They moved from 400/300 to 300/200 when the reducer + # stopped taking a monotonic reading it no longer needs: a flush now reads + # the turn clock once where it used to read two clocks, so each window is + # one tick shorter on this fixture's read-driven timeline. Nothing about + # real elapsed time changed — the 100ms tool, which is still two reads + # apart, is unmoved. + assert span_ms == pytest.approx(300.0) + assert second.generation_duration_ms == pytest.approx(200.0) async def test_a_straddling_tool_is_charged_only_for_its_in_window_part(monkeypatch): @@ -1909,3 +1935,106 @@ async def test_timing_change_moves_no_token_bucket(): # local re-implementation of two of its four buckets. assert_reconciliation(record.model_dump(mode="json")) assert all(m.generation_duration_ms is not None for m in _assistant(record)) + + +async def test_the_published_window_reconciles_to_its_own_bounds(monkeypatch): + """The reducer subtracted exactly the spans the record carries. + + The per-migrated-reducer check its three siblings gained when they moved + onto `close_window`; antigravity could not have it until its span stopped + being monotonic while these intervals were wall. `decompose_run.py` and the + evalboard's Unaccounted cell both recompute the tool UNION from the + recorded command spans and subtract it from the recorded window bounds, so + this asserts the reducer fed the window that same set. + """ + from coder_eval.timing import busy_ms + + _install_clock(monkeypatch, _Clock()) + steps = [ + _step("THINKING", "DONE", thinking="plan", usage=_usage(100, 0, 5, 5)), + _step( + "TOOL_CALL", + "ACTIVE", + target="TARGET_ENVIRONMENT", + tool_calls=[_tc("run_command", "t1", {"command_line": "ls"})], + ), + _step( + "TOOL_CALL", + "DONE", + target="TARGET_ENVIRONMENT", + tool_calls=[_tc("run_command", "t1", {"command_line": "ls", "exit_code": 0})], + ), + _step("THINKING", "DONE", thinking="second", usage=_usage(100, 0, 5, 5)), + ] + record = await _agent_with_steps(steps).communicate("go") + + second = _assistant(record)[1] + spans = [ + (c.execution_started_at, c.execution_completed_at) + for c in record.commands + if c.execution_started_at is not None and c.execution_completed_at is not None + ] + span_ms = (second.completed_at - second.started_at).total_seconds() * 1000.0 + expected = span_ms - busy_ms(spans, second.started_at, second.completed_at) + assert second.generation_duration_ms == pytest.approx(expected) + + +async def test_the_window_is_measured_without_relying_on_the_negative_clamp(monkeypatch): + """A positive window, and no clamp underneath it. + + The span used to be read off `time.monotonic()` while the tool intervals + were wall, so the two could disagree and drive the result negative; the + clamp that caught it published a `0.0` indistinguishable from a real + instant generation, and a debug line was the only trace. One basis makes + that unrepresentable: `busy_ms` clips to the window and unions overlaps, so + it cannot exceed a span derived from the same clock. + """ + _install_clock(monkeypatch, _Clock()) + steps = [ + _step("THINKING", "DONE", thinking="plan", usage=_usage(100, 0, 5, 5)), + _step( + "TOOL_CALL", + "ACTIVE", + target="TARGET_ENVIRONMENT", + tool_calls=[_tc("run_command", "t1", {"command_line": "ls"})], + ), + _step( + "TOOL_CALL", + "DONE", + target="TARGET_ENVIRONMENT", + tool_calls=[_tc("run_command", "t1", {"command_line": "ls", "exit_code": 0})], + ), + _step("THINKING", "DONE", thinking="second", usage=_usage(100, 0, 5, 5)), + ] + record = await _agent_with_steps(steps).communicate("go") + + second = _assistant(record)[1] + assert second.generation_duration_ms > 0.0 + assert second.completed_at > second.started_at + # The branch and its debug line are deleted, not merely unreachable. + source = inspect.getsource(agent_module) + assert "Generation window went negative" not in source + assert "_gen_mark_monotonic" not in source + + +async def test_each_turn_gets_a_fresh_clock(): + """A second turn on the same agent re-anchors rather than inheriting. + + One clock per turn is the rule: a clock outliving its turn would stamp the + next one with the previous turn's wall origin, and over a long run would + accumulate drift against real wall time. + """ + step = _step("THINKING", "DONE", thinking="a", usage=_usage(100, 0, 5, 5)) + agent = _agent_with_steps([step]) + first = _assistant(await agent.communicate("go")) + # The fake conversation yields one batch and is then spent, so borrow a + # fresh one. The agent INSTANCE is deliberately the same: what is under + # test is that its second turn builds its own clock rather than inheriting + # the first turn's origin. + agent._sdk_agent = _agent_with_steps([step])._sdk_agent + second = _assistant(await agent.communicate("again")) + + assert first and second + # Re-anchored: the later turn's window opens after the earlier one closed. + assert second[0].started_at >= first[0].completed_at + assert second[0].completed_at > second[0].started_at diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index ea2a62d5f..ea5f9a5e2 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4035,8 +4035,10 @@ def test_the_real_agents_tree_is_clean(self): assert not found, found def test_each_suppression_is_load_bearing(self): - # A noqa nobody needs is a noqa that outlives its reason. Both of these - # must correspond to a violation the rule actually raises. + # A noqa nobody needs is a noqa that outlives its reason, so the set is + # pinned rather than merely non-empty. It has already earned that: + # antigravity carried a TEMPORARY suppression until it moved onto + # `close_window`, and this test is what failed when the reason expired. import ast import pathlib @@ -4048,7 +4050,7 @@ def test_each_suppression_is_load_bearing(self): for path in sorted(root.glob("*.py")) if WindowViaCloseWindow(str(path)).check(ast.parse(path.read_text(encoding="utf-8"))) } - assert suppressed == {"claude_code_agent.py", "antigravity_agent.py"} + assert suppressed == {"claude_code_agent.py"} class TestRuffExternalCoversEveryRule: diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index e7bd8558f..cf12ab3c4 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -23,7 +23,6 @@ import pytest -from coder_eval.agents import pi_agent as agent_module from coder_eval.agents.pi_agent import PiAgent, _PiTurnState, _result_text from coder_eval.errors import AgentCrashError, TurnTimeoutError from coder_eval.models import AgentKind, AssistantMessage, CommandTelemetry, PiAgentConfig @@ -39,6 +38,7 @@ TurnEndStatus, TurnStartEvent, ) +from coder_eval.timing import TurnClock from tests._fixtures.golden_streams.pi_fixtures import ( EXPECTED_CACHE_READ, EXPECTED_COST, @@ -1093,6 +1093,16 @@ async def test_zero_reported_cost_on_an_unpriced_model_stays_zero(self, patch_ex assert record.token_usage.total_cost_usd == 0.0 +class _FixedClock: + """A `TurnClock` stand-in frozen at one instant, injected into the state.""" + + def __init__(self, at: datetime) -> None: + self.at = at + + def now(self) -> datetime: + return self.at + + class TestGenerationWindowExcludesToolExecution: """A tool running inside a turn is not model time. @@ -1102,21 +1112,22 @@ class TestGenerationWindowExcludesToolExecution: counted the same milliseconds twice, which the task page's Unaccounted cell renders as a ~-100% residual. - Driven at the reducer: the window is two `datetime.now()` reads and the - tool interval comes from the event payload, so only setting both - explicitly makes the arithmetic deterministic. + Driven at the reducer with an injected clock frozen at `WINDOW_END`: the + window's end and the tool intervals both have to be set explicitly for the + arithmetic to be deterministic. """ WINDOW_START = datetime(2026, 1, 1, 12, 0, 0) WINDOW_END = datetime(2026, 1, 1, 12, 0, 1) # a 1000ms turn - def _finish_turn(self, monkeypatch, spans, open_starts=()): - class _Clock(datetime): - @staticmethod - def now(tz=None): - return TestGenerationWindowExcludesToolExecution.WINDOW_END - - state = _PiTurnState(task_id="t", iteration=1, user_input="x", model="m") + def _finish_turn(self, spans, open_starts=()): + state = _PiTurnState( + task_id="t", + iteration=1, + user_input="x", + model="m", + clock=_FixedClock(self.WINDOW_END), + ) state.turn_started_at = self.WINDOW_START state.turn_tool_spans = list(spans) for i, started in enumerate(open_starts): @@ -1126,7 +1137,6 @@ def now(tz=None): timestamp=started, execution_started_at=started, ) - monkeypatch.setattr(agent_module, "datetime", _Clock) state.on_turn_end( {"message": {"role": "assistant", "usage": {"input": 100, "output": 20}, "stopReason": "stop"}} ) @@ -1134,23 +1144,21 @@ def now(tz=None): assert len(assistant) == 1 return assistant[0] - def test_tool_time_inside_the_turn_is_subtracted(self, monkeypatch): + def test_tool_time_inside_the_turn_is_subtracted(self): message = self._finish_turn( - monkeypatch, [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], ) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 assert span_ms == pytest.approx(1000.0) assert message.generation_duration_ms == pytest.approx(500.0) - def test_a_turn_with_no_tools_keeps_its_whole_window(self, monkeypatch): - assert self._finish_turn(monkeypatch, []).generation_duration_ms == pytest.approx(1000.0) + def test_a_turn_with_no_tools_keeps_its_whole_window(self): + assert self._finish_turn([]).generation_duration_ms == pytest.approx(1000.0) - def test_concurrent_tools_are_subtracted_once(self, monkeypatch): + def test_concurrent_tools_are_subtracted_once(self): # Two overlapping 500ms tools occupy 600ms, not 1000ms. Summing them # would leave 0 generation for a turn that generated 400. message = self._finish_turn( - monkeypatch, [ (self.WINDOW_START + timedelta(milliseconds=100), self.WINDOW_START + timedelta(milliseconds=600)), (self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700)), @@ -1158,35 +1166,32 @@ def test_concurrent_tools_are_subtracted_once(self, monkeypatch): ) assert message.generation_duration_ms == pytest.approx(400.0) - def test_the_window_never_goes_negative(self, monkeypatch): + def test_the_window_never_goes_negative(self): message = self._finish_turn( - monkeypatch, [(self.WINDOW_START - timedelta(seconds=30), self.WINDOW_END + timedelta(seconds=30))], ) assert message.generation_duration_ms == 0.0 - def test_a_tool_still_open_at_the_boundary_is_subtracted(self, monkeypatch): + def test_a_tool_still_open_at_the_boundary_is_subtracted(self): # A call that opens inside this turn and closes inside the NEXT one # straddles the boundary. Counting only closed intervals published the # pre-boundary 400ms as generation while the call's own duration_ms # counted it again. message = self._finish_turn( - monkeypatch, [], open_starts=[self.WINDOW_START + timedelta(milliseconds=600)], ) assert message.generation_duration_ms == pytest.approx(600.0) - def test_an_open_tool_overlapping_a_closed_one_is_counted_once(self, monkeypatch): + def test_an_open_tool_overlapping_a_closed_one_is_counted_once(self): # Union, not sum, across the closed and still-open sets alike. message = self._finish_turn( - monkeypatch, [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], open_starts=[self.WINDOW_START + timedelta(milliseconds=500)], ) assert message.generation_duration_ms == pytest.approx(200.0) - def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): + def test_the_published_window_reconciles_to_its_own_bounds(self): """The reducer subtracted exactly the spans the record carries. `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell @@ -1204,7 +1209,7 @@ def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): closed = [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))] open_start = self.WINDOW_START + timedelta(milliseconds=500) - message = self._finish_turn(monkeypatch, closed, open_starts=[open_start]) + message = self._finish_turn(closed, open_starts=[open_start]) spans = [*closed, (open_start, message.completed_at)] span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 @@ -1215,19 +1220,21 @@ def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): _SPAN_BASE = datetime(2026, 3, 1, 9, 0, 0) -class _SteppedClock(datetime): - """A clock the test moves by hand, in ms from `_SPAN_BASE`. +class _SteppedClock: + """A `TurnClock` stand-in the test moves by hand, in ms from `_SPAN_BASE`. - Pi self-stamps its tool spans with `datetime.now()`, so the tool intervals - and the window bounds come from this one source; scripting it is what makes - the span arithmetic deterministic. + INJECTED, never monkeypatched onto the module. Pi derives every wall stamp + from its turn clock now, so patching `agent_module.datetime` would no + longer reach it: the tests would quietly start measuring the real clock and + pass by accident instead of failing. Injection also puts the "one clock per + turn" lifetime in the constructor signature where it can be read. """ - at_ms = 0.0 + def __init__(self, at_ms: float = 0.0) -> None: + self.at_ms = at_ms - @staticmethod - def now(tz=None): - return _SPAN_BASE + timedelta(milliseconds=_SteppedClock.at_ms) + def now(self) -> datetime: + return _SPAN_BASE + timedelta(milliseconds=self.at_ms) def _turn_end_payload(): @@ -1248,26 +1255,25 @@ class TestGenerationWindowsTileTheTurn: which is the half that carries the weight. """ - def _two_turns(self, monkeypatch): - monkeypatch.setattr(agent_module, "datetime", _SteppedClock) - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m") - _SteppedClock.at_ms = 0 + def _two_turns(self): + clock = _SteppedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) state.on_turn_start() - _SteppedClock.at_ms = 1000 + clock.at_ms = 1000 state.on_turn_end(_turn_end_payload()) - _SteppedClock.at_ms = 1600 + clock.at_ms = 1600 state.on_turn_start() - _SteppedClock.at_ms = 2000 + clock.at_ms = 2000 state.on_turn_end(_turn_end_payload()) return [m for m in state.messages if m.role == "assistant"] - def test_the_second_window_abuts_the_first(self, monkeypatch): - messages = self._two_turns(monkeypatch) + def test_the_second_window_abuts_the_first(self): + messages = self._two_turns() assert len(messages) == 2 assert messages[1].started_at == messages[0].completed_at - def test_the_inter_turn_gap_is_inside_a_window_rather_than_unaccounted(self, monkeypatch): - messages = self._two_turns(monkeypatch) + def test_the_inter_turn_gap_is_inside_a_window_rather_than_unaccounted(self): + messages = self._two_turns() # 1000 -> 2000, which includes the 600ms between `turn_end` and the # next `turn_start`. Untiled this reported 400ms and lost the 600. assert messages[1].generation_duration_ms == pytest.approx(1000.0) @@ -1284,42 +1290,41 @@ class TestToolSpansSurviveTheTurnBoundary: changes land in one commit, reset first. """ - def _run(self, monkeypatch): - monkeypatch.setattr(agent_module, "datetime", _SteppedClock) - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m") + def _run(self): + clock = _SteppedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) # The resolved telemetry leaves the state via ToolEnd; the identity # case below reconciles against what was RECORDED, not against the # clock the test scripted. resolved: list[Any] = [] state.bind(lambda e: resolved.append(e.tool) if isinstance(e, ToolEndEvent) else None) - _SteppedClock.at_ms = 0 state.on_turn_start() - _SteppedClock.at_ms = 100 + clock.at_ms = 100 state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) - _SteppedClock.at_ms = 1000 + clock.at_ms = 1000 state.on_turn_end(_turn_end_payload()) - _SteppedClock.at_ms = 1500 + clock.at_ms = 1500 state.on_tool_execution_end({"toolCallId": "c1", "result": "ok"}) # closes in the GAP - _SteppedClock.at_ms = 1600 + clock.at_ms = 1600 state.on_turn_start() - _SteppedClock.at_ms = 2000 + clock.at_ms = 2000 state.on_turn_end(_turn_end_payload()) return resolved, [m for m in state.messages if m.role == "assistant"] - def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self, monkeypatch): - _, messages = self._run(monkeypatch) + def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self): + _, messages = self._run() # Window 2 tiles 1000 -> 2000. c1 ran for 1000 -> 1500 of it, so 500ms # is model time. With the reset left at `turn_start` this reads 1000.0. assert messages[1].generation_duration_ms == pytest.approx(500.0) - def test_the_call_is_subtracted_from_exactly_one_window(self, monkeypatch): - _, messages = self._run(monkeypatch) + def test_the_call_is_subtracted_from_exactly_one_window(self): + _, messages = self._run() # Window 1 bounded c1 at its own close (100 -> 1000); window 2 takes # only the remainder. assert messages[0].generation_duration_ms == pytest.approx(100.0) assert messages[1].generation_duration_ms == pytest.approx(500.0) - def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self, monkeypatch): + def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self): """generation + UNION(tool) accounts for the whole span, to the ms. This is the assertion the golden corpus CANNOT make: `_scrub.py` masks @@ -1332,7 +1337,7 @@ def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self, monke """ from coder_eval.timing import busy_ms - resolved, messages = self._run(monkeypatch) + resolved, messages = self._run() lo, hi = messages[0].started_at, messages[1].completed_at generation_ms = sum(m.generation_duration_ms or 0.0 for m in messages) command = next(c for c in resolved if c.tool_id == "c1") @@ -1340,20 +1345,19 @@ def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self, monke assert generation_ms + tool_ms == pytest.approx((hi - lo).total_seconds() * 1000.0) - def test_a_turn_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self, monkeypatch): - monkeypatch.setattr(agent_module, "datetime", _SteppedClock) - state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m") - _SteppedClock.at_ms = 0 + def test_a_turn_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self): + clock = _SteppedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) state.on_turn_start() - _SteppedClock.at_ms = 1000 + clock.at_ms = 1000 state.on_turn_end(_turn_end_payload()) mark_after_flush = state.gen_mark - _SteppedClock.at_ms = 1600 + clock.at_ms = 1600 state.on_turn_start() - _SteppedClock.at_ms = 1700 + clock.at_ms = 1700 state.on_tool_execution_start({"toolCallId": "c2", "toolName": "bash", "args": {}}) - _SteppedClock.at_ms = 1900 + clock.at_ms = 1900 state.close_open_tools() # crash/timeout orphan sweep — no message appended # Published nothing, so tiling past it would hand its time to whichever @@ -1363,3 +1367,50 @@ def test_a_turn_that_never_finishes_neither_advances_the_mark_nor_clears_the_spa assert [(s, e) for s, e in state.turn_tool_spans] == [ (_SPAN_BASE + timedelta(milliseconds=1700), _SPAN_BASE + timedelta(milliseconds=1900)) ] + + +class TestClockIsFreshPerTurn: + """A retried turn must not inherit the crashed turn's clock. + + `TurnClock` anchors once and derives every later stamp from that anchor, so + one surviving a retry would stamp the new turn against the old turn's wall + origin — and over a long run accumulate drift against real wall time. The + lifetime is structural (the clock is built with the turn state, and the + state is built per `communicate()`), which is exactly the kind of property + that stays true only while someone is checking. + """ + + async def test_a_turn_after_a_crash_is_anchored_to_a_fresh_clock(self, patch_exec, tmp_path): + agent = _agent() + patch_exec(_FakeProcess([], returncode=1, stderr=b"boom: bad model")) + with pytest.raises(AgentCrashError): + await _run(agent, tmp_path) + crashed_clock = agent # the state is gone; only the agent survives a crash + + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await crashed_clock.communicate("try again") + + # The recovered turn measured a real window of its own, rather than one + # anchored before the crash — which a stale clock would have produced + # as an inflated first generation. + windows = [m for m in record.messages if m.role == "assistant" and m.generation_duration_ms is not None] + assert windows + for message in windows: + assert message.completed_at >= message.started_at + assert message.generation_duration_ms < 60_000, "a window spanning the crashed turn means a stale clock" + + async def test_the_agent_retains_no_clock_between_turns(self, patch_exec, tmp_path): + """Nothing to reset, because nothing survives — the structural half. + + The clock is reachable only through the turn state, and the turn state + is a local of `communicate()`. If either were ever hoisted onto the + agent (a plausible refactor — several other fields are), the next turn + would silently inherit the previous turn's anchor and no assertion + about a single turn's numbers would notice. + """ + agent = _agent() + patch_exec(_FakeProcess(HAPPY_STREAM)) + await _run(agent, tmp_path) + + leaked = [name for name, value in vars(agent).items() if isinstance(value, _PiTurnState | TurnClock)] + assert not leaked, f"a turn's clock outlived its turn via {leaked}" diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py index a55143cb4..adb6384c7 100644 --- a/tests/test_timing_close_window.py +++ b/tests/test_timing_close_window.py @@ -128,3 +128,52 @@ def test_mark_is_keyword_only_and_has_no_default(self): close_window(MARK, _at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[misc] with pytest.raises(TypeError): close_window(now=_at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[call-arg] + + +class TestTurnClock: + """One (wall, monotonic) pair per turn, every later stamp derived from it.""" + + def test_successive_reads_never_go_backwards(self): + from coder_eval.timing import TurnClock + + clock = TurnClock() + stamps = [clock.now() for _ in range(50)] + assert stamps == sorted(stamps) + + def test_a_derived_stamp_advances_by_the_monotonic_delta(self): + import time as _time + + from coder_eval.timing import TurnClock + + clock = TurnClock() + before = clock.now() + mono_before = _time.monotonic() + while _time.monotonic() - mono_before < 0.01: + pass + elapsed_ms = (_time.monotonic() - mono_before) * 1000.0 + derived_ms = (clock.now() - before).total_seconds() * 1000.0 + assert derived_ms == pytest.approx(elapsed_ms, abs=5.0) + + def test_a_fresh_clock_anchors_on_its_own_pair(self): + """Each clock holds its OWN (wall, monotonic) origin — the per-turn part. + + Note what this deliberately does NOT assert: that two clocks report + different times. They should AGREE, and closely, because both derive + from the same monotonic source — re-anchoring exists to correct drift + against real wall time, not to introduce an offset. An earlier version + of this test asserted `second.now() != first.now()`; that passed only + on sub-microsecond skew between the two constructors' reads, so it was + flaky under load and asserted the opposite of the design. + """ + import time as _time + + from coder_eval.timing import TurnClock + + first = TurnClock() + mono = _time.monotonic() + while _time.monotonic() - mono < 0.005: + pass + second = TurnClock() + + assert second._mono0 > first._mono0 + assert second._wall0 >= first._wall0 From d0d30f24bf67caa96f24123395c6a20cfc944a55 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 19:36:24 -0700 Subject: [PATCH 24/54] =?UTF-8?q?docs(harness):=206/6=20=E2=80=94=20the=20?= =?UTF-8?q?timing=20architecture=20as=20it=20now=20stands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Corrects the Pi row, which still claimed a window opening at its own `turn_start`, and adds two rows the table never had: which clock basis each harness's recorded stamps come from, and which of them build their window through the shared helper. The identity row gets a footnote rather than a bare "yes". Its committed sensor is one-sided — it catches a bucket claiming more time than the turn contains and nothing about one claiming less — and it cannot see the magnitudes at all, because the golden scrubber masks every timing value to a placeholder. A doc that asserts an invariant should say what actually checks it. Folds in the time-to-first-token design, which was living in an uncommitted scratch note that had gone stale in four separate ways — including naming a file that never existed. The design is recorded as rules with reasons (name it `first_delta_latency_ms`, never a fifth bucket, first delta of ANY kind, never 0.0) and deliberately without a table of private attribute names, since transcribing those is how the note died: one of them was deleted in 5/6. Nothing is implemented here. No field, no reducer change, no model change. Co-Authored-By: Claude Opus 5 (1M context) --- docs/agents/HARNESS_PARITY.md | 95 +++++++++++++++++++++++++++++++++-- 1 file changed, 90 insertions(+), 5 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 121c6d17b..7ef71f122 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -24,7 +24,7 @@ wall clock its numbers account for. | Field | claude-code | codex | antigravity | opencode | pi | |---|---|---|---|---|---| -| `generation_duration_ms` source | harness clock: previous SDK event → this message | SDK item stamps, minus tool execution inside the window | harness clock: previous flush → this flush, minus tool execution inside the window | harness clock per CLI step, minus tool execution inside the step | harness clock per CLI turn, minus tool execution inside the turn | +| `generation_duration_ms` source | harness clock: previous SDK event → this message | SDK item stamps, minus tool execution inside the window | harness clock: previous flush → this flush, minus tool execution inside the window | harness clock: previous `step_finish` → this one, minus tool execution inside the window | harness clock: previous `turn_end` → this one, minus tool execution inside the window | | what the **first** window covers | turn start → msg0, so dispatch + TTFT are INSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | turn start → first flush, so dispatch + TTFT are INSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | | `harness_startup_ms` (turn head) | 0.0 — the window above already covers it | ~3.1 s — CLI boot fused with TTFT | 0.0 — the window above already covers it | ~2.5 s — CLI boot fused with TTFT | ~0.23 s — CLI boot fused with TTFT | | `harness_teardown_ms` (turn tail) | ~1.3 s | ~13 ms | ~7 ms | ~26 ms | ~19 ms | @@ -32,16 +32,29 @@ wall clock its numbers account for. | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | | `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID`; `None` when absent | CLI `responseId`; `None` when absent | -| `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes | yes | yes | yes | yes | +| `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | +| clock basis for recorded stamps | monotonic duration, wall bounds | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | +| window built by `timing.py::close_window` | no — see below | yes | yes | yes | yes | + +[^identity]: "yes" is load-bearing but the committed sensor is one-sided. +`tests/_fixtures/golden_streams/_scrub.py` asserts only `overshoot <= …`, so it +catches a bucket claiming MORE time than the turn contains and says nothing +about one claiming less — an unmeasured bucket passes every test in the suite. +Worse, that suite cannot see the magnitudes at all: `SCRUB_KEYS` masks +`generation_duration_ms` and both bounds to a placeholder, so a golden snapshot +records that a window was measured, never what it measured. The two-sided check +is `scripts/timing/decompose_run.py --max-residual-pct N`, which gates on each +turn's `|residual|` as a share of its own wall clock. It is report-only and +nothing runs it on a schedule; run it by hand against real `task.json` files. **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** All five harnesses can have tool execution inside a generation window, and all five subtract it. Four interleave it structurally: Antigravity reports a `Step` for the tool and only a later `usage_metadata` `Step` cuts the message; Codex's message window is seeded from -the first item's start and extended to the last item's completion; OpenCode -opens its window at `step_start` and closes it at `step_finish`, and Pi at -`turn_start` / `turn_end`, with every tool call running inside. In each the +the first item's start and extended to the last item's completion; OpenCode and Pi +tile: each window opens where the previous `step_finish` / `turn_end` closed it +and runs to the next, with every tool call in between running inside. In each the span between the recorded bounds legitimately CONTAINS tool time that the model did not spend generating, so each subtracts it — the **union** of the closed tool intervals clipped to the window (`coder_eval/timing.py::busy_ms`), never the sum, because @@ -55,6 +68,37 @@ the whole measured window was that tool running, so the recorded generation time is legitimately `0.0`. That is a measurement, not a placeholder — `None` is what "never measured" looks like. +**One helper builds four of the five windows.** Codex, OpenCode, Pi and +Antigravity call `coder_eval/timing.py::close_window`, which is the whole +arithmetic in one place: tile from the mark, keep a stamp that went backwards +from inverting the span, bound the calls still open at the boundary, subtract +the union clipped to the window, clamp at zero. It had been copy-pasted four +times, and Pi shipped a variant of it that measured from its own turn start — +so every inter-turn gap fell into no bucket, and nothing failed, because the +identity above is asserted on one side only. **CE061** now requires any module +in `agents/` that publishes a measured `generation_duration_ms` to import the +helper. claude-code is the single documented exception and carries the only +`# noqa: CE061`: it subtracts once at finalization (below) rather than per +flush, a shape `close_window` cannot take without a mode flag. + +**Two clock bases remain, and the row above says which.** Antigravity and Pi +derive every recorded wall stamp from one `TurnClock` per turn, so a turn's +bounds and the tool spans subtracted from them cannot disagree. Antigravity +needed it: its span was monotonic while its tool intervals were wall, which is +the only reason its window could go negative, and the clamp that caught it was +indistinguishable from a real instant generation. Pi needed it for a different +reason — its stamps were naive-local, so a DST transition or an NTP step inside +a turn lands directly in a generation window. + +Codex and OpenCode are **not** converted and the hazard is narrowed rather than +removed. Their tool spans are the CLI's own epoch-millisecond stamps +(`codex_agent.py::_ms_to_dt`, `opencode_agent.py::_epoch_ms_to_dt`), which +cannot be re-derived host-side; converting only the window bounds would put two +bases inside one `busy_ms` subtraction, relocating the defect instead of +removing it. Both therefore keep the naive-local exposure. Deadlines on every +harness stay on raw `time.monotonic()` and must — a deadline may not move when +the wall clock steps. + **`claude-code` subtracts at finalization, not as it flushes.** It was once exempt entirely, on the premise that because it marks the end of the previous SDK event and reads again when the next message arrives, a tool's execution @@ -176,6 +220,47 @@ which is the case CE060 cannot see (it requires the kwarg to be present, not non-`None` at runtime). OpenCode tiles its windows contiguously too, so it is the other harness where a missing id can still collapse a turn. +### Time to first token is not measured + +Nothing records it today. There is no `ttft` or `first_token` symbol anywhere +in `src/`, `evalboard/`, `docs/` or `tests/`, and it **cannot be derived from +what is stored**: `generation_duration_ms` is the whole window, and the latency +in question is a sub-interval of it. This section is the design, so the next +person to want it does not re-derive it. Nothing below is implemented. + +**The mark is the measure-from point, and every reducer already keeps one.** +Each one records the moment its current generation window opened — which is +exactly what a latency is measured from. Read the current attribute off `src/` +rather than trusting a table here; the last note that transcribed those names +went stale in precisely that way. + +**The first-delta signal already exists in every reducer.** claude-code has raw +`content_block_delta` (already delivered — `include_partial_messages=True`), +codex `item/agentMessage/delta`, antigravity `step.content_delta`, OpenCode the +text part event, Pi `text_delta`. + +Four rules, each of which changes what gets built: + +- **Name it `first_delta_latency_ms`, never `ttft_ms`.** Four harnesses' windows + tile, so the mark is the *previous step's close* and the interval fuses + queueing and tool time. That is queue latency, not prefill latency. Only + claude-code's `message_start` sits near "the request went out". This is the + same rule the head and tail already follow: a field is named for the interval + it MEASURES, never for what it contains. +- **It is never a fifth bucket.** It is a sub-interval of head + first window. + Adding it to the four-bucket identity breaks the disjointness the whole + design rests on. Report it beside the identity, never inside it. +- **Take the first delta of ANY kind**, not the first visible-text delta. The + codex, OpenCode and Pi handlers ignore thinking deltas, so a reasoning-heavy + turn would report its first token late by the entire thinking phase. +- **Never write `0.0` for "not measured"** (CE058). Use `None` when no delta + arrived. + +The verification hook is `tests/_fixtures/golden_streams/_scrub.py`'s +`assert_timing_captured`, where a floor belongs; the five +`tests/_fixtures/golden_streams/*_fixtures.py` modules already carry the deltas +needed to drive it. + ### Known divergences - **Delegate (`delegate-sdk`, out of tree)** records `duration_ms` but no From b2a8c30bf027aa6433bfa2d41f7352e27773a166 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 19:54:34 -0700 Subject: [PATCH 25/54] fix: code review fixes for timing-architecture-standardization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A duplicate `turn_end` / `step_finish` with no intervening start republished the previous window in full. `close_window`'s `min(mark, item_start)` exists to stop a backwards clock from inverting a span, but a start stamp left in place after its turn was PUBLISHED is not a backwards clock — it is a stale value sitting before the mark, so the guard reopened the next window back at the previous turn's start. Reproduced by driving the real state object: 3000 ms of generation published for a 2000 ms turn, which `decompose_run.py` would read as a large negative residual and the evalboard would simply sum. The stamp is now cleared at the flush alongside the mark and the span list, for the same reason they are: it has been spent. Regression test on both harnesses. `close_window`'s own docstring had gone stale in the way it was written to prevent. Phase 2 wrote it, then 3/6 gave pi the mark it said pi lacked and 5/6 migrated the antigravity window it said the signature could not express — so the shared helper disagreed with the parity doc about which harnesses use it. The gate script now counts turns it cannot time at all. They were the one exclusion with no tally, in a file built around not discarding evidence silently. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/timing/decompose_run.py | 14 +++++++++--- src/coder_eval/agents/opencode_agent.py | 9 ++++++++ src/coder_eval/agents/pi_agent.py | 8 +++++++ src/coder_eval/timing.py | 29 +++++++++++++------------ tests/test_custom_lint.py | 4 ++-- tests/test_opencode_agent.py | 26 ++++++++++++++++++++++ tests/test_pi_agent.py | 26 ++++++++++++++++++++++ 7 files changed, 97 insertions(+), 19 deletions(-) diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index 42a634c52..dbfd2560e 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -151,6 +151,11 @@ def main(argv: list[str]) -> int: by_harness: dict[str, list[tuple[Path, int, tuple[float, float, float, float, float]]]] = defaultdict(list) skipped_crashed = 0 skipped_no_window = 0 + # A turn `_turn_buckets` cannot place on the timeline at all (no numeric + # `duration_seconds`). Counted rather than silently dropped, for the same + # reason as the two above: an exclusion nobody can see understates how much + # of the corpus the gate actually looked at. + skipped_untimed = 0 for path in args.task_json: try: record = json.loads(path.read_text(encoding="utf-8")) @@ -179,8 +184,10 @@ def main(argv: list[str]) -> int: skipped_no_window += int(no_window) continue buckets = _turn_buckets(turn) - if buckets is not None: - by_harness[harness].append((path, index, buckets)) + if buckets is None: + skipped_untimed += 1 + continue + by_harness[harness].append((path, index, buckets)) if not by_harness: print("no timed turns found", file=sys.stderr) @@ -249,7 +256,8 @@ def main(argv: list[str]) -> int: print(f"worst single-turn |residual| = {worst_turn:.3f}ms") print( f"skipped: {skipped_crashed} crashed, {skipped_no_window} no-window " - f"(a turn can be both), {skipped_short} short (< {args.min_turn_ms:.0f}ms)" + f"(a turn can be both), {skipped_untimed} untimed, " + f"{skipped_short} short (< {args.min_turn_ms:.0f}ms)" ) if not gateable_total: diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index eb2f16cb6..f8d73e6d7 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -750,6 +750,15 @@ def on_step_finish(self, part: dict[str, Any]) -> None: # it, and only with it — see `on_step_start`. self.gen_mark = completed self.step_tool_spans = [] + # And so is this step's own start stamp, because it has now been SPENT. + # It is passed to `close_window` as `item_start`, whose `min()` pulls + # the window open to cover it; left in place, a second `step_finish` + # with no intervening `step_start` would reopen the next window back at + # the previous step's start and publish that whole span a second time. + # The `min()` still defends a genuinely OPEN step against a backwards + # clock, which is what it is for — this reducer's stamps are raw + # `datetime.now()` and are not on a `TurnClock`. + self.step_started_at = None self.emit( TurnEndEvent( task_id=self.task_id, diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 51d089a00..d7d78fcdb 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -646,6 +646,14 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: # it, and only with it — see `on_turn_start`. self.gen_mark = completed self.turn_tool_spans = [] + # And so is this turn's own start stamp, because it has now been SPENT. + # It is passed to `close_window` as `item_start`, whose `min()` pulls + # the window open to cover it; left in place, a second `turn_end` with + # no intervening `turn_start` — a duplicate or replayed line, which + # this reducer promises to survive — would reopen the next window back + # at the previous turn's start and publish that whole span a second + # time. Reproduced: 3000 ms of generation for a 2000 ms turn. + self.turn_started_at = None self.emit( TurnEndEvent( task_id=self.task_id, diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 173c591fb..d478e58b0 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -116,20 +116,21 @@ def close_window( ) -> tuple[datetime, float]: """Close one generation window at ``now``: its ``(started, generation_ms)``. - The shape the tiling harnesses had copy-pasted; codex, opencode and pi call - it today. Antigravity is not merely unmigrated — it derives its span from - the MONOTONIC clock while unioning WALL-clock tool spans, which this - signature cannot express — and claude-code subtracts once at finalization - across every emission instead. - - ``mark`` is where the window opens — normally the previous flush's close, - which is what makes the windows TILE the turn contiguously instead of - leaving the model time that PRODUCED an item attributed to nothing. It is - keyword-only and has NO default so that no reducer can open a window - without stating what it tiles from. That constrains the call SHAPE, not the - VALUE: pi still passes its own turn start, so its inter-turn gaps are still - in no bucket until it grows a mark of its own. The signature makes the - omission visible; it does not fix it. + The shape four reducers had copy-pasted. Codex, opencode, pi and + antigravity all call it; claude-code is the one exception and carries the + only ``# noqa: CE061``, because it subtracts tool time once at finalization + across every emission rather than per flush — a call issued by an earlier + emission is still running when the next window closes. + + ``mark`` is where the window opens: the previous flush's close, which is + what makes the windows TILE the turn contiguously instead of leaving the + model time that PRODUCED an item attributed to nothing. It is keyword-only + and has NO default so that no reducer can open a window without stating + what it tiles from — which is the defect pi shipped with, measuring from + its own turn start so that every inter-turn gap fell into no bucket at all. + Note what the signature does and does not buy: it constrains the call + SHAPE, not the VALUE. A reducer can still pass the wrong mark; what it + cannot do is fail to have one. ``item_start`` is this emission's own first stamp, when the harness has one. The ``min()`` against ``mark`` is the tiling defense and nothing else: diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index ea5f9a5e2..4431d2a3b 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4023,8 +4023,8 @@ def test_keys_on_the_helper_name_rather_than_a_literal(self): assert _helper.__name__ == rule_mod._HELPER def test_the_real_agents_tree_is_clean(self): - # After the two suppressions: claude-code's permanent one, and - # antigravity's temporary one pending its 5/6 migration. + # After claude-code's single permanent suppression. Antigravity + # carried a temporary one until it moved onto `close_window`. import pathlib from tests.lint.rules.ce061_window_via_close_window import WindowViaCloseWindow diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 458a51bf8..1d2f9e762 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -2097,6 +2097,32 @@ def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self, monke assert generation_ms + tool_ms == pytest.approx((hi - lo).total_seconds() * 1000.0) + def test_a_duplicate_step_finish_does_not_republish_the_previous_window(self, monkeypatch): + """A spent `step_started_at` must not seed the next window. + + `close_window`'s `min(mark, item_start)` pulls the window open to cover + the item's own start. That is the backwards-clock defence — which this + reducer genuinely needs, since its stamps are raw `datetime.now()` and + not on a `TurnClock`. But a start stamp left in place after its step was + published is not a backwards clock: it is a stale value BEFORE the mark, + so the guard reopens the next window at the previous step's start and + publishes that whole span again. Reproduced on Pi's identical twin + before the fix: 3000 ms of generation for a 2000 ms turn. + """ + monkeypatch.setattr(agent_module, "datetime", _SteppedClock) + state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") + _SteppedClock.at_ms = 0 + state.on_step_start({"messageID": "m1"}) + _SteppedClock.at_ms = 1000 + state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) + _SteppedClock.at_ms = 2000 + state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) + + messages = [m for m in state.messages if m.role == "assistant"] + assert len(messages) == 2 + assert messages[1].started_at == messages[0].completed_at + assert sum(m.generation_duration_ms or 0.0 for m in messages) == pytest.approx(2000.0) + def test_a_step_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self, monkeypatch): monkeypatch.setattr(agent_module, "datetime", _SteppedClock) state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index cf12ab3c4..baa2ce952 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -1345,6 +1345,32 @@ def test_the_four_bucket_identity_closes_exactly_across_the_boundary(self): assert generation_ms + tool_ms == pytest.approx((hi - lo).total_seconds() * 1000.0) + def test_a_duplicate_turn_end_does_not_republish_the_previous_window(self): + """A spent `turn_started_at` must not seed the next window. + + `close_window`'s `min(mark, item_start)` pulls the window open to cover + the item's own start. That is the backwards-clock defence, but a start + stamp left in place after its turn was published is not a backwards + clock — it is a stale value BEFORE the mark, so the guard reopens the + next window at the previous turn's start and publishes that whole span + again. Reproduced before the fix: 3000 ms of generation for a 2000 ms + turn. This reducer promises to survive a malformed stream, and Pi's CLI + retries internally, so a duplicate or replayed `turn_end` is a transport + hiccup rather than a hypothetical. + """ + clock = _SteppedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) + state.on_turn_start() + clock.at_ms = 1000 + state.on_turn_end(_turn_end_payload()) + clock.at_ms = 2000 + state.on_turn_end(_turn_end_payload()) # no intervening `turn_start` + + messages = [m for m in state.messages if m.role == "assistant"] + assert len(messages) == 2 + assert messages[1].started_at == messages[0].completed_at + assert sum(m.generation_duration_ms or 0.0 for m in messages) == pytest.approx(2000.0) + def test_a_turn_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self): clock = _SteppedClock() state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) From e1c4af9f96e6441b8b879c973aa4101f3d46bfe6 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 19:54:50 -0700 Subject: [PATCH 26/54] test(harness): stamp the two codex fixtures that timed themselves with now() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `c_reasoning_placeholder` and `h_no_turn_completed_crash` injected no item stamps, so `_flush_message` took `_ms_to_dt(None)` for BOTH window bounds — two adjacent `datetime.now()` reads. They collide at microsecond resolution often enough that `assert_timing_captured`'s `completed_at > started_at` failed roughly one run in twenty under parallel load, naming a different scenario each time and giving no hint of the cause. Two separate reviewers of this branch hit it on two different scenarios. Real bounds fix it, at the cost of joining `FICTIONAL_DURATIONS`: integer-ms SDK stamps cannot reconcile against a replay that runs in under a millisecond. That trade is stated where the set is defined. It costs little — a window of width zero reconciled trivially, so the identity check it gives up was near-vacuous, and what replaces it is a stable bounds-span assertion. Co-Authored-By: Claude Opus 5 (1M context) --- .../golden_streams/codex_fixtures.py | 24 ++++++++++++++++--- tests/test_agent_golden_master.py | 12 ++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/tests/_fixtures/golden_streams/codex_fixtures.py b/tests/_fixtures/golden_streams/codex_fixtures.py index a04def75d..e0a157fdf 100644 --- a/tests/_fixtures/golden_streams/codex_fixtures.py +++ b/tests/_fixtures/golden_streams/codex_fixtures.py @@ -221,9 +221,21 @@ def _build_catalogue() -> list[CodexScenario]: CodexScenario( name="c_reasoning_placeholder", notifications=[ - _item("item/completed", _reasoning(text="")), + # Real bounds, and they are load-bearing rather than decorative: + # with none, `_flush_message` takes `_ms_to_dt(None)` for BOTH + # ends, which is two adjacent `datetime.now()` reads. Those + # collide at microsecond resolution often enough that this + # scenario failed `assert_timing_captured`'s + # `completed_at > started_at` roughly one run in twenty under + # parallel load, naming a different scenario each time. + _item("item/completed", _reasoning(text=""), started_at_ms=_T0_MS, completed_at_ms=_T0_MS + 40), _delta("final answer"), - _item("item/completed", _agent_message("final answer")), + _item( + "item/completed", + _agent_message("final answer"), + started_at_ms=_T0_MS + 40, + completed_at_ms=_T0_MS + 300, + ), _token_usage(inp=100, out=50, cached=8, reasoning=20), _turn_completed(), ], @@ -294,7 +306,13 @@ def _build_catalogue() -> list[CodexScenario]: name="h_no_turn_completed_crash", notifications=[ _delta("partial"), - _item("item/completed", _agent_message("partial")), + # Bounded for the same reason as (c) above. + _item( + "item/completed", + _agent_message("partial"), + started_at_ms=_T0_MS, + completed_at_ms=_T0_MS + 200, + ), _token_usage(inp=100, out=40, cached=8), ], expects=AgentCrashError, diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index 7655a5daa..18d5fe892 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -78,12 +78,24 @@ def _expect_window(harness: str, scenario_name: str) -> bool: # runs in well under one. No rebasing closes that; the agent's own clock would # have to be faked too. Everything else — every claude, antigravity and pi # scenario, and the codex/opencode ones that inject nothing — is checked. +# +# The last two entries were ADDED to buy stability, and the trade is worth +# stating. They previously injected NO stamps at all, so `_flush_message` took +# `_ms_to_dt(None)` for both window bounds — two adjacent `datetime.now()` +# reads, which collide at microsecond resolution often enough that +# `assert_timing_captured`'s `completed_at > started_at` failed roughly one run +# in twenty under parallel load, naming a different scenario each time. Their +# identity check was near-vacuous anyway (a window of width zero reconciles +# trivially), so giving them real bounds trades that for a stable, meaningful +# bounds-span assertion. FICTIONAL_DURATIONS: frozenset[str] = frozenset( { "codex_b_command_execution", # 250 ms command + 150 ms generation + "codex_c_reasoning_placeholder", # 300 ms of item time — see below "codex_d_cross_flush_is_error", # 400 ms command "codex_e_orphan_tool", # command started, never completed "codex_f_collab_fallback", # 900 ms collab wait + "codex_h_no_turn_completed_crash", # 200 ms of item time — see below "opencode_b_tool_call_resolved", # 17 ms tool interval } ) From 702935e49e285f6cb66662cadf6f65dd47e5a486 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 19:54:56 -0700 Subject: [PATCH 27/54] docs(harness): register that the golden corpus cannot see a timing value move A whole phase of the timing plan was written expecting the golden master to go red when generation numbers changed. It never did: the scrubber masks every timing value, and the one assertion that reads magnitudes is one-sided. Record what closing it would actually take, since it is more than a tolerance constant. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/harness-candidates.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 2bc1fbbd2..f5048417a 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -674,3 +674,25 @@ divergences, so the deferred-work record is one place. Measurements in CE054-shaped but over a `str` description rather than a key. The cheap version is to fix the sentence in the next change that touches the model. Caught in: the CE060 / antigravity `message_id` final review. + +- [ ] **The golden corpus pins that a timing value EXISTS, never what it is.** + `tests/_fixtures/golden_streams/_scrub.py::SCRUB_KEYS` masks + `generation_duration_ms`, `started_at`, `completed_at` and both + `execution_*_at` to a placeholder, and the one assertion that does look at + magnitudes (`assert_timing_captured`'s four-bucket check) is an UPPER BOUND — + it catches a bucket claiming more time than the turn contains and says + nothing about one claiming less. So the committed suite cannot see a + per-harness generation number move at all, in either direction. Not + hypothetical: a whole phase of the timing plan was written on the premise + that changing those numbers would turn the golden master red, and it never + did. The two-sided check exists (`scripts/timing/decompose_run.py + --max-residual-pct`) but runs only against live `task.json` files, by hand. + Not cheap to guard: porting the two-sided residual into `_scrub.py` means + deciding a per-scenario tolerance for replays whose real wall clock is under + a millisecond while their SDK stamps declare hundreds — the same problem + `FICTIONAL_DURATIONS` already exempts six scenarios from, so the honest + version needs those scenarios to fake the agent's own clock too, not just + their item stamps. Interim cover is the per-reducer ms-exact identity test + added on pi and opencode + (`test_the_four_bucket_identity_closes_exactly_across_the_boundary`). + Caught in: the timing-architecture-standardization final review. From 3baaa5311ba5e4f82dd7a05bbc206172d05b09b0 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 21:59:59 -0700 Subject: [PATCH 28/54] =?UTF-8?q?feat(timing):=201/7=20=E2=80=94=20a=20com?= =?UTF-8?q?mitted,=20ms-exact=20magnitude=20sensor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nothing in the suite could see a timing VALUE move. The golden corpus masks `generation_duration_ms`, both window bounds, both `execution_*_at` stamps and both head/tail fields to a placeholder, and its identity check is one-sided (`overshoot <= ...`), so an UNDERCOUNT — the defect class this area keeps producing — passed every test. A prototype of the next phase changed published generation figures on two harnesses and left all 5340 tests green. `tests/test_timing_identity_contract.py` is that sensor. Each of the five harnesses drives its own reducer off a clock the test moves by hand, then feeds the messages and commands it produced through a real `EventCollector` — the same seam production measures the head and tail at — and asserts head + Σ generation + UNION(tool) + tail == the scripted span with `pytest.approx`, an equality and so two-sided. Magnitudes are real only where a scripted clock makes them real, which is why this cannot live in `_scrub.py`: those replays run in ~0.3 ms of synthetic wall clock, where a relative bound passes essentially anything. That file gains one docstring paragraph saying where the two-sided check went and why, and no code change. `test_the_sensor_sees_a_window_that_stops_tiling` is the gating mutation check, committed rather than attested: it re-drives the pi case with tiling defeated — the defect pi actually shipped — and asserts both the exact 600 ms the mutation loses and that the identity assertion fires. `test_every_built_in_harness_has_a_case` derives its set from `AgentKind` (not the open registry, which a third-party plugin also populates), so a sixth built-in harness fails here rather than shipping unmeasured. `coder_eval.timing.union_ms` extracts the `min`/`max`/`busy_ms` tail the golden sensor and the live residual gate had each copied. The shared corpus gains a `union_cases` array replayed by BOTH suites — TypeScript through `toolExecutionMs`, which derives its own extent and was the untested half. CI gets the live two-sided gate at no infrastructure cost: the smoke-pass step already runs a real agent and leaves real `task.json` files, so `decompose_run.py --max-residual-pct 5` is one step against them. It covers claude-code only (`experiments/default.yaml`), which the step name says. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- .github/workflows/pr-checks.yml | 15 + .../lib/__tests__/timing-union-parity.test.ts | 41 +- scripts/timing/decompose_run.py | 10 +- src/coder_eval/timing.py | 22 + tests/_fixtures/golden_streams/_scrub.py | 14 +- tests/_fixtures/timing_union_cases.json | 61 +- tests/test_timing_close_window.py | 70 ++- tests/test_timing_identity_contract.py | 558 ++++++++++++++++++ tests/test_timing_union_parity.py | 24 +- 9 files changed, 800 insertions(+), 15 deletions(-) create mode 100644 tests/test_timing_identity_contract.py diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index cd0150bed..f08442a23 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -593,6 +593,21 @@ jobs: test "$FAILED" = "0" || { echo "smoke-pass had unexpected failures"; exit 1; } test "$ERRORED" = "0" || { echo "smoke-pass had errors"; exit 1; } + # The four wall-clock buckets (head + generation + UNION(tool) + tail) + # must account for each turn's own duration. This is the TWO-SIDED gate: + # the committed golden sensor only catches an OVERSHOOT, so a bucket that + # claims LESS time than it should — the defect class this area keeps + # producing — passes every test in the suite. It needs live task.json + # files, which the smoke-pass run above already leaves on disk. + # + # COVERS CLAUDE-CODE ONLY: experiments/default.yaml sets type: claude-code, + # so every turn here is that harness. The other four are covered by + # tests/test_timing_identity_contract.py, which is ms-exact but synthetic. + - name: Verify timing residual (claude-code only) + run: | + .venv/bin/python scripts/timing/decompose_run.py \ + $(find runs/ci-smoke-pass -name task.json) --max-residual-pct 5 + - name: Verify smoke-fail bucket run: | F=runs/ci-smoke-fail/experiment.json diff --git a/evalboard/lib/__tests__/timing-union-parity.test.ts b/evalboard/lib/__tests__/timing-union-parity.test.ts index f2df82bb1..58bcfd391 100644 --- a/evalboard/lib/__tests__/timing-union-parity.test.ts +++ b/evalboard/lib/__tests__/timing-union-parity.test.ts @@ -25,7 +25,19 @@ interface UnionCase { expected_ms: number; } -const corpus: { cases: UnionCase[] } = JSON.parse(readFileSync(fixture, "utf8")); +// `union_cases` carries no window: the extent is the spans' own bounds. It is +// the half that pins `toolExecutionMs`, which derives that extent with its own +// min/max instead of being handed one — the Python twin is +// `coder_eval.timing.union_ms`. +interface ExtentCase { + name: string; + spans: [number, number][]; + expected_ms: number; +} + +const corpus: { cases: UnionCase[]; union_cases: ExtentCase[] } = JSON.parse( + readFileSync(fixture, "utf8"), +); describe("busyMs matches the shared union corpus", () => { test("the corpus is non-empty (a silently emptied file must not pass)", () => { @@ -138,3 +150,30 @@ describe("toolExecutionMs", () => { expect(toolExecutionMs([message([toolUse({})])])).toBe(0); }); }); + +describe("toolExecutionMs matches the shared extent corpus", () => { + test("the extent corpus is non-empty (a silently emptied file must not pass)", () => { + expect(corpus.union_cases.length).toBeGreaterThan(5); + }); + + // Each span becomes one bounded tool call on one message, so + // `toolExecutionMs` has to derive the extent itself — the one part of the + // union rule the windowed `cases` above cannot reach. Python replays the + // same array through `coder_eval.timing.union_ms`. + for (const c of corpus.union_cases) { + test(c.name, () => { + const ms = toolExecutionMs([ + message( + c.spans.map(([s, e], i) => + toolUse({ + toolUseId: `t${i}`, + execStartMs: s, + execEndMs: e, + }), + ), + ), + ]); + expect(ms).toBeCloseTo(c.expected_ms, 6); + }); + } +}); diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index dbfd2560e..75202e6a8 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -21,7 +21,7 @@ Not wired into `make`: it needs live runs, not fixtures. NOTE `scripts/` is outside the Makefile's LINT_PATHS, so this file is neither formatted nor ruff-checked — keep it small and dependency-free (stdlib plus the one shared -`busy_ms` import, so the union rule has a single definition). +`union_ms` import, so the union rule has a single definition). """ from __future__ import annotations @@ -34,7 +34,7 @@ from datetime import datetime from pathlib import Path -from coder_eval.timing import busy_ms +from coder_eval.timing import union_ms def _parse(stamp: object) -> datetime | None: @@ -49,7 +49,7 @@ def _parse(stamp: object) -> datetime | None: def _tool_ms(turn: dict) -> float: """Wall ms this turn spent executing tools — the UNION, not the sum. - The same rule `coder_eval.timing.busy_ms` applies when a harness subtracts + The same rule `coder_eval.timing.union_ms` applies when a harness subtracts tool time out of a generation window, and it has to be the same rule here or the identity does not close: Pi resolved a `Write` and a `Bash` that overlapped by 18.4 ms in one measured turn, and summing their durations @@ -64,9 +64,7 @@ def _tool_ms(turn: dict) -> float: end = _parse(command.get("execution_completed_at")) if start is not None and end is not None and end >= start: spans.append((start, end)) - if not spans: - return 0.0 - return busy_ms(spans, min(s for s, _ in spans), max(e for _, e in spans)) + return union_ms(spans) def _turn_buckets(turn: dict) -> tuple[float, float, float, float, float] | None: diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index d478e58b0..8d3dd7587 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -106,6 +106,28 @@ def busy_ms(spans: list[tuple[datetime, datetime]], lo: datetime, hi: datetime) return total + (open_end - open_start).total_seconds() * 1000.0 +def union_ms(spans: list[tuple[datetime, datetime]]) -> float: + """Wall milliseconds at least ONE span was running, over their full extent. + + ``busy_ms`` with the window set to the spans' own bounds. It exists because + two callers had copy-pasted that same ``min``/``max``/``busy_ms`` tail — + ``tests/_fixtures/golden_streams/_scrub.py`` (the golden sensor) and + ``scripts/timing/decompose_run.py`` (the live residual gate) — and they + answer the same question about the same recorded commands, so a divergence + would let one pass while the other failed. Each keeps its OWN stamp parsing + and span building, because their input shapes genuinely differ; only this + tail is shared. + + It does NOT filter ``end < start``. Both callers already drop those while + building their span lists, so guarding again here would be a second rule + about the same input in a second place; keeping it at the caller preserves + today's behaviour exactly. + """ + if not spans: + return 0.0 + return busy_ms(spans, min(s for s, _ in spans), max(e for _, e in spans)) + + def close_window( *, mark: datetime, diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index 9fb4705fd..dddd51e8a 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -5,7 +5,7 @@ from datetime import datetime from typing import Any -from coder_eval.timing import busy_ms +from coder_eval.timing import union_ms SCRUB_PLACEHOLDER = "" @@ -125,9 +125,7 @@ def _tool_union_ms(record: dict[str, Any]) -> float: end = _parse_stamp(command.get("execution_completed_at")) if start is not None and end is not None and end >= start: spans.append((start, end)) - if not spans: - return 0.0 - return busy_ms(spans, min(s for s, _ in spans), max(e for _, e in spans)) + return union_ms(spans) def _parse_stamp(value: Any) -> datetime | None: @@ -202,6 +200,14 @@ def assert_timing_captured( tool force-closed inside the tail, booked both as tool and as teardown, was found reconciling at -86% of wall clock while all 72 golden tests passed. + The check is ONE-SIDED on purpose and stays that way. A symmetric bound + would be a sensor in name only here: the replays run in ~0.3 ms of + synthetic wall clock, so ``abs(residual) <= max(0.1 ms, 20% x wall)`` + passes essentially any magnitude. The two-sided, millisecond-exact check + lives in ``tests/test_timing_identity_contract.py``, where a scripted clock + makes the magnitudes real, and the live two-sided gate is + ``scripts/timing/decompose_run.py --max-residual-pct``. + ``check_identity`` is off for the scenarios that inject their own SDK timestamps (see ``FICTIONAL_DURATIONS``): those declare integer-millisecond item durations of 17-900 ms while the replay itself takes ~0.3 ms of real diff --git a/tests/_fixtures/timing_union_cases.json b/tests/_fixtures/timing_union_cases.json index a9e0ce604..e4ac69c80 100644 --- a/tests/_fixtures/timing_union_cases.json +++ b/tests/_fixtures/timing_union_cases.json @@ -9,7 +9,13 @@ "They answer the same question about the same task.json, so a divergence", "means the evalboard and the harness disagree about how long the tools ran.", "tests/test_timing_union_parity.py and evalboard/lib/__tests__/timing-union-parity.test.ts", - "both replay this file; neither owns the numbers." + "both replay this file; neither owns the numbers.", + "", + "`union_cases` is the same question with no window given: the extent is the", + "spans' own min/max. Python replays it through coder_eval.timing.union_ms;", + "TypeScript through evalboard/lib/timing.ts::toolExecutionMs, which derives", + "that extent itself rather than being handed one — which is exactly why it", + "needs its own cases instead of being assumed to agree." ], "cases": [ { @@ -96,5 +102,58 @@ "spans": [[100, 500], [150, 550], [200, 600], [250, 650]], "expected_ms": 550 } + ], + + "union_cases": [ + { + "name": "no spans at all", + "spans": [], + "expected_ms": 0 + }, + { + "name": "one span is its own length", + "spans": [[200, 700]], + "expected_ms": 500 + }, + { + "name": "two disjoint spans add up", + "spans": [[100, 200], [400, 900]], + "expected_ms": 600 + }, + { + "name": "overlapping spans are counted once, not summed", + "spans": [[100, 600], [200, 700]], + "expected_ms": 600 + }, + { + "name": "a span fully contained in another adds nothing", + "spans": [[100, 900], [300, 400]], + "expected_ms": 800 + }, + { + "name": "adjacent spans merge without a gap", + "spans": [[100, 400], [400, 700]], + "expected_ms": 600 + }, + { + "name": "unsorted input gives the same answer as sorted", + "spans": [[600, 800], [100, 300], [200, 250]], + "expected_ms": 400 + }, + { + "name": "four concurrent calls are the union, never the sum", + "spans": [[100, 500], [150, 550], [200, 600], [250, 650]], + "expected_ms": 550 + }, + { + "name": "a zero-length span contributes nothing", + "spans": [[500, 500], [600, 900]], + "expected_ms": 300 + }, + { + "name": "the extent is the spans' own bounds, not a window", + "spans": [[10000, 10250]], + "expected_ms": 250 + } ] } diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py index adb6384c7..ec261f493 100644 --- a/tests/test_timing_close_window.py +++ b/tests/test_timing_close_window.py @@ -6,11 +6,13 @@ which pin that a given reducer feeds it the right bounds and spans. """ +import importlib.util from datetime import datetime, timedelta +from pathlib import Path import pytest -from coder_eval.timing import close_window +from coder_eval.timing import close_window, union_ms MARK = datetime(2026, 9, 11, 12, 0, 0) @@ -20,6 +22,16 @@ def _at(ms: int) -> datetime: return MARK + timedelta(milliseconds=ms) +def _load_decompose_run(): + """Import `scripts/timing/decompose_run.py`, which is not an importable package.""" + path = Path(__file__).parents[1] / "scripts" / "timing" / "decompose_run.py" + spec = importlib.util.spec_from_file_location("decompose_run_for_test", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + class TestCloseWindow: def test_no_tools_keeps_the_whole_window(self): started, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[]) @@ -130,6 +142,62 @@ def test_mark_is_keyword_only_and_has_no_default(self): close_window(now=_at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[call-arg] +class TestUnionMs: + """`union_ms` is `busy_ms` over the spans' own extent. + + Extracted because the golden sensor (`tests/_fixtures/golden_streams/_scrub.py`) + and the live residual gate (`scripts/timing/decompose_run.py`) had copied + that same `min`/`max`/`busy_ms` tail. Both answer the same question about + the same recorded commands, so the two copies could only ever agree by + hand — `test_the_two_recorded_command_readers_agree` below is the half + that pins them together. + """ + + def test_no_spans_is_zero_not_a_min_of_an_empty_sequence(self): + assert union_ms([]) == 0.0 + + def test_overlapping_spans_are_the_union_not_the_sum(self): + # Two 500 ms calls overlapping by 400 ms occupy 600 ms of wall clock. + assert union_ms([(_at(100), _at(600)), (_at(200), _at(700))]) == pytest.approx(600.0) + + def test_disjoint_spans_add(self): + assert union_ms([(_at(100), _at(200)), (_at(400), _at(900))]) == pytest.approx(600.0) + + def test_the_extent_is_the_spans_own_bounds(self): + # No window is passed, so nothing clips: a span far from the origin is + # measured in full rather than dropped as out of range. + assert union_ms([(_at(10_000), _at(10_250))]) == pytest.approx(250.0) + + def test_the_two_recorded_command_readers_agree(self): + """`_scrub.py` and `decompose_run.py` must report one tool total. + + They read the SAME `task.json` shape — the golden sensor from a dumped + record, the gate from the file on disk — and a divergence would let one + pass while the other failed on identical bytes. They keep their own + stamp parsing (the inputs differ in how they are reached); the union + tail is what this pins. + """ + from tests._fixtures.golden_streams._scrub import _tool_union_ms + + # Loaded by path: `scripts/` is deliberately not a package (it sits + # outside the Makefile's LINT_PATHS), so there is no import to make. + _tool_ms = _load_decompose_run()._tool_ms + + turn = { + "commands": [ + {"execution_started_at": _at(100).isoformat(), "execution_completed_at": _at(600).isoformat()}, + {"execution_started_at": _at(200).isoformat(), "execution_completed_at": _at(700).isoformat()}, + # Never timed: contributes nothing on either side. + {"execution_started_at": None, "execution_completed_at": None}, + # Inverted bounds: both readers drop these while BUILDING their + # span list, which is why `union_ms` does not filter them. + {"execution_started_at": _at(900).isoformat(), "execution_completed_at": _at(800).isoformat()}, + ] + } + assert _tool_union_ms(turn) == pytest.approx(600.0) + assert _tool_ms(turn) == _tool_union_ms(turn) + + class TestTurnClock: """One (wall, monotonic) pair per turn, every later stamp derived from it.""" diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py new file mode 100644 index 000000000..5b9d5cb9b --- /dev/null +++ b/tests/test_timing_identity_contract.py @@ -0,0 +1,558 @@ +"""The four-bucket identity, to the millisecond, on every harness. + + head + Σ generation + UNION(tool) + tail == the turn's own span + +This is the committed MAGNITUDE sensor, and it exists because nothing else in +the suite is one: + +* the golden corpus masks ``generation_duration_ms``, both window bounds, both + ``execution_*_at`` stamps and both head/tail fields to a placeholder + (``_scrub.py::SCRUB_KEYS``), so a snapshot records that a window was measured + and never what it measured — a timing value can move by seconds with every + golden test still green; +* ``_scrub.py::assert_timing_captured``'s own identity check is ONE-SIDED + (``overshoot <= ...``), so an UNDERCOUNT — a bucket claiming less time than + it should, which is the defect class this whole area keeps producing — passes + it silently. It cannot be made two-sided either: the replays run in ~0.3 ms of + synthetic wall clock, where a relative bound is vacuous; +* ``scripts/timing/decompose_run.py --max-residual-pct`` IS two-sided, but needs + live ``task.json`` files. + +Magnitudes are only real where a scripted clock makes them real, so each case +drives the harness's own REDUCER with a clock it moves by hand, then feeds the +messages and commands it produced through a real ``EventCollector`` — the same +seam production uses to compute the head and the tail. Every number asserted is +therefore one the harness computed, against a span the test declared. + +Three clock-injection styles are needed, and all three already exist in the +per-harness suites (this module reuses their idiom rather than inventing a +fourth): + +* an injected ``TurnClock`` — pi and antigravity take ``clock=`` / build one + through a patched ``TurnClock`` factory; +* a ``datetime`` SUBCLASS monkeypatched onto the module — opencode, which also + calls ``datetime.fromtimestamp`` through the same global (see + ``tests/test_opencode_agent.py``'s ``_SteppedClock`` for why a stub breaks); +* ``time.monotonic`` AND ``datetime`` both patched — claude-code, which derives + the DURATION from the monotonic clock and the BOUNDS from the wall clock, so + patching one leaves the other real and the test measures nothing. + +Codex is the fifth and takes its stamps from SDK epoch milliseconds rather than +from any host clock, so its case scripts those stamps directly. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from types import SimpleNamespace +from typing import Any + +import pytest + +from coder_eval.models import ( + AgentKind, + AssistantMessage, + CommandTelemetry, + TokenUsage, + TranscriptMessage, + TurnRecord, + parse_agent_config, +) +from coder_eval.streaming.callbacks import CompositeStreamCallback +from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.events import ( + AgentEndEvent, + AgentEndStatus, + AgentStartEvent, + ToolEndEvent, + ToolEndStatus, +) +from coder_eval.timing import union_ms + + +# The two CLI harnesses (opencode, codex) report their stamps as epoch +# milliseconds and convert them with the real ``datetime.fromtimestamp``. So +# the shared origin is DERIVED from an epoch value rather than written as a +# wall time: that is what puts a scripted ``now()`` read and a converted CLI +# stamp on ONE timeline, without patching the conversion itself. +EPOCH_MS = 1_800_000_000_000 +BASE = datetime.fromtimestamp(EPOCH_MS / 1000.0) + + +def at(ms: float) -> datetime: + return BASE + timedelta(milliseconds=ms) + + +@dataclass(frozen=True) +class Turn: + """What one harness case produces: a scripted span plus what it recorded. + + ``started_ms`` / ``ended_ms`` are the turn's own bounds — where the + ``AgentStartEvent`` and ``AgentEndEvent`` land. Everything else came out of + the reducer. + """ + + started_ms: float + ended_ms: float + messages: list[TranscriptMessage] + commands: list[CommandTelemetry] + + +def _record(turn: Turn) -> TurnRecord: + """Reduce a scripted turn through the production collector seam. + + Deliberately the real ``EventCollector`` rather than a direct + ``decompose_turn`` call: the head and the tail are only as correct as the + arguments that seam builds for them, and those (the main-thread generation + filter, the command span set) are half of what this file is asserting. + """ + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=at(turn.started_ms))) + for command in turn.commands: + collector.on_event(ToolEndEvent(task_id="t", turn_id="turn", tool=command, status=ToolEndStatus.OK)) + collector.on_event( + AgentEndEvent( + task_id="t", + status=AgentEndStatus.COMPLETED, + iteration=1, + user_input="go", + messages=turn.messages, + usage=TokenUsage(), + duration_seconds=(turn.ended_ms - turn.started_ms) / 1000.0, + timestamp=at(turn.ended_ms), + ) + ) + return collector.build_turn_record() + + +def assert_identity_closes(turn: Turn) -> None: + """head + Σ generation + UNION(tool) + tail == the scripted span, EXACTLY. + + ``pytest.approx`` rather than an order-of-magnitude bound: every input is + scripted, so the only slack is float representation. A bound wide enough to + absorb a real defect is the sensor this module exists to replace. + + Main thread only on the generation side, mirroring the collector and + ``decompose_run.py``: a sub-agent's generations bubble into the same stream + and the spawning Agent call's own interval already spans them. + """ + record = _record(turn) + span_ms = turn.ended_ms - turn.started_ms + + generation_ms = sum( + m.generation_duration_ms or 0.0 + for m in record.messages + if isinstance(m, AssistantMessage) and m.parent_tool_use_id is None + ) + tool_ms = union_ms( + [ + (c.execution_started_at, c.execution_completed_at) + for c in record.commands + if c.execution_started_at is not None and c.execution_completed_at is not None + ] + ) + assert record.harness_startup_ms is not None, "a turn that generated has a measured head" + assert record.harness_teardown_ms is not None, "a turn that generated has a measured tail" + bucket_sum = record.harness_startup_ms + generation_ms + tool_ms + record.harness_teardown_ms + + assert bucket_sum == pytest.approx(span_ms), ( + f"the four buckets sum to {bucket_sum:.4f} ms against a {span_ms:.4f} ms turn " + f"(off by {bucket_sum - span_ms:+.4f} ms): head={record.harness_startup_ms:.4f}, " + f"generation={generation_ms:.4f}, tool_union={tool_ms:.4f}, tail={record.harness_teardown_ms:.4f}. " + "They tile the turn, so a sum UNDER it means some interval is booked nowhere — the " + "defect class the golden corpus cannot see — and a sum OVER it means one is booked twice." + ) + + +# -------------------------------------------------------------------------- +# pi — an injected TurnClock +# -------------------------------------------------------------------------- + + +class _InjectedClock: + """A ``TurnClock`` stand-in the test moves by hand, in ms from ``BASE``. + + Injected rather than monkeypatched: pi and antigravity derive every wall + stamp from their per-turn clock, so patching the module's ``datetime`` + would no longer reach them and the case would quietly measure the real + clock and pass by accident. + """ + + def __init__(self, at_ms: float = 0.0) -> None: + self.at_ms = at_ms + + def now(self) -> datetime: + return at(self.at_ms) + + +def _pi_turn(*, untile: bool = False) -> Turn: + """Two tiled windows around a tool, with a real head and a real tail. + + The tool closes INSIDE the first window rather than across the boundary — + that case is pinned by ``tests/test_pi_agent.py``. What this adds is the + two ends: pi's first window opens at its first ``turn_start``, so the CLI + boot before it is head, and the turn runs on past the last ``turn_end``. + + ``untile`` reproduces the defect pi actually shipped with — each window + measured from its OWN ``turn_start`` instead of from the previous flush — + so that the sensor can be shown to catch it. See + ``test_the_sensor_sees_a_window_that_stops_tiling``. + """ + from coder_eval.agents.pi_agent import _PiTurnState + + payload = {"message": {"role": "assistant", "usage": {"input": 10, "output": 5}, "stopReason": "stop"}} + clock = _InjectedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) + commands: list[CommandTelemetry] = [] + state.bind(lambda e: commands.append(e.tool) if isinstance(e, ToolEndEvent) else None) + + clock.at_ms = 500 # CLI boot: head + state.on_turn_start() + clock.at_ms = 700 + state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) + clock.at_ms = 1200 + state.on_tool_execution_end({"toolCallId": "c1", "result": "ok"}) + clock.at_ms = 2000 + state.on_turn_end(payload) + clock.at_ms = 2600 # the inter-turn gap, which window 2 tiles back over + state.on_turn_start() + if untile: + state.gen_mark = None + clock.at_ms = 3000 + state.on_turn_end(payload) + + return Turn( + started_ms=0.0, + ended_ms=3500.0, # process teardown after the last turn: tail + messages=list(state.messages), + commands=commands, + ) + + +# -------------------------------------------------------------------------- +# opencode — a datetime subclass on the module +# -------------------------------------------------------------------------- + + +class _SteppedDatetime(datetime): + """A clock the test moves by hand, in ms from ``BASE``. + + Subclasses ``datetime`` rather than stubbing it, because the reducer also + calls ``datetime.fromtimestamp`` through the same module global to convert + the CLI's epoch stamps, and that must keep resolving to the real + implementation — the CLI's stamps and the reducer's own ``now()`` reads + have to land on ONE timeline for the arithmetic to mean anything. + """ + + at_ms = 0.0 + + @staticmethod + def now(tz: Any = None) -> datetime: # type: ignore[override] + return at(_SteppedDatetime.at_ms) + + +def _opencode_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: + """The same two-window shape, driven through OpenCode's step stream.""" + from coder_eval.agents import opencode_agent as opencode_module + from coder_eval.agents.opencode_agent import _OpenCodeTurnState + + monkeypatch.setattr(opencode_module, "datetime", _SteppedDatetime) + state = _OpenCodeTurnState(task_id="t", iteration=1, user_input="go", model="m") + commands: list[CommandTelemetry] = [] + state.bind(lambda e: commands.append(e.tool) if isinstance(e, ToolEndEvent) else None) + finish = {"reason": "stop", "tokens": {"input": 10, "output": 5}} + + _SteppedDatetime.at_ms = 500 # Node boot: head + state.on_step_start({"messageID": "m1"}) + _SteppedDatetime.at_ms = 1200 + state.on_tool_use( + { + "callID": "c1", + "tool": "bash", + "state": {"status": "completed", "time": {"start": EPOCH_MS + 700, "end": EPOCH_MS + 1200}}, + } + ) + _SteppedDatetime.at_ms = 2000 + state.on_step_finish(finish) + _SteppedDatetime.at_ms = 2600 + state.on_step_start({"messageID": "m2"}) + _SteppedDatetime.at_ms = 3000 + state.on_step_finish(finish) + + return Turn(started_ms=0.0, ended_ms=3500.0, messages=list(state.messages), commands=commands) + + +# -------------------------------------------------------------------------- +# antigravity — an injected TurnClock, at the reducer +# -------------------------------------------------------------------------- + + +def _antigravity_turn() -> Turn: + """One interleaved window per generation, with a tool inside the first. + + Driven at ``_AntigravityTurnState`` rather than through ``communicate()``: + the fake conversation yields with no delay, so an end-to-end run cannot + distinguish a window that opened at the turn's start from one that opened + later. The state's own ``_gen_mark_wall`` is stamped at construction, so + constructing it AFTER the scripted agent start is what gives this harness a + measurable head at all. + """ + from coder_eval.agents.antigravity_agent import AntigravityAgent, _AntigravityTurnState + from tests._fixtures.golden_streams.antigravity_fixtures import _step, _tc, _usage + + agent = AntigravityAgent(parse_agent_config(type=AgentKind.ANTIGRAVITY, model="gemini-3.5-flash")) + collector = EventCollector() + clock = _InjectedClock(at_ms=500) # dispatch before the first Step: head + state = _AntigravityTurnState( + agent=agent, + emit=CompositeStreamCallback([collector]), + task_id="t", + turn_id="turn", + collector=collector, + user_input="go", + iteration=1, + model="gemini-3.5-flash", + turn_start_time=0.0, + clock=clock, + ) + + clock.at_ms = 700 + state.process_step( + _step( + "TOOL_CALL", + "ACTIVE", + target="TARGET_ENVIRONMENT", + tool_calls=[_tc("run_command", "c1", {"command_line": "ls"})], + ) + ) + clock.at_ms = 1200 + state.process_step( + _step( + "TOOL_CALL", + "DONE", + target="TARGET_ENVIRONMENT", + tool_calls=[_tc("run_command", "c1", {"command_line": "ls", "exit_code": 0})], + ) + ) + clock.at_ms = 2000 + state.process_step(_step("THINKING", "DONE", thinking="plan", usage=_usage(100, 0, 5, 5))) + clock.at_ms = 3000 + state.process_step(_step("TEXT_RESPONSE", "DONE", content="done", complete=True, usage=_usage(200, 0, 10, 0))) + + return Turn(started_ms=0.0, ended_ms=3500.0, messages=list(state.messages), commands=list(state.commands)) + + +# -------------------------------------------------------------------------- +# codex — scripted SDK epoch-millisecond stamps +# -------------------------------------------------------------------------- + + +def _codex_turn() -> Turn: + """Two tiled windows, the first SPLIT across two sub-messages. + + Codex is the only harness that cuts one window into several messages + (thinking and action, apportioned by output-token share), and they share + one pair of bounds. The identity has to close over the GROUP, so this case + drives that split deliberately rather than the simpler one-message shape. + + Its stamps are the SDK's own epoch milliseconds, unreachable from any host + clock, so ``_flush_message`` is driven with them set by hand — the idiom + ``tests/test_codex_agent.py::TestFlushMessageWindowBounds`` already uses. + """ + from coder_eval.agents.codex_agent import CodexAgent, _CodexTurnState, _ms_to_dt + from coder_eval.models import ContentBlock + + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5.5")) + collector = EventCollector() + state = _CodexTurnState( + agent, + emit=CompositeStreamCallback([collector]), + task_id="t", + turn_id="turn", + collector=collector, + commands=[], + messages=[], + user_input="go", + iteration=1, + turn_start_time=0.0, + ) + command = CommandTelemetry( + tool_name="bash", + tool_id="c1", + timestamp=_ms_to_dt(EPOCH_MS + 700), + execution_started_at=_ms_to_dt(EPOCH_MS + 700), + execution_completed_at=_ms_to_dt(EPOCH_MS + 1200), + duration_ms=500.0, + result_status="success", + ) + state.commands.append(command) + + # Window 1 — no mark yet, so it opens at its own first item (+500): the CLI + # boot before that is head. Thinking + text, so the flush cuts two + # sub-messages sharing the window. + state.open_blocks = [ + ContentBlock(block_type="thinking", sequence=0, thinking="plan"), + ContentBlock(block_type="text", sequence=0, text="answer"), + ] + state.open_start_ms = EPOCH_MS + 500 + state.open_end_ms = EPOCH_MS + 2000 + state._flush_message( + SimpleNamespace(input_tokens=500, cached_input_tokens=0, output_tokens=100, reasoning_output_tokens=80) + ) + + # Window 2 — tiles back from the mark (+2000), covering the gap before its + # own first item at +2600. + state.open_blocks = [ContentBlock(block_type="text", sequence=0, text="more")] + state.open_start_ms = EPOCH_MS + 2600 + state.open_end_ms = EPOCH_MS + 3000 + state._flush_message(SimpleNamespace(input_tokens=10, cached_input_tokens=0, output_tokens=5)) + + return Turn(started_ms=0.0, ended_ms=3500.0, messages=list(state.messages), commands=[command]) + + +# -------------------------------------------------------------------------- +# claude-code — BOTH the monotonic and the wall clock patched +# -------------------------------------------------------------------------- + + +def _claude_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: + """A tool call between two emissions, with a real head and a real tail. + + This reducer derives the window's DURATION from ``time.monotonic()`` and + its BOUNDS from ``datetime.now()``, so both module globals are patched off + one counter. Patching either alone leaves the other reading the real clock, + and the case would then assert a measured span against an unmeasured one. + + Note where its windows do NOT tile: the tool result resets both marks, so + the interval between the emission that ISSUED the call and the result is + left outside every window. That gap is the tool's own execution, which is + exactly what the tool bucket claims — which is why the identity still + closes to the millisecond. + """ + from coder_eval.agents import claude_code_agent as claude_module + from coder_eval.agents.claude_code_agent import ClaudeCodeAgent, _ClaudeTurnState + from coder_eval.streaming.events import AgentEndStatus as _AgentEndStatus + from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage as SdkAssistantMessage + from tests._fixtures.golden_streams.claude_fixtures import ToolUseBlock, UserMessage + + class _Stepped(datetime): + at_ms = 0.0 + + @staticmethod + def now(tz: Any = None) -> datetime: # type: ignore[override] + return at(_Stepped.at_ms) + + def _monotonic() -> float: + return _Stepped.at_ms / 1000.0 + + monkeypatch.setattr(claude_module, "datetime", _Stepped) + monkeypatch.setattr(claude_module, "time", SimpleNamespace(monotonic=_monotonic)) + + agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits")) + collector = EventCollector() + commands: list[CommandTelemetry] = [] + + _Stepped.at_ms = 500 # CLI spawn + dispatch, before the state exists: head + state = _ClaudeTurnState( + agent, + emit=CompositeStreamCallback( + [ + collector, + SimpleNamespace(on_event=lambda e: commands.append(e.tool) if isinstance(e, ToolEndEvent) else None), + ] + ), + collector=collector, + task_id="t", + user_input="go", + iteration=1, + max_turns=None, + log=agent._log, + turn_start_time=_monotonic(), + deadline=None, + ) + + _Stepped.at_ms = 1000 + state.on_assistant_message( + SdkAssistantMessage( + [ToolUseBlock("c1", "Bash", {"command": "ls"})], + usage={"input_tokens": 10, "output_tokens": 5}, + message_id="m1", + ) + ) + _Stepped.at_ms = 1800 # the tool ran for the whole gap + state.on_user_message(UserMessage("c1", False, "ok")) + _Stepped.at_ms = 2500 + state.on_assistant_message(SdkAssistantMessage([], usage={"input_tokens": 10, "output_tokens": 5}, message_id="m2")) + state.finalize(_AgentEndStatus.COMPLETED) + + return Turn(started_ms=0.0, ended_ms=3000.0, messages=list(state.sdk_messages), commands=commands) + + +# -------------------------------------------------------------------------- +# The contract +# -------------------------------------------------------------------------- + + +def test_pi_buckets_tile_the_turn(): + assert_identity_closes(_pi_turn()) + + +def test_opencode_buckets_tile_the_turn(monkeypatch: pytest.MonkeyPatch): + assert_identity_closes(_opencode_turn(monkeypatch)) + + +def test_antigravity_buckets_tile_the_turn(): + assert_identity_closes(_antigravity_turn()) + + +def test_codex_buckets_tile_the_turn(): + assert_identity_closes(_codex_turn()) + + +def test_claude_code_buckets_tile_the_turn(monkeypatch: pytest.MonkeyPatch): + assert_identity_closes(_claude_turn(monkeypatch)) + + +def test_every_built_in_harness_has_a_case(): + """A sensor that silently covers four of five is worse than one naming the gap. + + Derived from ``AgentKind`` rather than from a hand-written list, so a sixth + built-in harness fails here instead of shipping unmeasured. A harness whose + reducer genuinely cannot be driven without a live process belongs in an + exemption set carrying its reason — not in a weaker end-to-end assertion. + + From the ENUM and not from ``AgentRegistry``, which is open: a third-party + plugin agent registers there too, and an out-of-tree harness is not this + repo's to cover (``coder_eval_uipath``'s delegate-sdk is the live example). + """ + covered = {name for name in globals() if name.startswith("test_") and name.endswith("_buckets_tile_the_turn")} + # NONE is the agentless task double — no reducer, no generation window at + # all; UNKNOWN is a load-failure placeholder that never runs. + built_in = set(AgentKind) - {AgentKind.NONE, AgentKind.UNKNOWN} + missing = {kind for kind in built_in if f"test_{kind.value.replace('-', '_')}_buckets_tile_the_turn" not in covered} + assert not missing, f"no ms-exact identity case for {sorted(m.value for m in missing)}" + + +def test_the_sensor_sees_a_window_that_stops_tiling(): + """The gating mutation check, as a committed test rather than an attestation. + + A window seeded from its own turn start instead of from the previous + flush's close is the defect pi shipped with, and the whole point of this + module is that the SUITE notices it rather than a reviewer reproducing it + by hand. The golden corpus cannot: it masks every value involved. + + Asserted on the MAGNITUDE as well as on the failure, because "it raised" + would also pass if the mutation broke the case in some unrelated way. The + 600 ms is the scripted gap between one ``turn_end`` and the next + ``turn_start`` — real model time, which untiling books to nothing. + """ + healthy = _pi_turn() + mutated = _pi_turn(untile=True) + + def _generation_ms(turn: Turn) -> float: + return sum(m.generation_duration_ms or 0.0 for m in turn.messages if isinstance(m, AssistantMessage)) + + assert _generation_ms(healthy) - _generation_ms(mutated) == pytest.approx(600.0) + with pytest.raises(AssertionError, match="booked nowhere"): + assert_identity_closes(mutated) diff --git a/tests/test_timing_union_parity.py b/tests/test_timing_union_parity.py index f1a70ec2d..7579a39b1 100644 --- a/tests/test_timing_union_parity.py +++ b/tests/test_timing_union_parity.py @@ -20,14 +20,16 @@ import pytest -from coder_eval.timing import busy_ms +from coder_eval.timing import busy_ms, union_ms _FIXTURE = Path(__file__).parent / "_fixtures" / "timing_union_cases.json" _TS_TEST = Path(__file__).parents[1] / "evalboard" / "lib" / "__tests__" / "timing-union-parity.test.ts" _BASE = datetime(2026, 1, 1, 12, 0, 0) -_CASES = json.loads(_FIXTURE.read_text())["cases"] +_CORPUS = json.loads(_FIXTURE.read_text()) +_CASES = _CORPUS["cases"] +_UNION_CASES = _CORPUS["union_cases"] def _at(offset_ms: float) -> datetime: @@ -41,6 +43,20 @@ def test_busy_ms_matches_the_shared_corpus(case: dict) -> None: assert busy_ms(spans, _at(lo), _at(hi)) == pytest.approx(case["expected_ms"]) +@pytest.mark.parametrize("case", _UNION_CASES, ids=[c["name"] for c in _UNION_CASES]) +def test_union_ms_matches_the_shared_corpus(case: dict) -> None: + """The same union, with the extent derived rather than handed in. + + ``union_ms`` is what the golden sensor and the live residual gate both + call; the TypeScript side of these cases is ``toolExecutionMs``, which + derives the extent with its own ``min``/``max`` rather than being given + one. That derivation is the only part of the union rule the ``cases`` + array above cannot reach. + """ + spans = [(_at(s), _at(e)) for s, e in case["spans"]] + assert union_ms(spans) == pytest.approx(case["expected_ms"]) + + def test_the_typescript_half_replays_the_same_file() -> None: """A parity corpus only one side reads is not a parity corpus. @@ -53,3 +69,7 @@ def test_the_typescript_half_replays_the_same_file() -> None: # The TS side must exercise every case, not a hand-picked subset — it reads # the array rather than restating it. assert re.search(r"\.cases\b", source), "the TS test must iterate the corpus, not inline cases" + assert re.search(r"\.union_cases\b", source), ( + "the TS test must also iterate `union_cases`, the half that pins toolExecutionMs's " + "own min/max extent against union_ms's" + ) From 8cc587c3373a52ef976cc466c58a971a62ff5311 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 22:10:17 -0700 Subject: [PATCH 29/54] =?UTF-8?q?test(harness):=202/7=20=E2=80=94=20OpenCo?= =?UTF-8?q?de=20and=20Pi=20get=20a=20corpus=20worth=20replaying?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit They were the two newest reducers, the two that shipped the generation-mark defect, and the two with the thinnest golden corpus: 2 scenarios each against 9 for claude and 8 for codex. OpenCode now has 5 and Pi 6. Each gains the three shapes the older harnesses already cover — two tiled generations with a tool between them, an orphan force-closed at finalization, and a crash whose partial record must survive — plus, on Pi, the duplicate `turn_end` its reducer explicitly promises to survive and that had a unit test and no snapshot. The crash scenarios need an `expects` knob, so both scenario dataclasses now carry the one `ClaudeScenario` already had, for the same reason: a crash partial is a real capture path and nobody was comparing it against a snapshot on these two harnesses. Only `opencode_c_multi_step_tiling` is exempted from the identity check, and the reason is structural rather than convenient: OpenCode takes its tool bounds from the CLI payload, so every tool-resolving scenario of that harness injects millisecond stamps into a sub-millisecond replay. Pi derives its from its own TurnClock, so all four of its new scenarios stay inside the sensor. Also corrects the pi fixtures' text event. `_handle_line` dispatches on the outer `type`, and `text` is in neither the dispatch chain nor the recognized vocabulary, so the bare `{"type": "text"}` line `a_single_text_turn` used reached no handler: it captured nothing, and the snapshot's `agent_output` was empty under a scenario named for text. The new `_text()` helper emits the real `message_update` / `text_delta` shape, which is why that snapshot changes. Two Pi defects the new snapshots make visible are CAPTURED AND ANNOTATED, not fixed — this phase changes no `src/` file: * `f_duplicate_turn_end` shows `turn_text_parts` / `turn_tool_ids` cleared only in `on_turn_start`, so the second `turn_end` republishes the first turn's text as its own assistant message. `on_turn_end`'s own comment makes exactly this argument for the sibling `turn_started_at` reset it does perform. * `d_orphaned_tool` shows a `duration_ms` and a subtracted span published for a call that never returned — `_close_tool` guards on `execution_started_at is not None` while its comment claims it guards on "resolved", and the `execution_completed_at` is only the instant the sweep ran. claude-code leaves that field None here on purpose. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- .../opencode_c_multi_step_tiling.json | 108 +++++++++++++ .../expected/opencode_d_orphaned_tool.json | 90 +++++++++++ .../opencode_e_error_after_generation.json | 59 +++++++ .../expected/pi_a_single_text_turn.json | 14 +- .../expected/pi_c_multi_turn_tiling.json | 108 +++++++++++++ .../expected/pi_d_orphaned_tool.json | 90 +++++++++++ .../expected/pi_e_error_after_generation.json | 76 +++++++++ .../expected/pi_f_duplicate_turn_end.json | 86 ++++++++++ .../golden_streams/opencode_fixtures.py | 119 +++++++++++++- tests/_fixtures/golden_streams/pi_fixtures.py | 149 +++++++++++++++++- tests/test_agent_golden_master.py | 9 ++ 11 files changed, 897 insertions(+), 11 deletions(-) create mode 100644 tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json create mode 100644 tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json create mode 100644 tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json create mode 100644 tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json create mode 100644 tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json create mode 100644 tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json create mode 100644 tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json diff --git a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json new file mode 100644 index 000000000..ee573afc1 --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json @@ -0,0 +1,108 @@ +{ + "agent_output": "Listed it.", + "assistant_turn_count": 2, + "commands": [ + { + "assistant_turn_index": 1, + "duration_ms": "", + "error_message": null, + "execution_completed_at": "", + "execution_started_at": "", + "generation_completed_at": null, + "parameters": { + "command": "ls" + }, + "result_data": null, + "result_status": "success", + "result_summary": "main.py", + "result_tokens": 2, + "sequence_number": 1, + "timestamp": "", + "tool_id": "call_1", + "tool_name": "Bash" + } + ], + "crash_reason": null, + "crashed": false, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "max_turns_exhausted": false, + "messages": [ + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "tool_use", + "is_error": false, + "sequence": 0, + "signature": null, + "text": null, + "thinking": null, + "tool_use_id": "call_1" + } + ], + "generation_duration_ms": "", + "input_tokens": 100, + "message_id": "msg_1", + "model": "deepseek/deepseek-v4-pro", + "output_tokens": 20, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "tool-calls", + "tool_use_ids": [ + "call_1" + ] + }, + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "Listed it.", + "thinking": null, + "tool_use_id": null + } + ], + "generation_duration_ms": "", + "input_tokens": 50, + "message_id": "msg_2", + "model": "deepseek/deepseek-v4-pro", + "output_tokens": 30, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [] + } + ], + "model_used": "deepseek/deepseek-v4-pro", + "num_turns": 2, + "result_summary": { + "is_error": false, + "result": null, + "stop_reason": "stop", + "subtype": "completed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 150, + "output_tokens": 50, + "total_cost_usd": "", + "uncached_input_tokens": 150 + }, + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json new file mode 100644 index 000000000..3ebb56912 --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json @@ -0,0 +1,90 @@ +{ + "agent_output": "Waiting.", + "assistant_turn_count": 1, + "commands": [ + { + "assistant_turn_index": 1, + "duration_ms": null, + "error_message": "no result observed", + "execution_completed_at": "", + "execution_started_at": null, + "generation_completed_at": null, + "parameters": { + "command": "sleep 600" + }, + "result_data": null, + "result_status": "unknown", + "result_summary": null, + "result_tokens": 0, + "sequence_number": 1, + "timestamp": "", + "tool_id": "call_1", + "tool_name": "Bash" + } + ], + "crash_reason": null, + "crashed": false, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "max_turns_exhausted": false, + "messages": [ + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "Waiting.", + "thinking": null, + "tool_use_id": null + }, + { + "block_type": "tool_use", + "is_error": false, + "sequence": 1, + "signature": null, + "text": null, + "thinking": null, + "tool_use_id": "call_1" + } + ], + "generation_duration_ms": "", + "input_tokens": 100, + "message_id": "msg_1", + "model": "deepseek/deepseek-v4-pro", + "output_tokens": 20, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [ + "call_1" + ] + } + ], + "model_used": "deepseek/deepseek-v4-pro", + "num_turns": 1, + "result_summary": { + "is_error": false, + "result": null, + "stop_reason": "stop", + "subtype": "completed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 100, + "output_tokens": 20, + "total_cost_usd": "", + "uncached_input_tokens": 100 + }, + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json new file mode 100644 index 000000000..a757b0c84 --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json @@ -0,0 +1,59 @@ +{ + "agent_output": "Starting.", + "assistant_turn_count": 1, + "commands": [], + "crash_reason": "OpenCode error: 401 from the provider", + "crashed": true, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "max_turns_exhausted": false, + "messages": [ + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "Starting.", + "thinking": null, + "tool_use_id": null + } + ], + "generation_duration_ms": "", + "input_tokens": 100, + "message_id": "msg_1", + "model": "deepseek/deepseek-v4-pro", + "output_tokens": 20, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [] + } + ], + "model_used": "deepseek/deepseek-v4-pro", + "num_turns": 1, + "result_summary": { + "is_error": true, + "result": "OpenCode error: 401 from the provider", + "stop_reason": "stop", + "subtype": "crashed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 100, + "output_tokens": 20, + "total_cost_usd": "", + "uncached_input_tokens": 100 + }, + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json index 025af6ce6..f83268c0c 100644 --- a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json @@ -1,5 +1,5 @@ { - "agent_output": "", + "agent_output": "All done.", "assistant_turn_count": 1, "commands": [], "crash_reason": null, @@ -14,7 +14,17 @@ "cache_creation_tokens": 0, "cache_read_tokens": 64, "completed_at": "", - "content_blocks": [], + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "All done.", + "thinking": null, + "tool_use_id": null + } + ], "generation_duration_ms": "", "input_tokens": 100, "message_id": null, diff --git a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json new file mode 100644 index 000000000..2a06910a3 --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json @@ -0,0 +1,108 @@ +{ + "agent_output": "Listed it.", + "assistant_turn_count": 2, + "commands": [ + { + "assistant_turn_index": 1, + "duration_ms": "", + "error_message": null, + "execution_completed_at": "", + "execution_started_at": "", + "generation_completed_at": null, + "parameters": { + "command": "ls" + }, + "result_data": null, + "result_status": "success", + "result_summary": "main.py", + "result_tokens": 2, + "sequence_number": 1, + "timestamp": "", + "tool_id": "call_1", + "tool_name": "Bash" + } + ], + "crash_reason": null, + "crashed": false, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "max_turns_exhausted": false, + "messages": [ + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "tool_use", + "is_error": false, + "sequence": 0, + "signature": null, + "text": null, + "thinking": null, + "tool_use_id": "call_1" + } + ], + "generation_duration_ms": "", + "input_tokens": 100, + "message_id": null, + "model": "openrouter/moonshotai/kimi-k3", + "output_tokens": 20, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [ + "call_1" + ] + }, + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "Listed it.", + "thinking": null, + "tool_use_id": null + } + ], + "generation_duration_ms": "", + "input_tokens": 50, + "message_id": null, + "model": "openrouter/moonshotai/kimi-k3", + "output_tokens": 30, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [] + } + ], + "model_used": "openrouter/moonshotai/kimi-k3", + "num_turns": 2, + "result_summary": { + "is_error": false, + "result": null, + "stop_reason": "stop", + "subtype": "completed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 150, + "output_tokens": 50, + "total_cost_usd": "", + "uncached_input_tokens": 150 + }, + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json new file mode 100644 index 000000000..e2efaaad4 --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json @@ -0,0 +1,90 @@ +{ + "agent_output": "Waiting.", + "assistant_turn_count": 1, + "commands": [ + { + "assistant_turn_index": 1, + "duration_ms": "", + "error_message": "no result observed", + "execution_completed_at": "", + "execution_started_at": "", + "generation_completed_at": null, + "parameters": { + "command": "sleep 600" + }, + "result_data": null, + "result_status": "unknown", + "result_summary": null, + "result_tokens": 0, + "sequence_number": 1, + "timestamp": "", + "tool_id": "call_1", + "tool_name": "Bash" + } + ], + "crash_reason": null, + "crashed": false, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "max_turns_exhausted": false, + "messages": [ + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "Waiting.", + "thinking": null, + "tool_use_id": null + }, + { + "block_type": "tool_use", + "is_error": false, + "sequence": 1, + "signature": null, + "text": null, + "thinking": null, + "tool_use_id": "call_1" + } + ], + "generation_duration_ms": "", + "input_tokens": 100, + "message_id": null, + "model": "openrouter/moonshotai/kimi-k3", + "output_tokens": 20, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [ + "call_1" + ] + } + ], + "model_used": "openrouter/moonshotai/kimi-k3", + "num_turns": 1, + "result_summary": { + "is_error": false, + "result": null, + "stop_reason": "stop", + "subtype": "completed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 100, + "output_tokens": 20, + "total_cost_usd": "", + "uncached_input_tokens": 100 + }, + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json new file mode 100644 index 000000000..33a4d72e4 --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json @@ -0,0 +1,76 @@ +{ + "agent_output": "Starting.", + "assistant_turn_count": 2, + "commands": [], + "crash_reason": "Pi error: provider returned 529 after 5 retries", + "crashed": true, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "max_turns_exhausted": false, + "messages": [ + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "Starting.", + "thinking": null, + "tool_use_id": null + } + ], + "generation_duration_ms": "", + "input_tokens": 100, + "message_id": null, + "model": "openrouter/moonshotai/kimi-k3", + "output_tokens": 20, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [] + }, + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [], + "generation_duration_ms": "", + "input_tokens": 0, + "message_id": null, + "model": "openrouter/moonshotai/kimi-k3", + "output_tokens": 0, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "error", + "tool_use_ids": [] + } + ], + "model_used": "openrouter/moonshotai/kimi-k3", + "num_turns": 2, + "result_summary": { + "is_error": true, + "result": "Pi error: provider returned 529 after 5 retries", + "stop_reason": "error", + "subtype": "crashed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 100, + "output_tokens": 20, + "total_cost_usd": "", + "uncached_input_tokens": 100 + }, + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json new file mode 100644 index 000000000..66c7e39dc --- /dev/null +++ b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json @@ -0,0 +1,86 @@ +{ + "agent_output": "First.", + "assistant_turn_count": 1, + "commands": [], + "crash_reason": null, + "crashed": false, + "duration_seconds": "", + "harness_startup_ms": "", + "harness_teardown_ms": "", + "iteration": 1, + "max_turns_exhausted": false, + "messages": [ + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "First.", + "thinking": null, + "tool_use_id": null + } + ], + "generation_duration_ms": "", + "input_tokens": 100, + "message_id": null, + "model": "openrouter/moonshotai/kimi-k3", + "output_tokens": 20, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [] + }, + { + "cache_creation_tokens": 0, + "cache_read_tokens": 0, + "completed_at": "", + "content_blocks": [ + { + "block_type": "text", + "is_error": false, + "sequence": 0, + "signature": null, + "text": "First.", + "thinking": null, + "tool_use_id": null + } + ], + "generation_duration_ms": "", + "input_tokens": 10, + "message_id": null, + "model": "openrouter/moonshotai/kimi-k3", + "output_tokens": 5, + "parent_tool_use_id": null, + "reasoning_tokens": 0, + "role": "assistant", + "started_at": "", + "stop_reason": "stop", + "tool_use_ids": [] + } + ], + "model_used": "openrouter/moonshotai/kimi-k3", + "num_turns": 1, + "result_summary": { + "is_error": false, + "result": null, + "stop_reason": "stop", + "subtype": "completed" + }, + "timestamp": "", + "token_usage": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + "input_tokens": 110, + "output_tokens": 25, + "total_cost_usd": "", + "uncached_input_tokens": 110 + }, + "user_input": "do it" +} diff --git a/tests/_fixtures/golden_streams/opencode_fixtures.py b/tests/_fixtures/golden_streams/opencode_fixtures.py index 5a86908dd..5f1381479 100644 --- a/tests/_fixtures/golden_streams/opencode_fixtures.py +++ b/tests/_fixtures/golden_streams/opencode_fixtures.py @@ -30,6 +30,7 @@ from unittest.mock import patch from coder_eval.agents.opencode_agent import OpenCodeAgent +from coder_eval.errors import AgentCrashError from coder_eval.models import OpenCodeAgentConfig @@ -201,17 +202,22 @@ def _agent() -> OpenCodeAgent: class OpenCodeScenario: """One recorded CLI event stream. - No ``expects`` knob: every scenario here replays cleanly. The crash and - timeout paths live in the agent's own test module, which asserts on the - exception rather than on a snapshot. + ``expects`` names the exception a scenario is supposed to raise, and the + runner then snapshots ``pending_turn`` instead of the returned record — + the same knob ``ClaudeScenario`` carries, for the same reason: the partial + a crash preserves is a real capture path, and one nobody was comparing + against a snapshot on this harness. """ name: str lines: list[str] + expects: type[BaseException] | None = None async def run_opencode_scenario(scenario: OpenCodeScenario, working_dir: str) -> dict[str, Any]: """Replay one scenario and return the resulting record as a plain dump.""" + import pytest + proc = _FakeProcess(_rebase_lines(scenario.lines)) async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: @@ -225,7 +231,13 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: patch.object(os, "killpg", lambda _pgid, _sig: None, create=True), ): await agent.start(working_dir) - record = await agent.communicate("do it") + if scenario.expects is not None: + with pytest.raises(scenario.expects): + await agent.communicate("do it") + record = agent.pending_turn + assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" + else: + record = await agent.communicate("do it") return record.model_dump(mode="json") @@ -258,6 +270,105 @@ def _build_catalogue() -> list[OpenCodeScenario]: OpenCodeScenario(name="b_tool_call_resolved", lines=list(HAPPY_STREAM)), ) + # (c) two generations with a tool resolving between them. The TILING case: + # the second window opens at the first `step_finish`, not at its own + # `step_start`, so the wall clock between the two steps — the model time + # that produced the second one — lands inside a window rather than in no + # bucket at all. That is the defect this harness shipped with, and it had + # a unit test but no golden. + scenarios.append( + OpenCodeScenario( + name="c_multi_step_tiling", + lines=[ + _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}), + _evt( + "tool_use", + { + "id": "prt_2", + "messageID": "msg_1", + "tool": "bash", + "callID": "call_1", + "state": { + "status": "completed", + "input": {"command": "ls"}, + "output": "main.py", + "time": {"start": _T0_MS, "end": _T0_MS + 5}, + }, + }, + ), + _evt( + "step_finish", + {"id": "prt_3", "messageID": "msg_1", "reason": "tool-calls", "tokens": _tokens(100, 20)}, + ), + _evt("step_start", {"id": "prt_4", "messageID": "msg_2"}), + _evt("text", {"id": "prt_5", "messageID": "msg_2", "text": "Listed it."}), + _evt( + "step_finish", + {"id": "prt_6", "messageID": "msg_2", "reason": "stop", "tokens": _tokens(50, 30)}, + ), + ], + ) + ) + + # (d) a tool the CLI opens and never resolves — force-closed as `unresolved` + # by the orphan sweep at finalization. It carries NO `state.time`, which is + # the honest shape for a call that never returned: with no + # `execution_started_at` there is no `duration_ms` and no span. + # + # READ THE SNAPSHOT: the sweep still stamps `execution_completed_at`, which + # it does on every close path, so the record holds an end with no + # beginning. Compare `pi_d_orphaned_tool`, where the start IS stamped and a + # manufactured duration follows from it. + scenarios.append( + OpenCodeScenario( + name="d_orphaned_tool", + lines=[ + _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}), + _evt( + "tool_use", + { + "id": "prt_2", + "messageID": "msg_1", + "tool": "bash", + "callID": "call_1", + "state": {"status": "pending", "input": {"command": "sleep 600"}}, + }, + ), + _evt("text", {"id": "prt_3", "messageID": "msg_1", "text": "Waiting."}), + _evt( + "step_finish", + {"id": "prt_4", "messageID": "msg_1", "reason": "stop", "tokens": _tokens(100, 20)}, + ), + ], + ) + ) + + # (e) the CLI's own structured error AFTER a complete generation. `_settle_turn` + # crashes on it, and the partial `pending_turn` must still carry that + # generation and its head/tail — a crash does not un-measure what was + # measured before it. + scenarios.append( + OpenCodeScenario( + name="e_error_after_generation", + lines=[ + _evt("step_start", {"id": "prt_1", "messageID": "msg_1"}), + _evt("text", {"id": "prt_2", "messageID": "msg_1", "text": "Starting."}), + _evt( + "step_finish", + {"id": "prt_3", "messageID": "msg_1", "reason": "stop", "tokens": _tokens(100, 20)}, + ), + json.dumps( + { + "type": "error", + "sessionID": SESSION, + "error": {"name": "ProviderAuthError", "data": {"message": "401 from the provider"}}, + } + ), + ], + expects=AgentCrashError, + ) + ) + return scenarios diff --git a/tests/_fixtures/golden_streams/pi_fixtures.py b/tests/_fixtures/golden_streams/pi_fixtures.py index 26297241f..3dfb54998 100644 --- a/tests/_fixtures/golden_streams/pi_fixtures.py +++ b/tests/_fixtures/golden_streams/pi_fixtures.py @@ -24,6 +24,7 @@ from unittest.mock import patch from coder_eval.agents.pi_agent import PiAgent +from coder_eval.errors import AgentCrashError from coder_eval.models import PiAgentConfig @@ -63,10 +64,45 @@ def _turn_end(*, inp: int, out: int, cache_read: int = 0, cache_write: int = 0, ) +def _text(delta: str) -> str: + """One streamed assistant text delta. + + The CLI's real shape, which is `message_update` carrying an + `assistantMessageEvent` of type `text_delta` — NOT a bare `{"type": "text"}` + line. `_handle_line` dispatches on the outer `type`, so a bare `text` line + is unrecognized vocabulary: it reaches no handler, appends no text, and + leaves `agent_output` empty while the scenario still passes. + `a_single_text_turn` was written that way and asserted nothing about the + text capture its own name claims. + """ + return json.dumps({"type": "message_update", "assistantMessageEvent": {"type": "text_delta", "delta": delta}}) + + def _tool_start(call_id: str, name: str, args: dict[str, Any]) -> str: return json.dumps({"type": "tool_execution_start", "toolCallId": call_id, "toolName": name, "args": args}) +def _turn_end_error(message: str) -> str: + """A `turn_end` whose `stopReason` is the provider error pi could not retry away. + + `pi -p` exits 0 after exhausting its internal retries, so this is the only + signal that the turn died — `_settle_turn` crashes on it precisely so the + row does not book as a clean failure and silently depress the pass rate. + """ + return json.dumps( + { + "type": "turn_end", + "message": { + "role": "assistant", + "usage": {"input": 0, "output": 0, "cost": {"total": 0.0}}, + "stopReason": "error", + "errorMessage": message, + }, + "toolResults": [], + } + ) + + def _tool_end(call_id: str, name: str, text: str, *, is_error: bool = False) -> str: return json.dumps( { @@ -155,17 +191,22 @@ def _agent() -> PiAgent: class PiScenario: """One recorded CLI event stream. - No ``expects`` knob: every scenario here replays cleanly. The crash and - timeout paths live in the agent's own test module, which asserts on the - exception rather than on a snapshot. + ``expects`` names the exception a scenario is supposed to raise, and the + runner then snapshots ``pending_turn`` instead of the returned record — + the same knob ``ClaudeScenario`` carries, for the same reason: the partial + a crash preserves is a real capture path, and one nobody was comparing + against a snapshot on this harness. """ name: str lines: list[str] + expects: type[BaseException] | None = None async def run_pi_scenario(scenario: PiScenario, working_dir: str) -> dict[str, Any]: """Replay one scenario and return the resulting record as a plain dump.""" + import pytest + proc = _FakeProcess(scenario.lines) async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: @@ -179,7 +220,13 @@ async def fake_exec(*_argv: str, **_kwargs: Any) -> _FakeProcess: patch.object(os, "killpg", lambda _pgid, _sig: None, create=True), ): await agent.start(working_dir) - record = await agent.communicate("do it") + if scenario.expects is not None: + with pytest.raises(scenario.expects): + await agent.communicate("do it") + record = agent.pending_turn + assert record is not None, f"{scenario.name}: pending_turn was not set on the failure path" + else: + record = await agent.communicate("do it") return record.model_dump(mode="json") @@ -192,7 +239,7 @@ def _build_catalogue() -> list[PiScenario]: name="a_single_text_turn", lines=[ _turn_start(), - json.dumps({"type": "text", "text": "All done."}), + _text("All done."), _turn_end(inp=100, out=20, cache_read=64, cost=0.001), ], ) @@ -201,6 +248,98 @@ def _build_catalogue() -> list[PiScenario]: # (b) the captured live stream: three turns with resolved tool calls. scenarios.append(PiScenario(name="b_tool_call_resolved", lines=list(HAPPY_STREAM))) + # (c) two generations with a tool resolving between them. The TILING case: + # the second window opens at the first `turn_end`, not at its own + # `turn_start`, so the wall clock between the two turns — the model time + # that produced the second one — lands inside a window rather than in no + # bucket at all. Pi was the harness that shipped that defect, and it had a + # unit test but no golden. Every stamp here comes from the reducer's own + # TurnClock, so this scenario stays inside the identity check. + scenarios.append( + PiScenario( + name="c_multi_turn_tiling", + lines=[ + _turn_start(), + _tool_start("call_1", "bash", {"command": "ls"}), + _tool_end("call_1", "bash", "main.py"), + _turn_end(inp=100, out=20, cost=0.001), + _turn_start(), + _text("Listed it."), + _turn_end(inp=50, out=30, cost=0.002), + ], + ) + ) + + # (d) a tool the CLI opens and never resolves — force-closed as `unresolved` + # by the orphan sweep at finalization. + # + # READ THE SNAPSHOT: it carries a `duration_ms` and BOTH execution bounds, + # and that span is subtracted from the generation window. Its + # `execution_completed_at` is the instant the sweep ran, not a completion + # anybody observed, so the duration is manufactured — and `_close_tool`'s + # own comment ("Only a RESOLVED tool contributes: one force-closed without + # a result was never timed") describes a guard it does not have: the test + # is `execution_started_at is not None`, which an orphan passes. + # claude-code's `_finalize_commands` deliberately leaves `duration_ms` + # None in exactly this case, and says why. Captured rather than fixed: + # this scenario is what makes it visible. + scenarios.append( + PiScenario( + name="d_orphaned_tool", + lines=[ + _turn_start(), + _tool_start("call_1", "bash", {"command": "sleep 600"}), + _text("Waiting."), + _turn_end(inp=100, out=20, cost=0.001), + ], + ) + ) + + # (e) the provider error pi's internal retries could not clear, AFTER a + # complete generation. The CLI still exits 0, so `_settle_turn` crashes on + # `stopReason=error` alone — and the partial `pending_turn` must still carry + # that generation and its head/tail. A crash does not un-measure what was + # measured before it. + scenarios.append( + PiScenario( + name="e_error_after_generation", + lines=[ + _turn_start(), + _text("Starting."), + _turn_end(inp=100, out=20, cost=0.001), + _turn_start(), + _turn_end_error("provider returned 529 after 5 retries"), + ], + expects=AgentCrashError, + ) + ) + + # (f) a duplicate `turn_end` with no `turn_start` between — a transport + # hiccup this reducer explicitly promises to survive, since pi retries + # internally. A spent `turn_started_at` left in place reopens the next + # window at the PREVIOUS turn's start and republishes that whole span: + # reproduced as 3000 ms of generation for a 2000 ms turn. It had a unit test + # and no golden. + # + # READ THE SNAPSHOT: it records that the TIMING half of that reset is fixed + # and the CONTENT half is not. `turn_text_parts` / `turn_tool_ids` are + # cleared in `on_turn_start` only, so the second `turn_end` publishes the + # first turn's text a second time, as its own assistant message. The + # argument `on_turn_end`'s comment makes for moving `turn_started_at` out of + # `on_turn_start` applies to those two lists unchanged. Captured here rather + # than fixed: this scenario is what makes it visible at all. + scenarios.append( + PiScenario( + name="f_duplicate_turn_end", + lines=[ + _turn_start(), + _text("First."), + _turn_end(inp=100, out=20, cost=0.001), + _turn_end(inp=10, out=5, cost=0.0001), + ], + ) + ) + return scenarios diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index 18d5fe892..e437ff66a 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -97,6 +97,15 @@ def _expect_window(harness: str, scenario_name: str) -> bool: "codex_f_collab_fallback", # 900 ms collab wait "codex_h_no_turn_completed_crash", # 200 ms of item time — see below "opencode_b_tool_call_resolved", # 17 ms tool interval + # 5 ms tool interval, injected as CLI epoch stamps. OpenCode takes its + # tool bounds from the CLI payload rather than from its own clock, so + # every scenario of this harness that resolves a tool injects them — + # there is no version of this scenario that stays commensurable with a + # sub-millisecond replay. Its TILING property (the second window opens + # at the first `step_finish`) is what the scenario is for, and that is + # still snapshotted; the identity is asserted for this harness by + # tests/test_timing_identity_contract.py, on a scripted clock. + "opencode_c_multi_step_tiling", } ) From 95b52e24e28aed93a7d0d85ce85a8f898e9e1058 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 22:19:00 -0700 Subject: [PATCH 30/54] =?UTF-8?q?fix(timing):=203/7=20=E2=80=94=20a=20naiv?= =?UTF-8?q?e/aware=20mix=20names=20the=20pair=20that=20disagreed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `decompose_turn` subtracts stamps it is handed. Hand it one aware and one naive and Python raises "can't subtract offset-naive and offset-aware datetimes" from inside the arithmetic, straight out of `EventCollector.build_turn_record`, killing the turn with a message naming neither the field nor the harness. `busy_ms` has the same exposure one level down, where the clipping compares each span against the window bounds and the bare error reads "can't compare". `_require_same_awareness` replaces both with a statement of which pair disagreed, which side is aware, and what to do about it. One helper rather than two inline guards, so there is one wording; a test drives all five call sites and asserts the advice half is identical across them. This is unreachable from this repo, and that is the point. Every stamp in `agents/` and `streaming/` is a naive `datetime.now()` — zero `timezone.utc`, `astimezone` or `tzinfo` hits — so the guard protects the SEAM, not a live defect. Which is also why it is a guard and not a lint rule: the exposure that actually matters is a third-party agent registered through the `coder_eval.plugins` SPI, which lives outside `src/coder_eval/agents/` and which no rule scoped to that directory could ever see. The message addresses that reader directly, and tells them to make their stamps naive local rather than normalizing here — so their tool spans and their window bounds keep one basis. Only the MIX raises: all-naive and all-aware both work unchanged. An empty span list is checked NOT AT ALL, bounds included. The comprehension never runs, nothing is compared and nothing is subtracted, so there is no pair for the guard to be about, and raising there would reject a call that has always returned `0.0`. The mixed-bounds empty case is what pins this — the naive one passes either way and cannot tell the two behaviours apart. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- src/coder_eval/timing.py | 60 +++++++++++++++++++ tests/test_timing_close_window.py | 99 ++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 2 deletions(-) diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 8d3dd7587..6e2cd0142 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -76,6 +76,39 @@ def now(self) -> datetime: return self._wall0 + timedelta(seconds=time.monotonic() - self._mono0) +def _require_same_awareness(a: datetime, b: datetime, *, field: str) -> None: + """Raise if one stamp is timezone-aware and the other is naive. + + Subtracting the two raises ``TypeError: can't subtract offset-naive and + offset-aware datetimes`` deep inside the arithmetic below, which surfaces + out of ``EventCollector.build_turn_record`` and kills the turn with a + message naming neither the field nor the harness. This turns that into a + statement of which pair disagreed and which side is aware. + + Unreachable from this repo today, and that is the point: every stamp in + ``agents/`` and ``streaming/`` is a naive ``datetime.now()`` (verified by + grep — zero ``timezone.utc`` / ``astimezone`` / ``tzinfo`` hits), so this + guards the SEAM rather than a live defect. The exposure it is actually for + is a third-party agent registered through the ``coder_eval.plugins`` SPI, + which lives outside ``src/coder_eval/agents/`` and which no lint rule + scoped to that directory could ever see. That is why this is a runtime + guard and not a rule. + + Only the MIX raises. An agent that is internally consistent in UTC is not + this function's problem, and neither is one that is consistently naive. + """ + if (a.tzinfo is None) == (b.tzinfo is None): + return + aware, naive = ("first", "second") if a.tzinfo is not None else ("second", "first") + raise TypeError( + f"{field}: one stamp is timezone-aware and the other is naive (the {aware} is aware, " + + f"the {naive} is not), so the interval between them cannot be measured. Every stamp " + + "this harness records is a naive local `datetime.now()`; if you are writing an agent " + + "outside this repo (the `coder_eval.plugins` SPI), make its stamps naive local too " + + "rather than normalizing here, so its tool spans and its window bounds keep one basis." + ) + + def busy_ms(spans: list[tuple[datetime, datetime]], lo: datetime, hi: datetime) -> float: """Wall milliseconds inside ``[lo, hi]`` where at least ONE span was running. @@ -91,7 +124,27 @@ def busy_ms(spans: list[tuple[datetime, datetime]], lo: datetime, hi: datetime) Clipping to ``[lo, hi]`` is the other half: a tool that opened before this window only spent part of its life inside it, and only that part is not generation time here. + + Every stamp reaching this function is a naive ``datetime.now()`` today — + that is true of all of ``agents/`` and ``streaming/`` — so a mixed pair + means an agent has started recording aware stamps, and + ``_require_same_awareness`` names which pair rather than letting a bare + ``TypeError`` escape from the arithmetic. The spans are checked as well as + the bounds, not instead of them: the clipping below compares each span + against BOTH ``lo`` and ``hi``, so a guard on the bounds alone would leave + this function uncovered by it. + + An EMPTY span list is checked NOT AT ALL, bounds included. The comprehension + never runs, nothing is compared and nothing is subtracted, so there is no + pair for the guard to be about — and raising there would reject a call that + has always returned ``0.0``. """ + if not spans: + return 0.0 + _require_same_awareness(lo, hi, field="busy_ms window") + for span_start, span_end in spans: + _require_same_awareness(lo, span_start, field="busy_ms window vs a tool span's start") + _require_same_awareness(hi, span_end, field="busy_ms window vs a tool span's end") clipped = sorted((max(s, lo), min(e, hi)) for s, e in spans if min(e, hi) > max(s, lo)) if not clipped: return 0.0 @@ -229,6 +282,11 @@ def decompose_turn( MEASURE rather than for what they contain is the whole point; see docs/agents/HARNESS_PARITY.md for the per-harness composition. + Every stamp reaching this function is a naive ``datetime.now()`` — that is + true of all of ``agents/`` and ``streaming/`` today — so a mixed pair means + an agent has started recording aware stamps, and ``_require_same_awareness`` + says so rather than letting a bare ``TypeError`` escape and kill the turn. + ``None`` means never measured — a turn that produced no generation, or a snapshot taken before the terminal event. Never 0.0, which would claim a measurement was taken and came back instant (CE058). A measured inversion @@ -245,9 +303,11 @@ def decompose_turn( spans = tool_spans or [] head = tail = None if first_started_at is not None and agent_started_at is not None: + _require_same_awareness(agent_started_at, first_started_at, field="harness_startup_ms") elapsed = (first_started_at - agent_started_at).total_seconds() * 1000.0 head = max(elapsed - busy_ms(spans, agent_started_at, first_started_at), 0.0) if last_completed_at is not None and agent_ended_at is not None: + _require_same_awareness(last_completed_at, agent_ended_at, field="harness_teardown_ms") elapsed = (agent_ended_at - last_completed_at).total_seconds() * 1000.0 tail = max(elapsed - busy_ms(spans, last_completed_at, agent_ended_at), 0.0) return head, tail diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py index ec261f493..b47abe22c 100644 --- a/tests/test_timing_close_window.py +++ b/tests/test_timing_close_window.py @@ -7,12 +7,12 @@ """ import importlib.util -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from pathlib import Path import pytest -from coder_eval.timing import close_window, union_ms +from coder_eval.timing import busy_ms, close_window, decompose_turn, union_ms MARK = datetime(2026, 9, 11, 12, 0, 0) @@ -142,6 +142,101 @@ def test_mark_is_keyword_only_and_has_no_default(self): close_window(now=_at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[call-arg] +class TestNaiveAwareMix: + """A mixed naive/aware pair fails loudly at the seam, not cryptically inside it. + + `decompose_turn`'s bare arithmetic raised `TypeError: can't subtract + offset-naive and offset-aware datetimes` straight out of + `EventCollector.build_turn_record`, killing the turn with a message naming + neither the field nor the harness. + + Unreachable from this repo — every stamp in `agents/` and `streaming/` is a + naive `datetime.now()`. The exposure is a THIRD-PARTY agent registered + through the `coder_eval.plugins` SPI, which lives outside + `src/coder_eval/agents/` and which a lint rule scoped to that directory + could never see. That is why this is a guard and not a rule. + """ + + AWARE = MARK.replace(tzinfo=UTC) + + def test_a_mixed_head_names_the_field(self): + with pytest.raises(TypeError, match="harness_startup_ms"): + decompose_turn(self.AWARE, None, MARK, None) + + def test_a_mixed_tail_names_the_field(self): + with pytest.raises(TypeError, match="harness_teardown_ms"): + decompose_turn(None, self.AWARE, None, MARK) + + def test_a_mixed_busy_ms_window_names_the_field(self): + with pytest.raises(TypeError, match="busy_ms window"): + busy_ms([(MARK, _at(500))], MARK, self.AWARE) + + def test_a_mixed_span_start_is_caught_too_and_not_by_the_bare_comparison(self): + """The clipping compares each span against the window. + + Left unguarded that raises "can't compare offset-naive and offset-aware + datetimes" — the exact message this replaces — so checking only the + bounds would leave the guard not covering its own function. + """ + with pytest.raises(TypeError, match="tool span's start"): + busy_ms([(self.AWARE, self.AWARE)], MARK, _at(1000)) + + def test_a_mixed_span_end_is_caught_by_its_own_branch(self): + """The end is a separate check against `hi`, so it needs its own case. + + A span whose START matches the window and whose END does not passes the + previous branch and must still raise — otherwise that branch is live, + reachable and unexercised. + """ + with pytest.raises(TypeError, match="tool span's end"): + busy_ms([(MARK, self.AWARE)], MARK, _at(1000)) + + def test_one_wording_for_every_call_site(self): + """One helper, so one template — checked across ALL FIVE call sites. + + Two inline guards would drift, and a test asserting the text would then + pin only whichever one it happened to call. What varies between sites + is deliberate and only that: the field name, and which side is aware. + Everything after that clause is the advice, and it must be identical or + the sites are no longer sharing a helper. + """ + advice = set() + for call in ( + lambda: decompose_turn(self.AWARE, None, MARK, None), # head + lambda: decompose_turn(None, self.AWARE, None, MARK), # tail + lambda: busy_ms([(MARK, _at(500))], MARK, self.AWARE), # window bounds + lambda: busy_ms([(self.AWARE, self.AWARE)], MARK, _at(1000)), # span start + lambda: busy_ms([(MARK, self.AWARE)], MARK, _at(1000)), # span end + ): + with pytest.raises(TypeError) as excinfo: + call() + message = str(excinfo.value) + assert "is timezone-aware and the other is naive" in message + advice.add(message.split("), ", 1)[1]) + assert len(advice) == 1, advice + + def test_all_naive_is_unchanged(self): + assert decompose_turn(_at(1000), _at(2000), MARK, _at(3000)) == (1000.0, 1000.0) + + def test_all_aware_works_because_the_guard_is_about_the_mix(self): + def aware(ms: int) -> datetime: + return _at(ms).replace(tzinfo=UTC) + + assert decompose_turn(aware(1000), aware(2000), self.AWARE, aware(3000)) == (1000.0, 1000.0) + + def test_an_empty_span_list_is_not_checked_at_all(self): + """Not even the bounds, and the MIXED case is the one that proves it. + + With no spans the comprehension never runs: nothing is compared and + nothing is subtracted, so there is no pair for the guard to be about. + Checking the bounds anyway rejected a call that has always returned + `0.0` — asserted here on mixed bounds, because the naive case would + pass either way and so could not tell the two behaviours apart. + """ + assert busy_ms([], MARK, _at(1000)) == 0.0 + assert busy_ms([], MARK, self.AWARE) == 0.0 + + class TestUnionMs: """`union_ms` is `busy_ms` over the spans' own extent. From 306015df2538264761bb6522b324250041e6eaf5 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 22:41:44 -0700 Subject: [PATCH 31/54] =?UTF-8?q?feat(timing):=204/7=20=E2=80=94=20one=20m?= =?UTF-8?q?eaning=20for=20harness=5Fstartup=5Fms,=20on=20all=20five?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The field answered a different question per harness. codex, opencode and pi measured the wall clock before their CLI emitted its first event. claude-code and antigravity measured NOTHING: both stamped their first generation window's mark when the turn state was built, before `AgentStartEvent` was emitted, so `decompose_turn`'s `max(..., 0.0)` produced the `0.0` they published. A clamped inversion presented as "measured, and instant" — the exact confusion CE058 exists to prevent everywhere else — while everything those harnesses spent before their first model output was booked as the first generation instead: ~3.6 s per turn on claude-code and ~4.7 s on antigravity, inflating every generation figure, the Generation split and the 10 s slow-generation bar on the two most-used harnesses. The head is now defined once, for all five: wall clock from the turn starting until the harness first observed model output. That instant is also where the harness opens its first generation window, so the two buckets stay disjoint and the four-bucket identity still closes — verified to the millisecond by `test_timing_identity_contract.py`, which is the only thing in the suite that could see this move. `GOLDEN_REGEN=1` produces a ZERO diff: `SCRUB_KEYS` masks every value that changed, which is the audit's P1 demonstrated on the very change it was written about. Both re-seeds fire ONCE per turn. `message_start` and `Step` each arrive many times, and re-seeding on every one would stop the windows tiling and drop the gap before the next emission into no bucket — the defect Pi shipped with. Neither flag needs a reset: a fresh turn state is built per `communicate()`. Antigravity's is gated on the step SOURCE. The SDK streams SYSTEM and USER steps as well as MODEL ones, and seeding on those would put the mark before the model spoke and hand the remainder back to the first generation — the defect being fixed, one layer in. An unrecognized source degrades to the old behaviour rather than to a wrong one. The rejection this overturns rested on claude-code being an in-process SDK. It is not: `claude-agent-sdk` spawns the `claude` CLI over `anyio.open_process` and `_pump_messages` calls `query()` once per `communicate()` — a fresh CLI per turn. All 8 sites asserting otherwise are gone; the old reasoning is kept in HARNESS_PARITY.md as labelled HISTORY rather than deleted. Nor was antigravity the in-process counterexample it was described as. It spawns a `localharness` binary too — once, in `start()`, held across turns. The distinction that matters is WHEN a harness spawns its process, not whether, and that is what the docs now say. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- docs/agents/HARNESS_PARITY.md | 113 ++++++--- .../__tests__/message-timeline.test.tsx | 7 +- .../app/runs/[id]/[...task]/_sections.tsx | 2 +- .../lib/__tests__/harnessOverhead.test.ts | 5 +- evalboard/lib/__tests__/runs.test.ts | 6 +- src/coder_eval/agents/antigravity_agent.py | 57 +++++ src/coder_eval/agents/claude_code_agent.py | 48 ++++ src/coder_eval/models/results.py | 21 +- src/coder_eval/timing.py | 19 +- tests/lint/rules/ce058_no_timing_literal.py | 5 +- tests/test_agent_telemetry.py | 235 +++++++++++++++--- tests/test_antigravity_agent.py | 145 ++++++++++- tests/test_event_collector.py | 10 +- tests/test_timing_identity_contract.py | 18 +- 14 files changed, 591 insertions(+), 100 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 7ef71f122..d397ccfaa 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -25,8 +25,8 @@ wall clock its numbers account for. | Field | claude-code | codex | antigravity | opencode | pi | |---|---|---|---|---|---| | `generation_duration_ms` source | harness clock: previous SDK event → this message | SDK item stamps, minus tool execution inside the window | harness clock: previous flush → this flush, minus tool execution inside the window | harness clock: previous `step_finish` → this one, minus tool execution inside the window | harness clock: previous `turn_end` → this one, minus tool execution inside the window | -| what the **first** window covers | turn start → msg0, so dispatch + TTFT are INSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | turn start → first flush, so dispatch + TTFT are INSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | -| `harness_startup_ms` (turn head) | 0.0 — the window above already covers it | ~3.1 s — CLI boot fused with TTFT | 0.0 — the window above already covers it | ~2.5 s — CLI boot fused with TTFT | ~0.23 s — CLI boot fused with TTFT | +| what the **first** window covers | the first `message_start`, so CLI boot + TTFT are OUTSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | the first `Step`, so dispatch + TTFT are OUTSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | +| `harness_startup_ms` (turn head) | ~3.6 s — CLI boot fused with TTFT | ~3.1 s — CLI boot fused with TTFT | ~4.7 s — dispatch fused with TTFT (its harness process is spawned once at startup, not per turn) | ~2.5 s — CLI boot fused with TTFT | ~0.23 s — CLI boot fused with TTFT | | `harness_teardown_ms` (turn tail) | ~1.3 s | ~13 ms | ~7 ms | ~26 ms | ~19 ms | | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | | `execution_started_at` / `execution_completed_at` | derived from the measured duration | SDK stamps (both, or neither) | measured at ACTIVE / DONE | measured | measured | @@ -140,36 +140,74 @@ figures in the table above are means of six live `tasks/hello_date` turns per harness and move with CLI cache warmth, so read their ORDER OF MAGNITUDE, not the digits. -What the head CONTAINS differs per harness and is deliberately **not** -decomposed, because the divergence is real and unfixable in both directions: - -- On an **in-process SDK** (claude-code, antigravity) the first generation - window starts at turn entry, so dispatch and time-to-first-token are already - inside it. Excluding them is not possible — neither harness stamps a - per-message arrival to fall back to, and `started_at == completed_at` would - be the CE059 defect. **Read their `0.0` head as "nothing is left over", not - as a measured interval**: the window actually opens marginally BEFORE the - `AgentStartEvent` stamp (claude-code builds its turn state, then - `_build_claude_query`, and only then emits the event), so the raw figure is - negative and clamps. The setup between those two points is therefore booked - as generation — **measured at 0.03 ms, and 0.10 ms with four plugin roots**, - so it is the sub-millisecond skew the clamp exists for rather than hidden - overhead. Emitting the event earlier would make the `0.0` a measurement - instead of a clamp but would not change it, since the window's start stamp - also precedes the build; only re-seeding the window after the build would - surface that time, and that is the seeding change ruled out above. - `TestClaudeHeadIsStructurallyZero` pins the build cost so this stays true. -- On a **subprocess harness** (codex, opencode, pi) the first window cannot - start before the first event the CLI emits, so the head is one opaque - interval fusing CLI boot, provider resolution, dispatch and TTFT. Measured on - OpenCode: the process spawns in ~3 ms and its first `step_start` lands at - ~3.9 s, with no marker in between. +**The head means one thing on all five.** It is the wall clock from the turn +starting until the harness first observed **model output**, and that instant is +also where the harness opens its first generation window — which is what keeps +the head and the generation disjoint so the four-bucket identity still closes. +The per-harness first-output signal: + +| harness | first observed model output | +|---|---| +| claude-code | the first `message_start` stream event | +| codex | the first SDK item's own start | +| antigravity | the first `Step` | +| opencode | the first `step_start` | +| pi | the first `turn_start` | + +What the head CONTAINS still differs, and that part is deliberately **not** +decomposed. **All five spawn a process** — the distinction is WHEN. claude-code, +codex, opencode and pi spawn theirs per turn, so their head fuses that boot with +provider resolution, dispatch and TTFT, and the stream carries no marker between +them (measured on OpenCode: the process spawns in ~3 ms and its first +`step_start` lands at ~3.9 s). Antigravity spawns its bundled `localharness` +binary ONCE, in `start()`, and holds it across every `communicate()` — so there +is no boot inside the turn for its head to contain, and its head is dispatch plus +TTFT. That is a real property of the harness rather than a measurement artifact, +which is as far as unification can honestly go. So the fields are named for the **interval they measure**, never for what they contain. Do not rename them `cli_boot_ms` or `ttft_ms` — that would claim a split nobody performed. A measured `0.0` head is an answer; `None` is what "never measured" looks like (a turn that produced no assistant message). +**The table's head figures are SINGLE-TURN.** They are means of six live +`tasks/hello_date` turns. A simulation (dialog) task runs each turn as its own +`communicate()`, so on the per-turn-spawn harnesses turns 2..N book a full +process boot *plus* session-transcript replay into `harness_startup_ms`, and +will read well above these numbers. That is correct under the definition and is +an improvement — the same time was previously hidden inside the first +generation — but do not read a dialog run's larger head as a regression against +this table. + +**HISTORY — why claude-code and antigravity used to report `0.0`.** Both +stamped their first window's mark when the turn state was built, *before* +`AgentStartEvent` was emitted, so the head was a small negative that +`decompose_turn` clamped. The `0.0` was therefore a clamped inversion published +as "measured, and instant" — the exact confusion CE058 exists to prevent +everywhere else — and everything those harnesses spent before their first model +output was booked as the first generation instead: **~3.6 s per turn on +claude-code and ~4.7 s on antigravity**, inflating every generation figure, the +Generation split percentages and the 10 s slow-generation bar on the two +most-used harnesses. + +The re-seed was rejected once, on the premise that claude-code runs the model +in-process so "the interval from turn entry to the first message is msg0's +generation". That premise was simply wrong: `claude-agent-sdk` spawns the +`claude` CLI as a subprocess (`anyio.open_process`) and `_pump_messages` calls +`query()` once per `communicate()` — a fresh CLI per turn, the same shape as +codex, opencode and pi. Nor was antigravity ever the in-process counterexample +it was described as: it spawns `localharness` too, just once at `start()` +rather than per turn. + +Both re-seeds are **once per turn**. `message_start` and `Step` each arrive +many times; re-seeding on every one would stop the windows tiling and drop the +gap before the next emission — a tool result landing, then the next request +going out — into no bucket at all, which is the defect Pi shipped with. Neither +flag needs a reset: both harnesses build a fresh turn state per +`communicate()`, so it is per-attempt by construction. A turn that streams no +`message_start` / no `Step` never re-seeds, keeps the turn-entry mark and +clamps to `0.0` exactly as before. + **Why Codex leaves `generation_completed_at` as `None`.** It means "when the model finished emitting the `tool_use` block". Codex's stream does not carry that per tool; deriving it from the flush time would be a guess. Note also that @@ -222,11 +260,24 @@ the other harness where a missing id can still collapse a turn. ### Time to first token is not measured -Nothing records it today. There is no `ttft` or `first_token` symbol anywhere -in `src/`, `evalboard/`, `docs/` or `tests/`, and it **cannot be derived from -what is stored**: `generation_duration_ms` is the whole window, and the latency -in question is a sub-interval of it. This section is the design, so the next -person to want it does not re-derive it. Nothing below is implemented. +Nothing records it **as its own field** today — there is no `ttft` or +`first_token` symbol anywhere in `src/`, `evalboard/`, `docs/` or `tests/`. + +But most of its value for the TURN is already delivered: `harness_startup_ms` +now measures the wall clock up to the harness's first observed model output on +every harness, which is a time-to-first-output latency for the first generation. +Two things a separate `first_delta_latency_ms` would still add — and the design +below is about both, so do not read this paragraph as retiring it: + +1. **Per-generation latency**, not just the first. The design measures from + EVERY window's mark, so it reports a first-delta latency for each emission; + the head covers only the interval before the first one. +2. **The boot/prefill split** inside the head on the per-turn-spawn harnesses — + which is the part that genuinely cannot be derived, because no stream carries + a marker between them. + +This section is the design, so the next person to want it does not re-derive it. +Nothing below is implemented. **The mark is the measure-from point, and every reducer already keeps one.** Each one records the moment its current generation window opened — which is diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx index de4c431d5..55e6bc335 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx @@ -726,9 +726,10 @@ describe("MessageTimelineSection — Startup and Teardown cells", () => { }); test("a measured zero renders as 0ms, not as an em-dash", () => { - // claude-code and antigravity really do measure ~0 here — their first - // generation window already covers dispatch. "—" would report that - // honest measurement as a missing one. + // A head of 0 stays representable: a turn can reach its first model + // output with nothing measurable in front of it, and a clamped + // inversion is still a measurement because both ends were observed. + // "—" would report that as a missing one. renderStrip({ taskDurationSeconds: 10, harnessStartupMs: 0, diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index aae93cf99..671ca4344 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -454,7 +454,7 @@ export function MessageTimelineSection({
{messageCount}
-
+
Startup
diff --git a/evalboard/lib/__tests__/harnessOverhead.test.ts b/evalboard/lib/__tests__/harnessOverhead.test.ts index a39fc6ac4..e1ae71391 100644 --- a/evalboard/lib/__tests__/harnessOverhead.test.ts +++ b/evalboard/lib/__tests__/harnessOverhead.test.ts @@ -68,8 +68,9 @@ describe("readTaskDetail: harness startup/teardown", () => { }); test("a measured zero head is preserved as 0", async () => { - // An in-process SDK's first generation window already covers dispatch, - // so 0.0 is its honest answer and must not read as "never measured". + // A head of 0.0 stays representable: a turn can reach its first model + // output with nothing measurable in front of it. It must not read as + // "never measured", which is what null means. await writeTask([{ harness_startup_ms: 0.0, harness_teardown_ms: 834.7 }]); const { readTaskDetail } = await loadRuns(); const detail = await readTaskDetail(RUN, TASK); diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index bef4fa8dd..2a0cced0c 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -258,8 +258,10 @@ describe("sumHarnessOverhead", () => { }); test("a measured zero is a measurement and still sums", () => { - // An in-process SDK whose first generation window already covers - // dispatch legitimately reports 0.0 — that is a number, not a gap. + // A harness that reached its first model output with nothing + // measurable in front of it legitimately reports 0.0 — a clamped + // inversion where both ends were still observed. That is a number, + // not a gap, and the assertion holds however the head is produced. expect( sumHarnessOverhead([{ harness_startup_ms: 0, harness_teardown_ms: 3.5 }]), ).toEqual({ startupMs: 0, teardownMs: 3.5 }); diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 542ab00e0..b725f20b4 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -866,6 +866,9 @@ def __init__( # recorded bounds and the measured duration describe one span. # Advanced only by a flush that actually emitted a message. self._gen_mark_wall: datetime = clock.now() + # Re-seeded ONCE, at the first observed Step. See + # `_seed_first_generation_window`. + self._first_output_seen: bool = False # Execution intervals of tools that CLOSED since the mark. This harness # interleaves tool calls into one generation — the Step for the tool # arrives and only a later usage_metadata Step cuts the message — so a @@ -894,11 +897,65 @@ def max_turns_reached(self) -> bool: """ return self.max_turns is not None and self.collector.visible_turn_count >= self.max_turns + def _seed_first_generation_window(self, source: Any) -> None: + """Move the first window's mark to the first observed MODEL output. + + ``harness_startup_ms`` is defined as the wall clock from the turn + starting until the harness first observed model output, and that instant + is also where the first generation window opens — which is what keeps + the head and the generation disjoint so the four-bucket identity still + closes. + + Without this ``_gen_mark_wall`` is stamped when the turn state is built, + BEFORE ``AgentStartEvent`` is emitted, so the head is a small negative + that ``decompose_turn`` clamps to ``0.0`` — a clamped inversion + published as "measured, and instant", which is the exact confusion CE058 + exists to prevent everywhere else. Everything before the first ``Step`` + — dispatch and time to first token — was booked as the first + generation instead: ~4.7 s per turn on this harness, measured against a + later-window median of 3.3 s. + + What differs from claude-code is not in-process versus subprocess — + this harness spawns a ``localharness`` binary too. It is spawned ONCE, + in ``start()``, and held across every ``communicate()``, so there is no + boot inside a turn for the head to contain: it is dispatch plus time to + first token. claude-code spawns a fresh CLI per turn and so fuses that + boot in. The head means the same thing on both; only its COMPOSITION + differs, which is a real property of the harness rather than a + measurement artifact. + + GATED ON ``source``, because the field is defined as model output and + the SDK streams Steps that are not. ``StepSource`` carries ``SYSTEM`` + and ``USER`` besides ``MODEL``, and ``StepType`` carries + ``SYSTEM_MESSAGE`` / ``COMPACTION`` / ``FINISH``; the SDK's event + processor queues every ``step_update`` verbatim, so a turn can + legitimately open with one. Seeding on such a Step would put the mark + BEFORE the model spoke and hand the remainder back to msg0's + generation, which is the defect this method exists to remove. The same + gate guards text streaming a few lines below, for the same reason. + + ONCE PER TURN, and that is the whole contract. ``process_step`` runs for + every Step in the turn; re-seeding on each would stop the windows tiling + and drop the gap before the next emission into no bucket at all, which + is the defect pi shipped with. The flag needs no reset: a fresh turn + state (and a fresh ``TurnClock``) is built per ``communicate()``, so it + is per-attempt by construction. + + A turn that streams no MODEL Step at all never latches, keeps the + turn-entry mark and clamps to ``0.0`` exactly as before — the same + fail-safe degradation as an unrecognized source. + """ + if self._first_output_seen or _enum_value(source) != _SOURCE_MODEL: + return + self._first_output_seen = True + self._gen_mark_wall = self.clock.now() + def process_step(self, step: Any) -> None: """Route one streamed ``Step`` to events + transcript reconstruction.""" stype = _enum_value(step.type) sstatus = _enum_value(step.status) ssource = _enum_value(step.source) + self._seed_first_generation_window(ssource) starget = _enum_value(step.target) done = sstatus in (_STATUS_DONE, _STATUS_ERROR) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index f5e9fb349..537a8e54c 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -246,6 +246,9 @@ def __init__( self.last_assistant_message_index: int | None = None self.last_event_monotonic: float = turn_start_time self.last_event_wall: datetime = datetime.now() + # Re-seeded ONCE, at the first observed model output. See + # `_seed_first_generation_window`. + self.first_output_seen: bool = False # SDK ResultMessage capture. self.sdk_result_usage: dict[str, Any] | None = None @@ -497,12 +500,57 @@ def on_result_message(self, message: Message) -> None: last_msg.cache_read_tokens = int(self.sdk_result_usage.get("cache_read_input_tokens", 0) or 0) last_msg.reasoning_tokens = int(self.sdk_result_usage.get("reasoning_tokens", 0) or 0) + def _seed_first_generation_window(self) -> None: + """Move the first window's mark to the first observed model output. + + ``harness_startup_ms`` is defined as the wall clock from the turn + starting until the harness first observed model output, and that instant + is also where the first generation window opens — which is what keeps + the head and the generation disjoint so the four-bucket identity still + closes. + + Without this the two marks are stamped in ``__init__``, BEFORE + ``AgentStartEvent`` is emitted, so the head is a small negative that + ``decompose_turn`` clamps to ``0.0`` — a clamped inversion published as + "measured, and instant", which is the exact confusion CE058 exists to + prevent everywhere else. Everything the CLI spent booting, resolving a + provider and reaching its first token was booked as msg0's generation + instead: ~3.6 s per turn on this harness, inflating every generation + figure, the Generation split and the 10 s slow-generation bar. + + The old rejection rested on this harness running the model in-process. + It does not: ``claude-agent-sdk`` spawns the ``claude`` CLI over + ``anyio.open_process`` and ``_pump_messages`` calls ``query()`` once + per ``communicate()`` — a fresh CLI per turn, the same shape as codex, + opencode and pi. + + ONCE PER TURN, and that is the whole contract. ``message_start`` arrives + for every API call in the turn; re-seeding on each would stop the + windows tiling and drop the gap before the next emission — a tool result + landing, then the next request going out — into no bucket at all, which + is the defect pi shipped with. The flag needs no reset: a fresh + ``_ClaudeTurnState`` is built per ``communicate()``, so it is + per-attempt by construction. If a future harness reuses a turn state, + the reset belongs there and not here. + + A turn with no ``message_start`` — partial streaming off, a mocked + ``query()``, a crash before the first event — never calls this, keeps + the turn-entry mark and clamps to ``0.0`` exactly as before. That is the + correct degradation rather than a gap. + """ + if self.first_output_seen: + return + self.first_output_seen = True + self.last_event_monotonic = time.monotonic() + self.last_event_wall = datetime.now() + def on_stream_event(self, message: Message) -> None: """Recover cumulative output_tokens from raw ``message_start`` / ``message_delta`` stream events (handles both sub-cases internally).""" evt: dict[str, Any] = getattr(message, "event", None) or {} evt_type = evt.get("type") if evt_type == "message_start": + self._seed_first_generation_window() mid = (evt.get("message") or {}).get("id") self.current_stream_message_id = mid if isinstance(mid, str) else None elif evt_type == "message_delta": diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 765fc2816..9b8f152ec 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -328,14 +328,19 @@ class TurnRecord(BaseModel): harness_startup_ms: float | None = Field( default=None, description=( - "Wall milliseconds between the agent turn starting and the first generation window " - "opening, measured between AGENT EVENT stamps — not from timestamp/duration_seconds " - "above, which are orchestrator-level and a slightly different clock, so a consumer " - "recomputing this from those will get a near-but-not-equal number. Its COMPOSITION " - "differs per harness and is deliberately not decomposed: on an in-process SDK the " - "first window already covers dispatch and time-to-first-token so this reads ~0, while " - "on a subprocess harness it fuses CLI boot, provider resolution, dispatch and TTFT, " - "which the event stream gives no marker to separate. See docs/agents/HARNESS_PARITY.md. " + "Wall milliseconds from the agent turn starting until the harness first observed " + "MODEL OUTPUT — one definition on all five, and the same instant at which the harness " + "opens its first generation window, which is what keeps the two buckets disjoint. " + "Measured between AGENT EVENT stamps, not from timestamp/duration_seconds above, " + "which are orchestrator-level and a slightly different clock, so a consumer " + "recomputing this from those will get a near-but-not-equal number. Only its " + "COMPOSITION differs per harness, and that difference is a real property rather than " + "a measurement artifact: a harness that spawns its process PER TURN (claude-code, " + "codex, opencode, pi) fuses that boot, provider resolution, dispatch and TTFT here, " + "while one that spawns it once at startup and holds it across turns (antigravity) has " + "no boot inside the turn to fuse in. It is deliberately not decomposed further " + "— no stream carries a marker between those parts. " + "See docs/agents/HARNESS_PARITY.md. " "None when the turn produced no assistant message — never 0.0, which would mean " "'measured, and instant'." ), diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 6e2cd0142..712697ff9 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -273,14 +273,17 @@ def decompose_turn( call this, because ``task.json`` carries no ``AgentStartEvent`` stamp to recompute a head from. - What the head CONTAINS differs per harness and is deliberately NOT split. - On an in-process SDK the first window already covers dispatch and - time-to-first-token, so this reads ~0; on a subprocess harness it fuses CLI - boot, provider resolution, dispatch and TTFT, and the stream carries no - marker between them — measured on OpenCode, the process spawns in 3 ms and - the first event lands at 3921 ms. Naming these for the interval they - MEASURE rather than for what they contain is the whole point; see - docs/agents/HARNESS_PARITY.md for the per-harness composition. + The head means ONE thing on all five: wall clock from the turn starting + until the harness first observed model output. Every reducer opens its first + generation window at that same instant, which is what keeps the two buckets + disjoint. What the head CONTAINS still differs and is deliberately NOT + split: a harness that spawns its process PER TURN fuses that boot, provider + resolution, dispatch and TTFT — measured on OpenCode, the process spawns in + 3 ms and the first event lands at 3921 ms — while one that spawns it once at + startup and holds it across turns has no boot inside the turn to fuse in. No + stream carries a marker between those parts. Naming these for the + interval they MEASURE rather than for what they contain is the whole point; + see docs/agents/HARNESS_PARITY.md for the per-harness composition. Every stamp reaching this function is a naive ``datetime.now()`` — that is true of all of ``agents/`` and ``streaming/`` today — so a mixed pair means diff --git a/tests/lint/rules/ce058_no_timing_literal.py b/tests/lint/rules/ce058_no_timing_literal.py index 9021fed8e..2abe303f5 100644 --- a/tests/lint/rules/ce058_no_timing_literal.py +++ b/tests/lint/rules/ce058_no_timing_literal.py @@ -21,8 +21,9 @@ was never timed at either end, and a ``0.0`` there would claim the harness started instantly, which is exactly the reading that sends a real gap into the evalboard's ``Unaccounted`` cell while a named bucket says it was measured at -zero. ``0.0`` IS the right answer for an in-process SDK whose first generation -window already covers dispatch, so the two values must stay distinguishable. +zero. A measured ``0.0`` remains a legitimate answer — a window subtracted +down to nothing by the tool execution inside it, or a clamped inversion where +both ends really were observed — so the two values must stay distinguishable. Five syntactic forms, one invariant, one id — the shapes the codebase actually produced: diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py index 646c4c20e..b79e06818 100644 --- a/tests/test_agent_telemetry.py +++ b/tests/test_agent_telemetry.py @@ -1,6 +1,8 @@ """Tests for command telemetry status tracking (V2 fix).""" import time +from datetime import datetime, timedelta +from types import SimpleNamespace import pytest @@ -1280,33 +1282,36 @@ async def mock_query(prompt, options): agent_module.query = original_query -class TestClaudeHeadIsStructurallyZero: - """Why claude-code's `harness_startup_ms` is 0.0, and why that is left alone. - - `_ClaudeTurnState.__init__` stamps `last_event_wall`, which becomes the - FIRST generation window's `started_at`. `_build_claude_query` runs next, - and only then is `AgentStartEvent` emitted. So the head — agent start to - first window — is a small NEGATIVE that `decompose_turn` clamps to 0.0. - - Two changes were considered and rejected, and this class pins the facts - each rejection rests on, because both are the kind of thing that rots - silently: - - 1. *Emit `AgentStartEvent` before `_build_claude_query`.* It would turn the - clamp into a genuine measurement, but the value stays ~0 either way — - `last_event_wall` is stamped before the build too, so the build sits - inside msg0's window regardless. The cost is real: the event carries - `model=effective_model`, which the build resolves, so moving it means - the live renderers show the configured model rather than the effective - one. Not worth it for a sub-millisecond gain. - - 2. *Re-seed the first window after the build.* That WOULD surface the build - cost, and it is the generation-window seeding change ruled out in - docs/agents/HARNESS_PARITY.md — for an in-process SDK the interval from - turn entry to the first message is msg0's generation. - - Both rejections assume the build is cheap. This test is what keeps that - assumption honest. +class TestClaudeHeadIsMeasuredAtFirstOutput: + """claude-code's head is the wall clock up to the first observed model output. + + `_ClaudeTurnState.__init__` stamps `last_event_wall`, and + `_seed_first_generation_window` re-stamps it at the first `message_start`. + So the first window opens where the model first spoke, and the CLI spawn, + provider resolution and time to first token before it are the head. + + `_build_claude_query` is NOT in the head: it runs at `communicate`'s + `:1095`, before `AgentStartEvent` is emitted at `:1106`, so it precedes the + head's own start stamp. It used to sit inside msg0's generation window + (`last_event_wall` was stamped at state construction, ahead of the build); + it now sits inside `duration_seconds` but outside all four buckets, as + unexplained residual. That is why the budget below still matters and why it + is not the same guard it was: at 0.03-0.10 ms the residual is noise, and + the two tests keep it that way. + + It used to be `0.0`, and that was a CLAMPED NEGATIVE rather than a + measurement: both marks were stamped before `AgentStartEvent` was emitted, + so `decompose_turn`'s `max(..., 0.0)` produced it. The rejection rested on + claude-code running the model in-process. It does not — `claude-agent-sdk` + spawns the `claude` CLI over `anyio.open_process` and `_pump_messages` + calls `query()` once per `communicate()`, a fresh CLI per turn. + + The two budget tests below survive the rewrite with their meaning INVERTED. + `_build_claude_query`'s cost now lands in the head rather than inside msg0's + generation, so they no longer guard "the build is cheap enough to leave + hidden by the clamp" — they guard "our own setup is a negligible part of a + head that is now published", which is what makes the head readable as the + harness's latency rather than as ours. """ # Measured at 0.03 ms bare and 0.10 ms with four plugin roots. The bound is @@ -1332,18 +1337,18 @@ def _build_ms(**config_kwargs) -> float: samples.append((time.perf_counter() - started) * 1000.0) return min(samples) - def test_the_query_build_is_cheap_enough_to_leave_inside_msg0(self): + def test_the_query_build_is_a_negligible_part_of_the_published_head(self): elapsed = self._build_ms() assert elapsed < self.BUDGET_MS, ( f"_build_claude_query took {elapsed:.2f} ms, over the {self.BUDGET_MS} ms budget. It runs " - "BETWEEN the first generation window's start stamp and the AgentStartEvent, so this time " - "is booked as model generation and the clamped 0.0 head hides it. At a few hundred " - "microseconds that is the right trade; at this size it is not — revisit the two options " - "in this class's docstring." + "BEFORE the AgentStartEvent, so it is inside the turn's duration_seconds but outside " + "all four buckets — unexplained residual that no bucket accounts for. At a few hundred " + "microseconds that is noise; at this size the four buckets would visibly stop summing " + "to the turn and the gap would be ours, not the harness's." ) def test_plugin_resolution_does_not_change_that(self, tmp_path): - """The rejected proposal's motivating case was a plugin-heavy task.""" + """A plugin-heavy task is where our own setup could plausibly dominate.""" (tmp_path / "skills").mkdir() roots = [{"type": "local", "path": str(tmp_path)} for _ in range(4)] elapsed = self._build_ms(plugins=roots) @@ -1351,3 +1356,167 @@ def test_plugin_resolution_does_not_change_that(self, tmp_path): f"_build_claude_query with 4 plugin roots took {elapsed:.2f} ms, over the " f"{self.BUDGET_MS} ms budget — see the sibling test for why that matters." ) + + +class TestClaudeFirstWindowReseed: + """The first `message_start` moves the window mark; a later one must not. + + Driven at `_ClaudeTurnState` with both clocks patched off one counter. + claude-code derives the window's DURATION from `time.monotonic()` and its + BOUNDS from `datetime.now()`, so patching one leaves the other real and + these tests would measure nothing while still passing. + """ + + BASE = datetime(2026, 9, 11, 9, 0, 0) + + class _Stepped(datetime): + at_ms = 0.0 + + @staticmethod + def now(tz=None): # type: ignore[override] + return TestClaudeFirstWindowReseed.BASE + timedelta(milliseconds=TestClaudeFirstWindowReseed._Stepped.at_ms) + + def _state(self, monkeypatch): + from coder_eval.agents import claude_code_agent as claude_module + from coder_eval.agents.claude_code_agent import ClaudeCodeAgent, _ClaudeTurnState + from coder_eval.streaming.callbacks import CompositeStreamCallback + from coder_eval.streaming.collector import EventCollector + + stepped = self._Stepped + stepped.at_ms = 0.0 + monkeypatch.setattr(claude_module, "datetime", stepped) + monkeypatch.setattr(claude_module, "time", SimpleNamespace(monotonic=lambda: stepped.at_ms / 1000.0)) + + agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits")) + collector = EventCollector() + return stepped, _ClaudeTurnState( + agent, + emit=CompositeStreamCallback([collector]), + collector=collector, + task_id="t", + user_input="go", + iteration=1, + max_turns=None, + log=agent._log, + turn_start_time=0.0, + deadline=None, + ) + + @staticmethod + def _assistant(mid: str): + from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage as SdkAssistantMessage + + return SdkAssistantMessage([], usage={"input_tokens": 10, "output_tokens": 5}, message_id=mid) + + def test_cli_boot_before_the_first_message_start_is_not_msg0_generation(self, monkeypatch): + """The interval the CLI spent booting is head, not model time. + + Before the re-seed the window opened when the turn state was built, so + this whole interval was published as msg0's `generation_duration_ms` — + ~3.6 s per turn on the measured corpus. + """ + clock, state = self._state(monkeypatch) + clock.at_ms = 800 # CLI spawn + provider resolution + TTFT + state.on_stream_event(_message_start("m1")) + clock.at_ms = 1000 + state.on_assistant_message(self._assistant("m1")) + + message = state.sdk_messages[0] + assert message.started_at == self.BASE + timedelta(milliseconds=800) + assert message.generation_duration_ms == pytest.approx(200.0) + + def test_only_the_first_message_start_reseeds_so_the_windows_still_tile(self, monkeypatch): + """A second re-seed would drop the gap before the next emission. + + That gap — a tool result landing, then the next request going out — is + real model time, and falling into no bucket at all is the defect pi + shipped with. + """ + clock, state = self._state(monkeypatch) + clock.at_ms = 800 + state.on_stream_event(_message_start("m1")) + clock.at_ms = 1000 + state.on_assistant_message(self._assistant("m1")) + clock.at_ms = 1500 + state.on_stream_event(_message_start("m2")) + clock.at_ms = 2000 + state.on_assistant_message(self._assistant("m2")) + + first, second = state.sdk_messages[0], state.sdk_messages[1] + assert second.started_at == first.completed_at, "the second window must tile from the first" + assert second.generation_duration_ms == pytest.approx(1000.0) + + def test_seeding_twice_by_hand_is_a_no_op_the_second_time(self, monkeypatch): + """The once-per-turn guard, stated outright rather than inferred. + + The sibling test above would also fail if the guard were removed, but + only via the tiling it implies. This says the property directly, so a + reviewer does not have to reproduce a mutation to see it. + """ + clock, state = self._state(monkeypatch) + clock.at_ms = 800 + state._seed_first_generation_window() + seeded_wall = state.last_event_wall + seeded_monotonic = state.last_event_monotonic + + clock.at_ms = 5000 + state._seed_first_generation_window() + + assert state.last_event_wall == seeded_wall + assert state.last_event_monotonic == seeded_monotonic + + def test_a_stream_with_no_message_start_still_clamps_to_zero(self, monkeypatch): + """Partial streaming off, a mocked query(), or a crash before the first event. + + The re-seed never fires, the turn-entry mark stands, and the head + clamps exactly as it did before. That is the correct degradation, and + asserting it is what keeps it from becoming an untested branch. + """ + clock, state = self._state(monkeypatch) + clock.at_ms = 1000 + state.on_assistant_message(self._assistant("m1")) + + assert state.first_output_seen is False, "nothing latched, so the turn-entry mark stands" + # The window still opens at turn entry, which PRECEDES the + # AgentStartEvent — so the head is a negative that decompose_turn + # clamps, exactly as it did before this phase. Asserted on the mark + # rather than by re-deriving `max(elapsed, 0.0)` from hand-built + # arguments, which would restate the implementation and could not fail. + assert state.sdk_messages[0].started_at == self.BASE + + def test_the_four_buckets_account_for_a_tool_free_turn(self, monkeypatch): + """head + generation + tail == the turn, with the head read DIRECTLY. + + The sibling tests assert the window's `started_at`, which pins the mark + but never the published `harness_startup_ms` itself — so nothing here + read the field this phase exists to change. With no tool calls the tool + bucket is empty and the other three must tile the turn exactly. + """ + from coder_eval.models import TokenUsage + from coder_eval.streaming.collector import EventCollector + from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, AgentStartEvent + + clock, state = self._state(monkeypatch) + clock.at_ms = 800 + state.on_stream_event(_message_start("m1")) + clock.at_ms = 1000 + state.on_assistant_message(self._assistant("m1")) + + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=self.BASE)) + collector.on_event( + AgentEndEvent( + task_id="t", + status=AgentEndStatus.COMPLETED, + messages=list(state.sdk_messages), + usage=TokenUsage(), + timestamp=self.BASE + timedelta(milliseconds=1500), + ) + ) + record = collector.build_turn_record() + + assert record.harness_startup_ms == pytest.approx(800.0), "the CLI boot is the head, published" + assert record.harness_teardown_ms == pytest.approx(500.0) + generation = sum(m.generation_duration_ms or 0.0 for m in record.messages if m.role == "assistant") + assert generation == pytest.approx(200.0) + assert record.harness_startup_ms + generation + record.harness_teardown_ms == pytest.approx(1500.0) diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index bbbf47bad..d9528c252 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -1867,10 +1867,17 @@ async def test_a_no_op_flush_does_not_move_the_mark(monkeypatch): async def test_generation_and_tool_time_account_for_the_turn(): - """Σ generation + Σ tool execution lands inside the turn's own duration. + """Σ generation + Σ tool + head + tail lands inside the turn's own duration. Bounds, not equality: the fake conversation's own overhead sits in the - residual. Before this change the generation half was identically 0. + residual. Before the window existed the generation half was identically 0. + + The HEAD is part of the sum, and has to be: the first window now opens at + the first observed `Step` rather than at turn entry, so the dispatch before + it is a measured bucket instead of time hidden inside msg0's generation. + Asserting `generation + tool` alone against a share of the turn was an + assertion that the head stays empty — which is what this phase deliberately + stopped being true. """ steps = [ _step("THINKING", "DONE", thinking="plan", usage=_usage(100, 0, 5, 5)), @@ -1894,11 +1901,14 @@ async def test_generation_and_tool_time_account_for_the_turn(): gen_ms = sum(m.generation_duration_ms or 0.0 for m in _assistant(record)) tool_ms = sum(c.duration_ms or 0.0 for c in record.commands) + head_ms = record.harness_startup_ms or 0.0 + tail_ms = record.harness_teardown_ms or 0.0 turn_ms = record.duration_seconds * 1000.0 assert gen_ms > 0 - assert gen_ms + tool_ms <= turn_ms - assert gen_ms + tool_ms >= 0.5 * turn_ms + assert head_ms > 0, "the dispatch before the first Step is now a measured bucket, not 0.0" + assert gen_ms + tool_ms + head_ms + tail_ms <= turn_ms + assert gen_ms + tool_ms + head_ms + tail_ms >= 0.5 * turn_ms async def test_timing_change_moves_no_token_bucket(): @@ -2038,3 +2048,130 @@ async def test_each_turn_gets_a_fresh_clock(): # Re-anchored: the later turn's window opens after the earlier one closed. assert second[0].started_at >= first[0].completed_at assert second[0].completed_at > second[0].started_at + + +class TestAntigravityFirstWindowReseed: + """The first `Step` moves `_gen_mark_wall`; a later one must not. + + Driven at `_AntigravityTurnState` with an injected clock, NOT through + `communicate()`: the fake conversation yields with no delay, so an + end-to-end run cannot pin the MAGNITUDE — the two stamps land within + microseconds of each other, so no assertion there could say the mark moved + by the right amount. + + It can detect the mark moving at all, and does: + `test_generation_and_tool_time_account_for_the_turn` asserts `head_ms > 0` + and fails if the re-seed call is removed. These tests are the ones that say + WHERE it moved to and that it moves only once. + """ + + BASE = datetime(2026, 9, 11, 9, 0, 0) + + class _Clock: + def __init__(self, at_ms: float = 0.0) -> None: + self.at_ms = at_ms + + def now(self) -> datetime: + return TestAntigravityFirstWindowReseed.BASE + timedelta(milliseconds=self.at_ms) + + def _state(self, clock): + from coder_eval.agents.antigravity_agent import _AntigravityTurnState + from coder_eval.streaming.callbacks import CompositeStreamCallback + from coder_eval.streaming.collector import EventCollector + + agent = AntigravityAgent(parse_agent_config(type="antigravity", model="gemini-3.5-flash")) + collector = EventCollector() + return _AntigravityTurnState( + agent=agent, + emit=CompositeStreamCallback([collector]), + task_id="t", + turn_id="turn", + collector=collector, + user_input="go", + iteration=1, + model="gemini-3.5-flash", + turn_start_time=0.0, + clock=clock, + ) + + def test_the_first_step_moves_the_mark_off_the_turn_entry_stamp(self): + """Dispatch before the first Step is head, not the first generation. + + Before the re-seed the mark was stamped when the turn state was built, + so this interval was published as generation — ~4.7 s per turn against + a later-window median of 3.3 s. + """ + clock = self._Clock() + state = self._state(clock) + assert state._gen_mark_wall == self.BASE + + clock.at_ms = 900 # dispatch + TTFT + state.process_step(_step("THINKING", "ACTIVE", thinking="...")) + + assert state._gen_mark_wall == self.BASE + timedelta(milliseconds=900) + + def test_a_later_step_does_not_move_it(self): + """Re-seeding more than once per turn is the defect, not the feature.""" + clock = self._Clock() + state = self._state(clock) + clock.at_ms = 900 + state.process_step(_step("THINKING", "ACTIVE", thinking="...")) + seeded = state._gen_mark_wall + + clock.at_ms = 5000 + state.process_step(_step("THINKING", "ACTIVE", thinking="more")) + + assert state._gen_mark_wall == seeded + + def test_seeding_twice_by_hand_is_a_no_op_the_second_time(self): + """The once-per-turn guard, stated outright rather than inferred.""" + clock = self._Clock() + state = self._state(clock) + clock.at_ms = 900 + state._seed_first_generation_window("MODEL") + seeded = state._gen_mark_wall + + clock.at_ms = 5000 + state._seed_first_generation_window("MODEL") + + assert state._gen_mark_wall == seeded + + def test_a_flush_still_advances_the_mark_and_opens_at_the_reseeded_one(self): + """The re-seed must not break the tiling it sits in front of.""" + clock = self._Clock() + state = self._state(clock) + clock.at_ms = 900 + state.process_step(_step("THINKING", "ACTIVE", thinking="plan")) + clock.at_ms = 2000 + state.process_step(_step("THINKING", "DONE", thinking="plan", usage=_usage(100, 0, 5, 5))) + + message = _assistant(state)[0] + assert message.started_at == self.BASE + timedelta(milliseconds=900), "opens at the RE-SEEDED mark" + assert message.generation_duration_ms == pytest.approx(1100.0) + assert state._gen_mark_wall == self.BASE + timedelta(milliseconds=2000), "and the flush advances it" + + def test_a_non_model_step_does_not_seed_the_window(self): + """The field is MODEL output, and the SDK streams Steps that are not. + + `StepSource` carries SYSTEM and USER besides MODEL, and the SDK's event + processor queues every `step_update` verbatim, so a turn can open with + one. Seeding on it would put the mark before the model spoke and hand + the remainder back to msg0's generation — the defect being fixed. + """ + clock = self._Clock() + state = self._state(clock) + + clock.at_ms = 400 + state.process_step(_step("SYSTEM_MESSAGE", "DONE", source="SYSTEM", content="compacting")) + assert state._first_output_seen is False + assert state._gen_mark_wall == self.BASE, "a system Step must not open the generation window" + + clock.at_ms = 900 + state.process_step(_step("THINKING", "ACTIVE", thinking="...")) + assert state._gen_mark_wall == self.BASE + timedelta(milliseconds=900), "the first MODEL Step does" + + def test_a_turn_that_streams_no_step_keeps_the_turn_entry_mark(self): + clock = self._Clock() + state = self._state(clock) + assert state._first_output_seen is False + assert state._gen_mark_wall == self.BASE diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index f60dd4647..6b655eae0 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -478,10 +478,12 @@ class TestHarnessOverheadBuckets: Measured live across all five harnesses, these two plus generation plus tool execution account for the turn to within 0.1 ms — so what the evalboard shows as "Unaccounted" is fully explained rather than merely displayed. The head is - where the harnesses differ most (OpenCode ~3.0 s of CLI boot + TTFT fused, - claude-code a measured 0.0 because its first window already covers dispatch), - which is exactly why it is booked as its own bucket instead of being folded - into generation. + where the harnesses differ most — every one of them now measures it up to + its first observed model output, but what that interval CONTAINS ranges from + ~0.23 s on Pi to ~4.7 s on Antigravity, depending on whether the harness + spawns its process per turn and how long the provider takes to first token. + That spread is exactly why it is booked as its own bucket instead of being + folded into generation. """ @staticmethod diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index 5b9d5cb9b..a3354a66c 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -424,6 +424,12 @@ def _claude_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: one counter. Patching either alone leaves the other reading the real clock, and the case would then assert a measured span against an unmeasured one. + The first `message_start` re-seeds the window, so the CLI spawn and the + query build before it are head rather than msg0's generation. That a LATER + one must not re-seed is asserted directly in + `tests/test_agent_telemetry.py`; here it shows up as the windows still + tiling. + Note where its windows do NOT tile: the tool result resets both marks, so the interval between the emission that ISSUED the call and the result is left outside every window. That gap is the tool's own execution, which is @@ -434,7 +440,7 @@ def _claude_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: from coder_eval.agents.claude_code_agent import ClaudeCodeAgent, _ClaudeTurnState from coder_eval.streaming.events import AgentEndStatus as _AgentEndStatus from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage as SdkAssistantMessage - from tests._fixtures.golden_streams.claude_fixtures import ToolUseBlock, UserMessage + from tests._fixtures.golden_streams.claude_fixtures import ToolUseBlock, UserMessage, message_start class _Stepped(datetime): at_ms = 0.0 @@ -453,7 +459,7 @@ def _monotonic() -> float: collector = EventCollector() commands: list[CommandTelemetry] = [] - _Stepped.at_ms = 500 # CLI spawn + dispatch, before the state exists: head + _Stepped.at_ms = 500 # the turn state is built here; the head runs past it state = _ClaudeTurnState( agent, emit=CompositeStreamCallback( @@ -472,6 +478,12 @@ def _monotonic() -> float: deadline=None, ) + # The stream really does put `message_start` before the emission it + # announces — the recorded corpus shows it and the SDK guarantees it — and + # the FIRST one is what re-seeds the window, so an ordering this case got + # wrong would silently stop exercising the re-seed at all. + _Stepped.at_ms = 800 + state.on_stream_event(message_start("m1")) _Stepped.at_ms = 1000 state.on_assistant_message( SdkAssistantMessage( @@ -482,6 +494,8 @@ def _monotonic() -> float: ) _Stepped.at_ms = 1800 # the tool ran for the whole gap state.on_user_message(UserMessage("c1", False, "ok")) + _Stepped.at_ms = 2000 + state.on_stream_event(message_start("m2")) # does NOT re-seed: once per turn _Stepped.at_ms = 2500 state.on_assistant_message(SdkAssistantMessage([], usage={"input_tokens": 10, "output_tokens": 5}, message_id="m2")) state.finalize(_AgentEndStatus.COMPLETED) From 520c1ad1518f06779e3daeba1a9ab9c3da72d06b Mon Sep 17 00:00:00 2001 From: uipreliga Date: Fri, 11 Sep 2026 23:14:28 -0700 Subject: [PATCH 32/54] =?UTF-8?q?refactor(timing):=205/7=20=E2=80=94=20one?= =?UTF-8?q?=20tool-subtraction,=20at=20the=20collector=20seam?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tool execution came out of a generation window in five places: four inside `close_window` as the reducer flushed, claude-code once at finalization. The head and the tail were already computed ONCE, centrally, at the collector — and that asymmetry was the complexity. Every timing defect on this branch lived in the per-reducer bookkeeping around the subtraction rather than in the subtraction itself: when to reset a span list (clearing it at `step_start` wiped a span before the flush could subtract it, a 100% overstatement of that window), when to clear a spent start stamp (a second flush with no intervening start republished the previous span — 3000 ms of generation for a 2000 ms turn), when to advance the mark. `EventCollector.subtract_tool_time` now does it once, for all five. A reducer publishes the RAW window and keeps only the genuinely harness-shaped decision, which is where that window opens. Three span lists, their reset rules, the bounding of still-open calls and `close_window`'s two span parameters are gone. CE063 stops a sixth harness rebuilding them; CE061 is exemption-free, since claude-code now calls the same shrunken helper as the other four. Grouping is on the BOUNDS, not `message_id`. Codex splits one window into thinking and action sub-messages that share a pair of bounds; subtracting from each separately takes the overlap twice and the parts stop summing. OpenCode and Pi can legitimately carry `message_id is None`, so keying on the id would collapse a turn's id-less messages into one group instead. Non-mutating, and the reason is aliasing rather than repeated calls: every agent builds its terminal event as `AgentEndEvent(messages=list(...))`, which copies the LIST and not the messages, so an in-place write would reach back into the agent's own live state from the collector. Two behaviour changes, each with its own named test rather than hidden in a number: * A call still open when a window closes is no longer subtracted at that boundary. The collector sees every span at once, so it comes out of the windows the call's REAL interval overlaps, once it resolves. A call that never resolves was never timed and contributes nothing. * claude-code's window is measured on ONE clock. Its duration was a monotonic delta while its bounds were wall stamps — the split `TurnClock` exists to remove — and central subtraction makes that untenable, because it clips WALL spans against those WALL bounds. `turn_start_time` stays monotonic: the deadline must not move when the wall clock steps. Also fixes the P3 thread mix, and the divergence fixing it created. `_overhead_ms` filtered its generations to the main thread and passed EVERY command, so its claim to keep all four buckets on one thread held only because a child nests inside the parent Agent call. Filtering there alone then made the LIVE residual gate compute a different tool total than the harness — the worst place for a drift, since it is the only two-sided sensor. All three implementations (`_main_thread_tool_spans`, `_scrub.py`, `decompose_run.py`) now filter, and `TestTheThreeToolUnionsAgree` pins them together. `tests/_fixtures/timing_runs/` commits one scrubbed run per harness. Its README states plainly what the plan asked it to be and what it cannot be: the script reads STORED fields, so over a fixed corpus it prints the identical table before and after any code change. Its own claude-code row still reconciles at -481 ms and books a 0.0 head — both long fixed — which is the argument. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- CLAUDE.md | 2 +- docs/agents/HARNESS_PARITY.md | 156 ++++++--- evalboard/lib/timing.ts | 7 +- pyproject.toml | 1 + scripts/timing/decompose_run.py | 41 ++- src/coder_eval/agents/antigravity_agent.py | 43 +-- src/coder_eval/agents/claude_code_agent.py | 81 ++--- src/coder_eval/agents/codex_agent.py | 26 +- src/coder_eval/agents/opencode_agent.py | 41 +-- src/coder_eval/agents/pi_agent.py | 35 +- src/coder_eval/models/telemetry.py | 15 +- src/coder_eval/streaming/collector.py | 139 +++++++- src/coder_eval/timing.py | 75 ++--- tests/_fixtures/golden_streams/_scrub.py | 30 +- tests/_fixtures/timing_runs/README.md | 53 +++ tests/_fixtures/timing_runs/antigravity.json | 134 ++++++++ tests/_fixtures/timing_runs/claude-code.json | 225 +++++++++++++ tests/_fixtures/timing_runs/codex.json | 109 +++++++ tests/_fixtures/timing_runs/opencode.json | 171 ++++++++++ tests/_fixtures/timing_runs/pi.json | 141 ++++++++ .../lint/rules/ce063_no_busy_ms_in_agents.py | 105 ++++++ tests/lint/runner.py | 2 + tests/test_agent_golden_master.py | 9 + tests/test_agent_telemetry.py | 2 - tests/test_codex_agent.py | 126 +++++++- tests/test_custom_lint.py | 85 ++++- tests/test_event_collector.py | 301 +++++++++++++++++- tests/test_opencode_agent.py | 184 ++++++----- tests/test_pi_agent.py | 176 ++++++---- tests/test_timing_close_window.py | 214 ++++++++----- tests/test_timing_identity_contract.py | 15 +- 31 files changed, 2205 insertions(+), 539 deletions(-) create mode 100644 tests/_fixtures/timing_runs/README.md create mode 100644 tests/_fixtures/timing_runs/antigravity.json create mode 100644 tests/_fixtures/timing_runs/claude-code.json create mode 100644 tests/_fixtures/timing_runs/codex.json create mode 100644 tests/_fixtures/timing_runs/opencode.json create mode 100644 tests/_fixtures/timing_runs/pi.json create mode 100644 tests/lint/rules/ce063_no_busy_ms_in_agents.py diff --git a/CLAUDE.md b/CLAUDE.md index 6bf3bc529..cabde3f6b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,7 +236,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. The damage was not confined to the timeline, which is why "only granularity is lost" was the wrong way to describe it: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose prompt-cache cascade is quadratic in that count, so a single-shot Antigravity run had every cascade coefficient pinned at zero; the `Messages` count and the 10 s slow-generation bar were per-turn too. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). +Recent additions, each traceable to a shipped defect: **CE063** (no module in `src/coder_eval/agents/` may import `busy_ms` — tool execution comes out of a generation window in exactly ONE place, `streaming/collector.py::subtract_tool_time`. Five reducers used to do it themselves while the head and tail were already computed centrally at the same seam, and that asymmetry is where every timing defect on this branch lived — none of them in the arithmetic, all of them in the bookkeeping AROUND it: when to reset a per-step span list (clearing it at `step_start` wiped a span before the flush could subtract it, a 100% overstatement of that window), when to clear a spent start stamp (a second flush with no intervening start republished the previous span — 3000 ms of generation for a 2000 ms turn), when to advance the mark. A sixth harness reaching for `busy_ms` rebuilds that, and its tool time is then subtracted TWICE — by the reducer and again by the collector — under-reporting generation on one harness only, which takes a corpus comparison to notice. A separate id from CE061 rather than a rebody: CE061 asks where a window's ARITHMETIC came from and four reducers still call `close_window`, so its property is live and unsuperseded; this asks whether a reducer subtracts at all. It deliberately does NOT reuse CE061's `_imports_the_helper`, whose bare-module-import branch exists so `timing.close_window(...)` counts as reaching the helper — inverted into a ban that branch flags four of the five reducers. CE061 is now **exemption-free**: claude-code was its one permanent `# noqa` and, with the subtraction moved, calls the shrunken `close_window` like the other four), **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. The damage was not confined to the timeline, which is why "only granularity is lost" was the wrong way to describe it: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose prompt-cache cascade is quadratic in that count, so a single-shot Antigravity run had every cascade coefficient pinned at zero; the `Messages` count and the 10 s slow-generation bar were per-turn too. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index d397ccfaa..5c539447c 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -24,7 +24,8 @@ wall clock its numbers account for. | Field | claude-code | codex | antigravity | opencode | pi | |---|---|---|---|---|---| -| `generation_duration_ms` source | harness clock: previous SDK event → this message | SDK item stamps, minus tool execution inside the window | harness clock: previous flush → this flush, minus tool execution inside the window | harness clock: previous `step_finish` → this one, minus tool execution inside the window | harness clock: previous `turn_end` → this one, minus tool execution inside the window | +| `generation_duration_ms` RAW window (the reducer's part) | harness clock: previous SDK event → this message | SDK item stamps | harness clock: previous flush → this flush | harness clock: previous `step_finish` → this one | harness clock: previous `turn_end` → this one | +| tool time subtracted from it | centrally | centrally | centrally | centrally | centrally | | what the **first** window covers | the first `message_start`, so CLI boot + TTFT are OUTSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | the first `Step`, so dispatch + TTFT are OUTSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | | `harness_startup_ms` (turn head) | ~3.6 s — CLI boot fused with TTFT | ~3.1 s — CLI boot fused with TTFT | ~4.7 s — dispatch fused with TTFT (its harness process is spawned once at startup, not per turn) | ~2.5 s — CLI boot fused with TTFT | ~0.23 s — CLI boot fused with TTFT | | `harness_teardown_ms` (turn tail) | ~1.3 s | ~13 ms | ~7 ms | ~26 ms | ~19 ms | @@ -33,8 +34,8 @@ wall clock its numbers account for. | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | | `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID`; `None` when absent | CLI `responseId`; `None` when absent | | `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | -| clock basis for recorded stamps | monotonic duration, wall bounds | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | -| window built by `timing.py::close_window` | no — see below | yes | yes | yes | yes | +| clock basis for recorded stamps | wall bounds, wall duration (raw `datetime.now()`) | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | +| window built by `timing.py::close_window` | yes | yes | yes | yes | yes | [^identity]: "yes" is load-bearing but the committed sensor is one-sided. `tests/_fixtures/golden_streams/_scrub.py` asserts only `overshoot <= …`, so it @@ -48,40 +49,70 @@ turn's `|residual|` as a share of its own wall clock. It is report-only and nothing runs it on a schedule; run it by hand against real `task.json` files. **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** -All five harnesses can have tool execution inside a generation window, and all -five subtract it. Four interleave it structurally: Antigravity reports a `Step` -for the tool and only a later +All five harnesses can have tool execution inside a generation window, and it is +subtracted out of every one of them — **once, centrally**, by +`streaming/collector.py::subtract_tool_time`. No reducer does it itself; each +publishes the raw window (see the two sections below). Every harness has the +problem: Antigravity reports a `Step` for the tool and only a later `usage_metadata` `Step` cuts the message; Codex's message window is seeded from -the first item's start and extended to the last item's completion; OpenCode and Pi -tile: each window opens where the previous `step_finish` / `turn_end` closed it -and runs to the next, with every tool call in between running inside. In each the +the first item's start and extended to the last item's completion; OpenCode, Pi +and claude-code tile, each window opening where the previous one closed and +running to the next, with every tool call in between running inside. In each the span between the recorded bounds legitimately CONTAINS tool time that the model -did not spend generating, so each subtracts it — the **union** of the closed tool intervals -clipped to the window (`coder_eval/timing.py::busy_ms`), never the sum, because -tool calls overlap: Antigravity resolves several from one `Step` and backgrounds -anything over ten seconds, and Codex spawns collab agents concurrently. Summing -them over-subtracts by exactly the overlap and, with enough concurrency, drives -the result to a clamped zero. +did not spend generating. What comes out is the **union** of the resolved +main-thread tool intervals clipped to the window +(`coder_eval/timing.py::busy_ms`), never the sum, because tool calls overlap: +Antigravity resolves several from one `Step` and backgrounds anything over ten +seconds, and Codex spawns collab agents concurrently. Summing them +over-subtracts by exactly the overlap and, with enough concurrency, drives the +result to a clamped zero. The consequence worth knowing: on an emission that carries *only* a tool call, the whole measured window was that tool running, so the recorded generation time is legitimately `0.0`. That is a measurement, not a placeholder — `None` is what "never measured" looks like. -**One helper builds four of the five windows.** Codex, OpenCode, Pi and -Antigravity call `coder_eval/timing.py::close_window`, which is the whole -arithmetic in one place: tile from the mark, keep a stamp that went backwards -from inverting the span, bound the calls still open at the boundary, subtract -the union clipped to the window, clamp at zero. It had been copy-pasted four -times, and Pi shipped a variant of it that measured from its own turn start — -so every inter-turn gap fell into no bucket, and nothing failed, because the -identity above is asserted on one side only. **CE061** now requires any module -in `agents/` that publishes a measured `generation_duration_ms` to import the -helper. claude-code is the single documented exception and carries the only -`# noqa: CE061`: it subtracts once at finalization (below) rather than per -flush, a shape `close_window` cannot take without a mode flag. - -**Two clock bases remain, and the row above says which.** Antigravity and Pi +**One helper opens all five windows, and the subtraction is not in it.** Every +reducer calls `coder_eval/timing.py::close_window`, which is now only the +window's own geometry: tile from the mark, keep a stamp that went backwards +from inverting the span, clamp at zero. It had been copy-pasted four times, and +Pi shipped a variant that measured from its own turn start — so every +inter-turn gap fell into no bucket, and nothing failed, because the identity +above is asserted on one side only. **CE061** requires any module in `agents/` +publishing a measured `generation_duration_ms` to import the helper, and is now +**exemption-free**: claude-code was its one permanent `# noqa` and no longer +needs it. + +**Tool execution comes out of the windows ONCE, at the collector.** +`streaming/collector.py::subtract_tool_time` takes the union of the main-thread +tool intervals, clipped to each window, out of the raw spans the reducers +publish. Before, that happened five times in five places — four inside +`close_window` as the reducer flushed, claude-code once at finalization — while +the head and the tail were already computed centrally at the same seam. That +asymmetry was the complexity, and every timing defect on this branch lived in +the per-reducer bookkeeping around the subtraction rather than in the +subtraction: when to reset a span list (clearing it at `step_start` wiped a span +before the flush could subtract it — a 100% overstatement of that window), when +to clear a spent start stamp (a second flush with no intervening start +republished the previous span — 3000 ms of generation for a 2000 ms turn), when +to advance the mark. Those three lists, their reset rules, and the bounding of +still-open calls are all deleted. **CE063** stops a sixth harness rebuilding +them: no module in `agents/` may import `busy_ms`. + +Two consequences worth stating, because both are behaviour changes: + +- **A call still open when a window closes is no longer subtracted at that + boundary.** The reducer used to bound it at the window's end and take that + slice. The collector sees every span at once, so the call is subtracted from + the windows its REAL interval overlaps, once it resolves — no approximation. + A call that never resolves has no `execution_completed_at`, contributes + nothing, and says so. +- **Codex's two sub-messages are one group.** They share a pair of bounds and + divide the window by output-token share; the collector groups on the bounds + (not on `message_id`, which OpenCode and Pi can legitimately leave `None`), + subtracts the overlap once, and re-apportions so the parts still sum. + +**Three clock bases remain, and the row above says which.** Antigravity and Pi derive every recorded wall stamp from one `TurnClock` per turn, so a turn's bounds and the tool spans subtracted from them cannot disagree. Antigravity needed it: its span was monotonic while its tool intervals were wall, which is @@ -95,28 +126,51 @@ removed. Their tool spans are the CLI's own epoch-millisecond stamps (`codex_agent.py::_ms_to_dt`, `opencode_agent.py::_epoch_ms_to_dt`), which cannot be re-derived host-side; converting only the window bounds would put two bases inside one `busy_ms` subtraction, relocating the defect instead of -removing it. Both therefore keep the naive-local exposure. Deadlines on every -harness stay on raw `time.monotonic()` and must — a deadline may not move when -the wall clock steps. - -**`claude-code` subtracts at finalization, not as it flushes.** It was once -exempt entirely, on the premise that because it marks the end of the previous -SDK event and reads again when the next message arrives, a tool's execution -falls *between* two windows rather than inside one. Measured, that premise does -not hold: a tool's timer starts at the **emission** carrying its `tool_use` -block, and one assistant turn spans several emissions, so a later emission's -window runs concurrently with a tool already timing. On a task issuing five -parallel writes, five reads and two concurrent `Bash` calls the overlap was -482 ms and 340 ms on two ~18-25 s turns, and the four-bucket residual came out -at exactly `-481 ms` and `-339 ms`; the other four overlapped by ~2.0-2.3 s on -the same task and still reconciled to within 1.2 ms, because they subtract it. - -It cannot subtract while flushing, because a tool issued by an earlier emission -is still running when the next window closes and its interval does not exist -yet. `_ClaudeTurnState._subtract_tool_time_from_windows` therefore runs once at -finalization, when every span is known. After it, the same task reconciles to -**1.4 ms (0.006% of wall)** over four turns that all carried overlapping tool -calls. +removing it. Both therefore keep the naive-local exposure. + +claude-code is the third case and the newest. Its window duration used to be a +monotonic delta while its bounds were wall stamps — the split `TurnClock` +exists to remove — and central subtraction made that untenable, because it +clips WALL tool spans against those WALL bounds. It now measures the span from +the bounds, so the two agree; but the bounds are still raw `datetime.now()`, +so it keeps the same naive-local exposure as codex and opencode, for a +different reason: no epoch-stamp constraint, it simply has not been converted. +That conversion is the remaining improvement here and is not done. + +Deadlines on every harness stay on raw `time.monotonic()` and must — a deadline +may not move when the wall clock steps. + +**HISTORY — why claude-code needed a special case at all.** It was once exempt +from subtracting entirely, on the premise that because it marks the end of the +previous SDK event and reads again when the next message arrives, a tool's +execution falls *between* two windows rather than inside one. Measured, that +premise does not hold: a tool's timer starts at the **emission** carrying its +`tool_use` block, and one assistant turn spans several emissions, so a later +emission's window runs concurrently with a tool already timing. On a task +issuing five parallel writes, five reads and two concurrent `Bash` calls the +overlap was 482 ms and 340 ms on two ~18-25 s turns, and the four-bucket +residual came out at exactly `-481 ms` and `-339 ms`. (That run is pinned at +`tests/_fixtures/timing_runs/claude-code.json`, which still reconciles at +-481 ms — it is a RECORD of the defect, not of current behaviour; see the README +there.) + +It could not subtract while flushing, because a tool issued by an earlier +emission is still running when the next window closes and its interval does not +exist yet — so it subtracted once at finalization instead, in a method of its +own. Central subtraction dissolves the special case: the collector is *already* +the place where every span is known, so claude-code needs no separate pass and +no exemption. + +Its window is also now measured on ONE clock. The duration used to be a +monotonic delta while the bounds were wall stamps, which is exactly the split +`TurnClock` exists to eliminate — and it became load-bearing with central +subtraction, which clips WALL tool spans against those WALL bounds. A +monotonic-measured duration would have had the two disagreeing inside one +subtraction, which is the defect that let Antigravity's window go negative. +`turn_start_time` stays monotonic and is untouched: `duration_seconds` and the +turn deadline read it, and a deadline must not move when the wall clock steps. +Adopting a full `TurnClock` here (deriving the wall stamps from monotonic, as +antigravity and pi do) is the remaining improvement and is not done. **The head and tail are measured, not normalized.** Generation and tool are only two of the four buckets. The turn's **head** (turn start → first diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts index 5833ae065..37174600b 100644 --- a/evalboard/lib/timing.ts +++ b/evalboard/lib/timing.ts @@ -137,9 +137,10 @@ export function epochMs(value: string | null | undefined): number | null { // not the sum. // // The TypeScript twin of `coder_eval.timing.busy_ms`, deliberately the -// same algorithm — the agents subtract tool time from a generation window with -// it, and this file subtracts tool time from a task's wall clock, so the two -// must agree about what "tool execution took N ms" means. Held in step by +// same algorithm — the harness subtracts tool time from its generation windows +// with it (once, in streaming/collector.py::subtract_tool_time), and this file +// subtracts tool time from a task's wall clock, so the two must agree about +// what "tool execution took N ms" means. Held in step by // tests/_fixtures/timing_union_cases.json, which both suites replay. export function busyMs( spans: [number, number][], diff --git a/pyproject.toml b/pyproject.toml index 4159a0365..57aef68b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -297,6 +297,7 @@ external = [ "CE059", "CE060", "CE061", + "CE063", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index 75202e6a8..6c3bd68c7 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -46,20 +46,49 @@ def _parse(stamp: object) -> datetime | None: return None +def _sub_agent_tool_ids(turn: dict) -> set: + """Tool ids owned by a SUB-AGENT generation, which the main thread excludes. + + Derived the only way it can be: a child generation carries + `parent_tool_use_id`, and its `tool_use_ids` are the calls it made. + + This MUST match `EventCollector._main_thread_tool_spans`, which applies the + same filter when it computes the head, the tail and the generation + subtraction. The two used to disagree — the collector passed every command + while filtering its generations — and they agreed only by luck, because a + child nests inside the parent Agent call whose interval the union already + covers. Codex's recovered child tools carry the CHILD's clock, so the + nesting is not guaranteed, and a gate computing a different tool total than + the harness reports a residual that is an artifact of the disagreement + rather than a bucket error. This is the only two-sided live sensor for the + identity, so that is the worst place for the two to drift. + """ + ids = set() + for message in turn.get("messages") or []: + if message.get("role") == "assistant" and message.get("parent_tool_use_id") is not None: + ids.update(message.get("tool_use_ids") or []) + return ids + + def _tool_ms(turn: dict) -> float: - """Wall ms this turn spent executing tools — the UNION, not the sum. + """Wall ms this turn's MAIN-THREAD tools occupied — the UNION, not the sum. - The same rule `coder_eval.timing.union_ms` applies when a harness subtracts - tool time out of a generation window, and it has to be the same rule here - or the identity does not close: Pi resolved a `Write` and a `Bash` that - overlapped by 18.4 ms in one measured turn, and summing their durations + The same rule `coder_eval.timing.union_ms` applies when the collector + subtracts tool time out of a generation window, and it has to be the same + rule here or the identity does not close: Pi resolved a `Write` and a `Bash` + that overlapped by 18.4 ms in one measured turn, and summing their durations booked that overlap twice, which is precisely the 18.3 ms residual that found this. A command with no recorded bounds cannot be placed on the timeline at all, so it contributes nothing rather than being summed in - blind — see docs/agents/HARNESS_PARITY.md's Delegate divergence. + blind — see docs/agents/HARNESS_PARITY.md's Delegate divergence. Sub-agent + tools are excluded for the same reason their generations are; see + `_sub_agent_tool_ids`. """ + excluded = _sub_agent_tool_ids(turn) spans = [] for command in turn.get("commands") or []: + if command.get("tool_id") in excluded: + continue start = _parse(command.get("execution_started_at")) end = _parse(command.get("execution_completed_at")) if start is not None and end is not None and end >= start: diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index b725f20b4..5d016f4a4 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -869,13 +869,6 @@ def __init__( # Re-seeded ONCE, at the first observed Step. See # `_seed_first_generation_window`. self._first_output_seen: bool = False - # Execution intervals of tools that CLOSED since the mark. This harness - # interleaves tool calls into one generation — the Step for the tool - # arrives and only a later usage_metadata Step cuts the message — so a - # window legitimately contains tool time that is not model time. Kept - # as intervals, not a running total, because they overlap (see - # busy_ms). - self._tool_spans_since_mark: list[tuple[datetime, datetime]] = [] @property def ended_cleanly(self) -> bool: @@ -1046,12 +1039,6 @@ def _handle_tool_call(self, call: Any, step: Any, done: bool, sstatus: Any, call "duration_ms": tool_ms, } ) - # This tool closed inside the open generation window, so its time is - # not model time. The INTERVAL is recorded, not the duration: tool - # calls overlap here, and only their union may be subtracted (see - # busy_ms). Only the DONE path records one — a tool force-closed at - # finalize has duration_ms None and was never timed. - self._tool_spans_since_mark.append((started, completed)) self.commands.append(end_tel) self.emit.on_event( ToolEndEvent( @@ -1098,15 +1085,17 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: # skill-rpa-uia-google-search, a harness-local Read closed 8 ms after # it opened while 6.4 s of model time separated the two flushes around # it — a reset would have reported 8 ms and dropped the 6.4 s. - # Subtracting closed tool time handles that case AND its opposite (a - # 43 s Bash, where the model time really is the flush-to-DONE - # remainder). + # Publishing the RAW window and letting the collector subtract the tool + # union handles that case AND its opposite (a 43 s Bash, where the + # model time really is the flush-to-DONE remainder). # - # The window arithmetic itself, and why a call still open at this - # boundary counts against it, live in `close_window`'s docstring. - # Measured here before the helper existed: a Bash opening 1.7 ms before - # the flush drove Sum(generation) + Sum(command) 0.26 ms PAST the turn - # wall, on a turn whose whole headroom was 1.4 ms. + # This harness interleaves a tool INTO a window rather than tiling + # around it, so the window legitimately contains time that is not model + # time. `EventCollector.subtract_tool_time` clips the union to these + # bounds and takes it out. Measured here before any of that existed: a + # Bash opening 1.7 ms before the flush drove Sum(generation) + + # Sum(command) 0.26 ms PAST the turn wall, on a turn whose whole + # headroom was 1.4 ms. # # The span used to be read off `time.monotonic()` while these intervals # were wall, and subtracting one from the other is the only reason this @@ -1114,16 +1103,7 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: # real instant generation. Both bounds now derive from `self.clock`, so # the disagreement is unrepresentable and the branch that hid it is # gone. - _, generation_ms = close_window( - mark=self._gen_mark_wall, - now=now_wall, - closed_spans=self._tool_spans_since_mark, - open_started_ats=[ - tel.execution_started_at - for cid, tel in self._open_tools.items() - if cid not in self._closed_tools and tel.execution_started_at is not None - ], - ) + _, generation_ms = close_window(mark=self._gen_mark_wall, now=now_wall) for i, block in enumerate(self._blocks): block.sequence = i self.messages.append( @@ -1151,7 +1131,6 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: # early return above means a no-op flush leaves the window open, so a # later real generation still measures from where it began. self._gen_mark_wall = now_wall - self._tool_spans_since_mark = [] def _agent_output(self) -> str: if self._output_parts: diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 537a8e54c..e63821a69 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -76,7 +76,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.timing import busy_ms +from coder_eval.timing import close_window from coder_eval.utils import dump_dataclass, process_plugins @@ -244,7 +244,15 @@ def __init__( self.sequence_number = 0 self.last_assistant_message_index: int | None = None - self.last_event_monotonic: float = turn_start_time + # ONE clock basis for the window. The duration used to be a MONOTONIC + # delta while these bounds were wall, which is the split + # `timing.TurnClock` exists to eliminate: the central subtraction clips + # WALL tool spans to these WALL bounds, so a monotonic-measured + # duration would have the two disagreeing inside one subtraction — + # exactly the defect that let antigravity's window go negative. + # `turn_start_time` stays monotonic and is untouched: `duration_seconds` + # and the turn deadline read it, and a deadline must not move when the + # wall clock steps. self.last_event_wall: datetime = datetime.now() # Re-seeded ONCE, at the first observed model output. See # `_seed_first_generation_window`. @@ -315,10 +323,8 @@ def dispatch(self, message: Message) -> None: def on_assistant_message(self, message: Message) -> None: """Capture ToolUseBlocks + build the AssistantMessage telemetry record.""" - message_arrival_monotonic = time.monotonic() message_arrival_wall = datetime.now() generation_started_wall = self.last_event_wall - generation_duration_ms = (message_arrival_monotonic - self.last_event_monotonic) * 1000 current_turn_index = len(self.sdk_messages) self.assistant_turn_count += 1 @@ -428,17 +434,17 @@ def on_assistant_message(self, message: Message) -> None: out_tok = int(msg_usage.get("output_tokens", 0) or 0) self.pending_delta_output_tokens = None - # CE061's one permanent exception. This harness measures its - # window as a monotonic delta and subtracts tool time ONCE at - # finalization across every emission (`_subtract_tool_time_from_windows`), - # because a call issued by an earlier emission is still running when the - # next window closes. `close_window` subtracts per flush; forcing both - # shapes into it means a mode flag on a helper whose whole value is - # having one shape. It already uses the shared `busy_ms`. - assistant_telemetry = AssistantMessageTelemetry( # noqa: CE061 - started_at=generation_started_wall, + # The RAW window. Tool execution comes out of it once, centrally, in + # `EventCollector.build_turn_record` — so this harness now asks the same + # helper as the other four and CE061 no longer needs its one permanent + # exception. The mark is the only harness-shaped decision left, and it + # stays here: `started` is the mark, since this stream carries no + # per-emission item start to pull the window open to. + started, raw_generation_ms = close_window(mark=generation_started_wall, now=message_arrival_wall) + assistant_telemetry = AssistantMessageTelemetry( + started_at=started, completed_at=message_arrival_wall, - generation_duration_ms=max(0.0, generation_duration_ms), + generation_duration_ms=raw_generation_ms, content_blocks=turn_content_blocks, tool_use_ids=turn_tool_use_ids, input_tokens=in_tok, @@ -461,7 +467,6 @@ def on_assistant_message(self, message: Message) -> None: self.emission_proxies_by_id.setdefault(message_id, []).append(emission_content_chars) self.last_assistant_message_index = len(self.sdk_messages) - 1 - self.last_event_monotonic = message_arrival_monotonic self.last_event_wall = message_arrival_wall def on_task_notification(self, message: Message) -> None: @@ -541,7 +546,6 @@ def _seed_first_generation_window(self) -> None: if self.first_output_seen: return self.first_output_seen = True - self.last_event_monotonic = time.monotonic() self.last_event_wall = datetime.now() def on_stream_event(self, message: Message) -> None: @@ -573,7 +577,6 @@ def on_user_message(self, message: Message) -> None: """Process tool results (and a sub-agent's terminal generation) from a tool-result UserMessage. The sub-agent message is appended BEFORE the tool-result loop — its position in ``sdk_messages`` is observable.""" - self.last_event_monotonic = time.monotonic() self.last_event_wall = datetime.now() sub_msg = self._agent._synthesize_subagent_terminal_message(message, self.sdk_model_used) @@ -638,49 +641,6 @@ def _finalize_token_usage(self) -> TokenUsage: self._agent._reprice_for_litellm(usage, self.effective_model) return usage - def _subtract_tool_time_from_windows(self, commands: list[CommandTelemetry]) -> None: - """Take tool execution back out of the generation windows it overlapped. - - The other four harnesses do this as they flush, because their stream - interleaves tool calls into one window. claude-code was exempted on the - premise that a tool's execution falls BETWEEN two windows — but a tool's - timer starts at the emission carrying its ``tool_use`` block, and one - assistant turn spans several emissions, so a later emission's window - runs concurrently with a tool already timing. Measured on a task with - two concurrent ``Bash`` calls: 482 ms and 340 ms of a ~18-25 s turn - counted as both generation and tool, which is exactly the amount by - which the four-bucket identity missed. - - Deferred to finalization rather than done in ``on_assistant_message`` - because that is the first point where every span is known: a tool - issued by an earlier emission is still running when the next window - closes, so its interval does not exist yet. - - ``generation_duration_ms`` therefore means the same thing on all five - harnesses — wall time inside the window with no tool running. A window - entirely covered by tool execution legitimately reads ``0.0``; that is - a measurement, and ``None`` remains what "never measured" means. - """ - spans = [ - (c.execution_started_at, c.execution_completed_at) - for c in commands - if c.execution_started_at is not None and c.execution_completed_at is not None - ] - if not spans: - return - for emission in self.sdk_messages: - # A sub-agent's generation is not on this timeline: its own tools - # are not in `commands`, and the Agent call that spawned it already - # spans its whole run. A UserMessage / ReconciliationMessage has no - # window at all. - if not isinstance(emission, AssistantMessageTelemetry): - continue - if emission.generation_duration_ms is None or emission.parent_tool_use_id is not None: - continue - overlap = busy_ms(spans, emission.started_at, emission.completed_at) - if overlap > 0.0: - emission.generation_duration_ms = max(emission.generation_duration_ms - overlap, 0.0) - def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reason: str | None = None) -> None: """Close orphaned tools + the open turn, emit the terminal AgentEndEvent, and on a crash build the partial TurnRecord. Idempotent.""" @@ -689,7 +649,6 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso self.finalized = True commands = self._agent._finalize_commands(self.pending_commands, self.messages) - self._subtract_tool_time_from_windows(commands) for cmd in commands: if cmd.tool_id in self.emitted_tool_ends: continue diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index 6202f218b..cb438f68d 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -477,31 +477,17 @@ def _flush_message(self, last: Any) -> None: window_end_ms = self.open_end_ms if self.open_end_ms is not None else self.open_start_ms mark = _ms_to_dt(mark_ms) completed = _ms_to_dt(window_end_ms) - # The window is extended to the LAST item's completion, so any + # The RAW window. It is extended to the LAST item's completion, so a # generation containing a tool call already CONTAINS that tool's - # execution. Publishing the raw span as generation time double-counts - # it against the tool's own duration_ms: a tool-only emission reported - # 250ms of "generation" for a 250ms `echo hi`, and the task page's - # Generation + Tool exec then exceeded the wall clock they must - # reconcile to. - # - # Same shared helper as the other tiling harnesses: the UNION of the - # tool intervals clipped to this window, the open calls bounded at its - # end, and the double-subtraction rule they rest on — all in - # `close_window`'s docstring rather than restated here. - tool_spans = [ - (c.execution_started_at, c.execution_completed_at) - for c in self.commands - if c.execution_started_at is not None and c.execution_completed_at is not None - ] + # execution — but taking it back out is no longer this reducer's job. + # `EventCollector.subtract_tool_time` does it for all five, which is + # also what makes the sub-message split below safe: the two specs share + # these bounds, so the collector groups them and subtracts the overlap + # ONCE rather than once per part. started, gen_ms = close_window( mark=mark, now=completed, item_start=_ms_to_dt(self.open_start_ms) if self.open_start_ms is not None else None, - closed_spans=tool_spans, - open_started_ats=[ - t.execution_started_at for t in self.open_tools.values() if t.execution_started_at is not None - ], ) message_id = f"{self.turn_id}-msg-{self.gen_index}" diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index f8d73e6d7..c7bd6d561 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -325,14 +325,6 @@ def __init__(self, *, task_id: str, iteration: int, user_input: str, model: str self.gen_mark: datetime | None = None self.step_text_parts: list[str] = [] self.step_tool_ids: list[str] = [] - # Execution intervals of tools that CLOSED inside the open - # generation window. Every tool call runs INSIDE the window, so - # publishing the raw span as generation time counts the same - # milliseconds twice — once here and once as the tool's own - # duration_ms. Intervals, not a running total: they overlap - # whenever the harness runs tools concurrently, and only their - # union may be subtracted (timing.py::busy_ms). - self.step_tool_spans: list[tuple[datetime, datetime]] = [] # callID -> (telemetry, started_at) for tools awaiting a result. self.open_tools: dict[str, CommandTelemetry] = {} @@ -370,14 +362,13 @@ def on_step_start(self, part: dict[str, Any]) -> None: self.step_started_at = datetime.now() self.step_text_parts = [] self.step_tool_ids = [] - # `step_tool_spans` is deliberately NOT reset here. The window this - # list feeds opened at `gen_mark` — the PREVIOUS step's finish — so a - # call closing in the gap before this `step_start` belongs to it, and - # clearing the list now wipes the span before `step_finish` can - # subtract it. Reproduced: the window then published the call's - # execution as model time while the call's own `duration_ms` counted - # the same milliseconds again — a 100% overstatement of that window. - # It is cleared at the flush instead, right after the mark advances. + # There is no per-step span list to reset here any more, and that whole + # class of defect is gone with it: `EventCollector.subtract_tool_time` + # sees every span at once and clips each to the window it overlaps, so + # a call closing in the gap before this `step_start` needs nobody to + # remember it. The reset rule that used to live here was wrong once + # (clearing at `step_start` wiped the span before `step_finish` could + # subtract it — a 100% overstatement of that window). self.emit( TurnStartEvent( task_id=self.task_id, @@ -495,10 +486,6 @@ def _close_tool( telemetry.execution_completed_at = completed if telemetry.execution_started_at is not None: telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 - # This tool ran inside the open generation window, so its time is - # not model time. Only a RESOLVED tool contributes: one force-closed - # without a result was never timed. - self.step_tool_spans.append((telemetry.execution_started_at, completed)) telemetry.result_status = _RESULT_STATUS[status] # Stored untruncated by design (sub-agent returns must survive whole). telemetry.result_summary = summary @@ -715,16 +702,13 @@ def on_step_finish(self, part: dict[str, Any]) -> None: for i, tool_id in enumerate(self.step_tool_ids, start=len(blocks)): blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) - # Tile from the previous step's finish. The open calls and the double- - # subtraction rule they rest on live in `close_window`'s docstring. + # Tile from the previous step's finish. The RAW window only — + # `EventCollector.subtract_tool_time` takes the tool union back out of + # it, once, for every harness. started, generation_ms = close_window( mark=self.gen_mark if self.gen_mark is not None else step_start, now=completed, item_start=step_start, - closed_spans=self.step_tool_spans, - open_started_ats=[ - t.execution_started_at for t in self.open_tools.values() if t.execution_started_at is not None - ], ) self.messages.append( AssistantMessage( @@ -746,10 +730,9 @@ def on_step_finish(self, part: dict[str, Any]) -> None: # A message was appended, so the next window starts where this one # ended. Only `step_finish` advances the mark: a step that never # finished published nothing, so tiling past it would attribute its - # time to whichever step finishes next. The span list is cleared with - # it, and only with it — see `on_step_start`. + # time to whichever step finishes next. There is no span list to clear + # alongside it any more — see `on_step_start`. self.gen_mark = completed - self.step_tool_spans = [] # And so is this step's own start stamp, because it has now been SPENT. # It is passed to `close_window` as `item_start`, whose `min()` pulls # the window open to cover it; left in place, a second `step_finish` diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index d7d78fcdb..f53af2ca9 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -303,14 +303,6 @@ def __init__( self.turn_started_at: datetime | None = None self.turn_text_parts: list[str] = [] self.turn_tool_ids: list[str] = [] - # Execution intervals of tools that CLOSED inside the open - # generation window. Every tool call runs INSIDE the window, so - # publishing the raw span as generation time counts the same - # milliseconds twice — once here and once as the tool's own - # duration_ms. Intervals, not a running total: they overlap - # whenever the harness runs tools concurrently, and only their - # union may be subtracted (timing.py::busy_ms). - self.turn_tool_spans: list[tuple[datetime, datetime]] = [] # Where the NEXT generation window starts: the previous turn's end. # Pi was the only harness measuring from its own `turn_start`, so the # wall clock between one `turn_end` and the next `turn_start` — the @@ -381,11 +373,10 @@ def on_turn_start(self) -> None: self.turn_started_at = self.clock.now() self.turn_text_parts = [] self.turn_tool_ids = [] - # `turn_tool_spans` is deliberately NOT reset here — see the identical - # note in `opencode_agent.on_step_start`. Now that the window opens at - # `gen_mark` rather than at this `turn_start`, a call closing in the - # gap between them belongs to it, and clearing the list here would - # publish that call's execution as the next window's model time. + # No per-turn span list to reset here any more — see the identical note + # in `opencode_agent.on_step_start`. The collector subtracts from final + # bounds with every span known, so nothing has to remember a call that + # closed in the gap before this `turn_start`. self.emit( TurnStartEvent( task_id=self.task_id, @@ -470,10 +461,6 @@ def _close_tool( telemetry.execution_completed_at = completed if telemetry.execution_started_at is not None: telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 - # This tool ran inside the open generation window, so its time is - # not model time. Only a RESOLVED tool contributes: one force-closed - # without a result was never timed. - self.turn_tool_spans.append((telemetry.execution_started_at, completed)) telemetry.result_status = _RESULT_STATUS[status] # Stored untruncated by design (sub-agent returns must survive whole). telemetry.result_summary = summary @@ -610,17 +597,14 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: for i, tool_id in enumerate(self.turn_tool_ids, start=len(blocks)): blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) - # The open calls and the double-subtraction rule they rest on live in - # `close_window`'s docstring. + # Tile from the previous turn's end. The RAW window only — + # `EventCollector.subtract_tool_time` takes the tool union back out of + # it, once, for every harness. turn_start = self.turn_started_at if self.turn_started_at is not None else completed started, generation_ms = close_window( mark=self.gen_mark if self.gen_mark is not None else turn_start, now=completed, item_start=turn_start, - closed_spans=self.turn_tool_spans, - open_started_ats=[ - t.execution_started_at for t in self.open_tools.values() if t.execution_started_at is not None - ], ) self.messages.append( AssistantMessage( @@ -642,10 +626,9 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: # A message was appended, so the next window starts where this one # ended. Only a finished turn advances the mark: one that never # finished published nothing, so tiling past it would attribute its - # time to whichever turn finishes next. The span list is cleared with - # it, and only with it — see `on_turn_start`. + # time to whichever turn finishes next. There is no span list to clear + # alongside it any more — see `on_turn_start`. self.gen_mark = completed - self.turn_tool_spans = [] # And so is this turn's own start stamp, because it has now been SPENT. # It is passed to `close_window` as `item_start`, whose `min()` pulls # the window open to cover it; left in place, a second `turn_end` with diff --git a/src/coder_eval/models/telemetry.py b/src/coder_eval/models/telemetry.py index 65a8f5acb..a29d12f92 100644 --- a/src/coder_eval/models/telemetry.py +++ b/src/coder_eval/models/telemetry.py @@ -227,9 +227,15 @@ class AssistantMessage(BaseModel): description=( "Model-generation time for this emission, in milliseconds. None when the harness " "surfaced the message with no measurable window (a rollout rebuild, or a sub-agent " - "generation delivered as a tool result). Equals completed_at - started_at only when " - "no tool execution closed inside the window; every harness subtracts tool time that " - "ran inside one (claude-code does it at finalization, the other four as they flush). " + "generation delivered as a tool result). " + "WRITTEN BY THE COLLECTOR, not by the agent: a reducer publishes the RAW window it " + "measured, and streaming/collector.py::subtract_tool_time takes the UNION of the " + "main-thread tool intervals back out of it, once, for every harness. So this equals " + "completed_at - started_at only when no tool execution overlapped the window, and a " + "reader of an agent's own AssistantMessage(...) call is NOT looking at the published " + "value. Messages sharing one pair of bounds (Codex splits a window into thinking and " + "action sub-messages) are one group: the overlap comes out once and is re-apportioned " + "across them, so the parts still sum to the group's total. " "The property this field exists " "to make true — once every harness records a real window — is the FOUR-bucket identity: " "sum(generation_duration_ms) + UNION(command execution intervals) " @@ -238,7 +244,8 @@ class AssistantMessage(BaseModel): "uses one (timing.py::busy_ms): concurrent tool calls otherwise book their overlap twice. " "The last two are the turn's " "head and tail, which no message can carry because they are the wall clock OUTSIDE every " - "generation window; without them the identity holds only on a harness with no CLI to boot. " + "generation window; without them the identity holds only on a harness with nothing to " + "boot or dispatch before its first model output. " "Per-harness status is in docs/agents/HARNESS_PARITY.md; do not assume it holds " "for a harness that table does not yet claim it for." ), diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 4a08d54e0..07a0ef4c4 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -39,7 +39,86 @@ ToolEndEvent, TurnStartEvent, ) -from coder_eval.timing import decompose_turn +from coder_eval.timing import busy_ms, decompose_turn + + +def subtract_tool_time( + messages: list[TranscriptMessage], + spans: list[tuple[datetime, datetime]], +) -> list[TranscriptMessage]: + """Take tool execution back out of the generation windows it overlapped. + + THE one place this happens. Five reducers used to do it themselves — four + through ``close_window`` as they flushed, claude-code once at finalization — + while the head and tail were already computed centrally, right here. That + asymmetry was the complexity, and every timing defect this branch fixed + lived in the per-reducer bookkeeping around the subtraction rather than in + the subtraction itself: when to reset a span list, when to clear a start + stamp, when to advance a mark. A reducer now publishes the RAW window and + keeps only the genuinely harness-shaped decision, which is where its window + opens. + + NON-MUTATING, and the reason is aliasing rather than repeated calls. Every + agent builds its terminal event as ``AgentEndEvent(messages=list(...))`` — + that copies the LIST, not the message objects — so writing in place would + reach back into the agent's own live state from the collector, which is + exactly the layering "the collector is the sole capture seam" exists to + prevent. ``model_copy`` keeps it one-directional. It is also unconditionally + safe for any caller that builds a record twice: ``EarlyStopWatcher`` holds + one collector across a turn's tool-call rounds and calls + ``build_turn_record`` on every one. + + GROUPED BY IDENTICAL BOUNDS, not by ``message_id``. Codex splits one window + across two sub-messages (thinking and action) that share ``started_at`` and + ``completed_at`` and divide the window by output-token share; subtracting + the group's overlap from each part separately would subtract it twice and + stop the parts summing to the window. Bounds identity covers that, and it + also covers OpenCode and Pi, which can legitimately carry + ``message_id is None`` — so keying on the id would silently collapse every + id-less message of a turn into one group. + + MAIN THREAD ONLY. A sub-agent generation (``parent_tool_use_id`` set) is + skipped: its own tools are not in this span set, and the Agent call that + spawned it already covers its whole run. + + A ``generation_duration_ms`` of ``None`` means no window was ever measured + (codex's rollout rebuild, claude's synthesized sub-agent terminal), so there + is nothing to subtract from and it passes through untouched — never + coerced to ``0.0`` (CE058). Every non-``AssistantMessage`` entry — a + simulation ``UserMessage``, the appended ``ReconciliationMessage`` — passes + through by identity. + + A window entirely covered by tool execution reaches ``0.0``, and that is a + measurement rather than an absence. + """ + # (index, raw window ms) per group. The raw value is captured HERE, where + # the message is already narrowed to AssistantMessage, so the apportioning + # loop below needs no second narrowing. + groups: dict[tuple[datetime, datetime], list[tuple[int, float]]] = {} + for index, message in enumerate(messages): + if not isinstance(message, AssistantMessage): + continue + raw = message.generation_duration_ms + if raw is None or message.parent_tool_use_id is not None: + continue + groups.setdefault((message.started_at, message.completed_at), []).append((index, raw)) + + out = list(messages) + for (started, completed), members in groups.items(): + raw_total = sum(raw for _, raw in members) + # Nothing to apportion, and dividing by it is a ZeroDivisionError. A + # group already at zero stays at zero. + if raw_total <= 0: + continue + net = max(raw_total - busy_ms(spans, started, completed), 0.0) + assigned = 0.0 + for n, (index, raw) in enumerate(members): + # The last member takes the remainder so the parts reconstruct the + # group's net exactly, rather than drifting by the rounding. + share = net - assigned if n == len(members) - 1 else round(net * (raw / raw_total), 6) + out[index] = out[index].model_copy(update={"generation_duration_ms": share}) + assigned += share + return out class EventCollector: @@ -115,7 +194,9 @@ def visible_turn_count(self) -> int: def _ordered_commands(self) -> list[CommandTelemetry]: return sorted(self._commands.values(), key=lambda c: c.sequence_number) - def _overhead_ms(self, messages: list[TranscriptMessage]) -> tuple[float | None, float | None]: + def _overhead_ms( + self, messages: list[TranscriptMessage], tool_spans: list[tuple[datetime, datetime]] | None = None + ) -> tuple[float | None, float | None]: """The turn's head and tail — the wall clock the generations do not cover. Measured against ``AssistantMessage`` entries only: a simulation turn @@ -165,13 +246,43 @@ def _overhead_ms(self, messages: list[TranscriptMessage]) -> tuple[float | None, max(m.completed_at for m in generations), self._agent_start_at, self._agent_end.timestamp if self._agent_end is not None else None, - [ - (c.execution_started_at, c.execution_completed_at) - for c in self._commands.values() - if c.execution_started_at is not None and c.execution_completed_at is not None - ], + tool_spans if tool_spans is not None else self._main_thread_tool_spans(messages), ) + def _main_thread_tool_spans(self, messages: list[TranscriptMessage]) -> list[tuple[datetime, datetime]]: + """Bounded execution intervals of the MAIN THREAD's tool calls. + + The span set both the head/tail decomposition and the generation + subtraction are measured against, so they cannot disagree about which + calls exist. + + Sub-agent tools are excluded, and this used to be the gap: ``_overhead_ms`` + filtered its GENERATIONS to the main thread and then passed EVERY + command, so its docstring's claim to keep all four buckets measuring one + thread was true only by luck. It held because a child nests inside the + parent Agent call, whose own interval the union already covers — but + Codex's recovered child tools carry the CHILD's clock, so nothing made + it true by construction. The evalboard's twin (``toolExecutionMs``) does + filter, so the two implementations agreed by accident. + + A sub-agent's tool ids are reachable only through the messages that own + them: a child generation carries ``parent_tool_use_id``, and its + ``tool_use_ids`` are the calls it made. + """ + sub_agent_tool_ids = { + tool_id + for m in messages + if isinstance(m, AssistantMessage) and m.parent_tool_use_id is not None + for tool_id in m.tool_use_ids + } + return [ + (c.execution_started_at, c.execution_completed_at) + for c in self._commands.values() + if c.execution_started_at is not None + and c.execution_completed_at is not None + and c.tool_id not in sub_agent_tool_ids + ] + @staticmethod def _reconciled_messages(messages: list[TranscriptMessage], usage: TokenUsage) -> list[TranscriptMessage]: """Append a ``ReconciliationMessage`` so the transcript's token buckets @@ -259,10 +370,22 @@ def build_turn_record(self) -> TurnRecord: # to the total — making the stream self-reconciling for any downstream # consumer (e.g. the evalboard) without a competing aggregate. messages: list[TranscriptMessage] = list(end.messages) + # Tool execution comes out of the generation windows HERE, once, for + # every harness — the reducers publish raw windows. + # + # The span set is computed ONCE and handed to both consumers. That is + # the invariant worth protecting, and it is the one that is easy to + # break: the subtraction and the head/tail must agree about which calls + # exist, or the buckets stop being disjoint. (The ORDER of the two is + # not load-bearing — `_overhead_ms` reads only the bounds, the + # main-thread flag and whether the duration is `None`, none of which + # `subtract_tool_time` changes. Do not add a comment claiming it is.) + tool_spans = self._main_thread_tool_spans(messages) + messages = subtract_tool_time(messages, tool_spans) if token_usage is not None: messages = self._reconciled_messages(messages, token_usage) - startup_ms, teardown_ms = self._overhead_ms(messages) + startup_ms, teardown_ms = self._overhead_ms(messages, tool_spans) return TurnRecord( iteration=end.iteration or self._iteration, diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 712697ff9..e89c4161f 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -4,15 +4,18 @@ ``agents/`` because ``EventCollector`` consumes it, and importing anything under ``agents/`` pulls in every agent, which imports ``streaming/``. -EVERY harness now subtracts tool execution from its generation windows before -publishing ``generation_duration_ms``, and all of them subtract the same -thing: the UNION of the intervals, clipped to the window. Two interleave a -tool into a single window outright — Antigravity (the Step for the tool -arrives and only a later ``usage_metadata`` Step cuts the message) and Codex -(``_flush_message``'s window is extended to the last item's -``completed_at_ms``). The other three reach the same place from the opposite -direction: their windows tile the turn contiguously, so a call open at a -window boundary runs inside two of them. +NO harness subtracts tool execution from its own generation windows. Each +publishes the RAW window it measured, and ``streaming/collector.py::subtract_tool_time`` +takes the UNION of the tool intervals back out of them once, for all five, at +the single capture seam — the same place the head and the tail are already +computed. A reducer's only remaining timing decision is where its window +opens, which is the one genuinely harness-shaped part: two interleave a tool +into a single window outright (Antigravity, whose Step for the tool arrives and +only a later ``usage_metadata`` Step cuts the message, and Codex, whose +``_flush_message`` window extends to the last item's ``completed_at_ms``) while +the other three tile the turn contiguously, so a call open at a boundary runs +inside two windows. Central subtraction handles both without either reducer +knowing which it is. There is a TypeScript twin, ``evalboard/lib/timing.ts::busyMs``, which subtracts tool time from a task's WALL CLOCK to produce the Unaccounted @@ -66,6 +69,13 @@ class TurnClock: CLI's own epoch-millisecond stamps, unreachable from the host, so converting only the window bounds would put two bases inside one ``busy_ms`` subtraction — relocating the defect instead of removing it. + + claude-code does not use it either, but for no good reason: it has no + epoch-stamp constraint, it simply has not been converted. Its window bounds + and its span now share one basis (raw ``datetime.now()``), so the two cannot + disagree with each other — but both carry the naive-local exposure this + class removes. Converting it is the remaining work; see + docs/agents/HARNESS_PARITY.md. """ def __init__(self) -> None: @@ -181,21 +191,16 @@ def union_ms(spans: list[tuple[datetime, datetime]]) -> float: return busy_ms(spans, min(s for s, _ in spans), max(e for _, e in spans)) -def close_window( - *, - mark: datetime, - now: datetime, - item_start: datetime | None = None, - closed_spans: list[tuple[datetime, datetime]], - open_started_ats: list[datetime], -) -> tuple[datetime, float]: - """Close one generation window at ``now``: its ``(started, generation_ms)``. +def close_window(*, mark: datetime, now: datetime, item_start: datetime | None = None) -> tuple[datetime, float]: + """Open one generation window at ``mark`` and close it at ``now``: its ``(started, span_ms)``. - The shape four reducers had copy-pasted. Codex, opencode, pi and - antigravity all call it; claude-code is the one exception and carries the - only ``# noqa: CE061``, because it subtracts tool time once at finalization - across every emission rather than per flush — a call issued by an earlier - emission is still running when the next window closes. + The shape all five reducers share. What it returns is the RAW window — + tool execution is taken back out of it once, centrally, in + ``streaming/collector.py::subtract_tool_time``, which is the only place + that arithmetic lives. It used to happen here too, per flush, and in + claude-code at finalization; the per-reducer bookkeeping that required + (a span list, its reset rule, the set of still-open calls) is where every + timing defect on this branch actually lived. ``mark`` is where the window opens: the previous flush's close, which is what makes the windows TILE the turn contiguously instead of leaving the @@ -210,22 +215,12 @@ def close_window( ``item_start`` is this emission's own first stamp, when the harness has one. The ``min()`` against ``mark`` is the tiling defense and nothing else: a stamp that went backwards must never push the window start PAST the first - item and invert the span. - - A call still OPEN at this boundary counts against the window too, bounded - at ``now``. Subtracting only CLOSED intervals publishes the part of a - straddling call that ran inside this window as generation, while the call's - own ``duration_ms`` counts it again. - - NO DOUBLE SUBTRACTION, and this is the rationale that used to sit copy- - pasted at four call sites: when that open call later closes, the reducer - appends its FULL interval to the next window's ``closed_spans``, where - ``busy_ms`` clips it to the post-boundary remainder. Each millisecond of - tool time is therefore subtracted from exactly one window. + item and invert the span. claude-code passes none — its stream carries no + per-emission item start — so its window opens exactly at the mark. - The UNION is subtracted, never the sum (see ``busy_ms``), and the result is - clamped at ``0.0`` — an inverted window (``now`` before ``mark``, two - clocks disagreeing) is a measured zero, not a negative generation. + The result is clamped at ``0.0``: an inverted window (``now`` before + ``mark``, two clocks disagreeing) is a measured zero, not a negative + generation. It deliberately does NOT return ``completed``. The window always ends at ``now``, which the caller passed in, so handing it back would be an @@ -233,9 +228,7 @@ def close_window( write ``completed_at=now`` directly. """ started = min(mark, item_start) if item_start is not None else mark - bounded = [(s, now) for s in open_started_ats if s < now] - span_ms = (now - started).total_seconds() * 1000.0 - return started, max(0.0, span_ms - busy_ms(closed_spans + bounded, started, now)) + return started, max(0.0, (now - started).total_seconds() * 1000.0) def decompose_turn( diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index dddd51e8a..8afba06a2 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -117,10 +117,32 @@ def assert_reconciliation(record: dict[str, Any]) -> None: _IDENTITY_SHARE = 0.20 +def _sub_agent_tool_ids(record: dict[str, Any]) -> set[str]: + """Tool ids owned by a SUB-AGENT generation. + + Must match `EventCollector._main_thread_tool_spans` and + `scripts/timing/decompose_run.py::_sub_agent_tool_ids`: all three recompute + the tool union for the same identity, so a filter applied by one and not + the others reports a residual that is an artifact of the disagreement. + """ + ids: set[str] = set() + for message in record.get("messages") or []: + if message.get("role") == "assistant" and message.get("parent_tool_use_id") is not None: + ids.update(message.get("tool_use_ids") or []) + return ids + + def _tool_union_ms(record: dict[str, Any]) -> float: - """Wall ms this turn spent executing tools — the union, never the sum.""" + """Wall ms this turn's MAIN-THREAD tools occupied — the union, never the sum. + + Sub-agent tools are excluded for the same reason their generations are: the + spawning Agent call's own interval already spans the child's whole run. + """ + excluded = _sub_agent_tool_ids(record) spans: list[tuple[datetime, datetime]] = [] for command in record.get("commands") or []: + if command.get("tool_id") in excluded: + continue start = _parse_stamp(command.get("execution_started_at")) end = _parse_stamp(command.get("execution_completed_at")) if start is not None and end is not None and end >= start: @@ -275,8 +297,10 @@ def assert_timing_captured( "They are meant to be DISJOINT, so a sum this far over the turn means something is " "booked twice — most likely a tool that ran outside every generation window and was " "left in the head or tail as well as in the tool union, or a generation window that " - "kept tool time it should have subtracted (see docs/agents/HARNESS_PARITY.md — all " - "five harnesses subtract, claude-code at finalization rather than as it flushes)" + "kept tool time it should have subtracted (see docs/agents/HARNESS_PARITY.md — the " + "subtraction happens once, in streaming/collector.py::subtract_tool_time, so a " + "double-count is a span the collector saw twice or a reducer publishing a window it " + "already narrowed)" ) if not expect_generation_window: diff --git a/tests/_fixtures/timing_runs/README.md b/tests/_fixtures/timing_runs/README.md new file mode 100644 index 000000000..838b0c219 --- /dev/null +++ b/tests/_fixtures/timing_runs/README.md @@ -0,0 +1,53 @@ +# Pinned per-harness timing corpus + +One scrubbed, representative `task.json` per harness, from a real +`tasks/timing-parallel-tools` run. Prompts, outputs, tokens and cost are +stripped; only the wall-clock fields `scripts/timing/decompose_run.py` reads +survive. + +``` +uv run python scripts/timing/decompose_run.py tests/_fixtures/timing_runs/*.json --min-turn-ms 0 +``` + +## What this is NOT + +**It cannot show a before/after for a code change, and it was originally asked +to.** `decompose_run.py` READS STORED FIELDS — `harness_startup_ms`, +`generation_duration_ms`, the command bounds — out of a recorded record. It +recomputes nothing from `src/`. Run over a fixed corpus it prints the identical +table before and after any change to the harness, so a green result would be +meaningless rather than reassuring. + +Two rows here make that concrete: + +- **claude-code reconciles at −481 ms (−2.69%).** That is the pre-subtraction + defect `_subtract_tool_time_from_windows` was written to fix, frozen in a + record written before the fix landed. The live code has not had that defect + for some time. +- **claude-code and antigravity both book a `0.0` head.** That is the clamped + inversion the "one meaning for `harness_startup_ms`" change removed. Neither + harness produces it any more. + +Re-recording the corpus after a change would fix both, and would also destroy +the only thing the corpus is good for. + +## What it IS + +A reproducible statement of what real runs of each harness look like — the +shape of the buckets, the per-harness spread in the head, and a fixed input for +`decompose_run.py` itself. It is what the timing audit's P0 was read off: the +two harnesses reporting a `0.0` head had a FIRST generation window 2.4–3.8× their +own later median, and that excess was the startup they were not booking. + +## The instruments that DO move with the code + +- `tests/test_timing_identity_contract.py` — drives each of the five reducers + off a scripted clock and asserts `head + Σgeneration + UNION(tool) + tail` + equals the turn span to the millisecond. This is the sensor for the + arithmetic. +- `tests/test_agent_golden_master.py` — replays recorded streams end to end. + Note it masks every timing VALUE (`_scrub.py::SCRUB_KEYS`), so it sees shape + and not magnitude. +- `scripts/timing/decompose_run.py --max-residual-pct` over a **fresh** run, + which `.github/workflows/pr-checks.yml` does against the smoke-pass bucket. + A live run is the only way this script can say anything about current code. diff --git a/tests/_fixtures/timing_runs/antigravity.json b/tests/_fixtures/timing_runs/antigravity.json new file mode 100644 index 000000000..a662f2f9a --- /dev/null +++ b/tests/_fixtures/timing_runs/antigravity.json @@ -0,0 +1,134 @@ +{ + "_comment": "Scrubbed representative run for the timing residual report. Prompts, outputs, tokens and cost are removed; only the wall-clock fields decompose_run.py reads are kept. NOT a before/after instrument \u2014 see the header in this directory.", + "agent_type": "antigravity", + "iterations": [ + { + "duration_seconds": 11.191662207944319, + "iteration": 1, + "harness_startup_ms": 0.0, + "harness_teardown_ms": 0.116, + "crashed": false, + "messages": [ + { + "completed_at": "2026-09-11T21:25:56.137300", + "started_at": "2026-09-11T21:25:51.666942", + "generation_duration_ms": 4454.441, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T21:26:00.773915", + "started_at": "2026-09-11T21:25:56.137300", + "generation_duration_ms": 4574.724999999999, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T21:26:02.858513", + "started_at": "2026-09-11T21:26:00.773915", + "generation_duration_ms": 0.3619999999996253, + "role": "assistant", + "parent_tool_use_id": null + } + ], + "commands": [ + { + "tool_name": "Edit", + "execution_completed_at": "2026-09-11T21:25:55.126590", + "execution_started_at": "2026-09-11T21:25:55.122510", + "tool_id": "b7747e76d7fb722e11ea267731099383:2", + "result_status": "success", + "duration_ms": 4.08 + }, + { + "tool_name": "Edit", + "execution_completed_at": "2026-09-11T21:25:55.455388", + "execution_started_at": "2026-09-11T21:25:55.452684", + "tool_id": "b7747e76d7fb722e11ea267731099383:3", + "result_status": "success", + "duration_ms": 2.7039999999999997 + }, + { + "tool_name": "Edit", + "execution_completed_at": "2026-09-11T21:25:55.798964", + "execution_started_at": "2026-09-11T21:25:55.796438", + "tool_id": "b7747e76d7fb722e11ea267731099383:4", + "result_status": "success", + "duration_ms": 2.5260000000000002 + }, + { + "tool_name": "Edit", + "execution_completed_at": "2026-09-11T21:25:56.101913", + "execution_started_at": "2026-09-11T21:25:56.099180", + "tool_id": "b7747e76d7fb722e11ea267731099383:5", + "result_status": "success", + "duration_ms": 2.733 + }, + { + "tool_name": "Edit", + "execution_completed_at": "2026-09-11T21:25:56.140758", + "execution_started_at": "2026-09-11T21:25:56.133426", + "tool_id": "b7747e76d7fb722e11ea267731099383:6", + "result_status": "success", + "duration_ms": 7.332 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T21:25:59.328201", + "execution_started_at": "2026-09-11T21:25:59.323643", + "tool_id": "b7747e76d7fb722e11ea267731099383:8", + "result_status": "success", + "duration_ms": 4.558000000000001 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T21:25:59.550767", + "execution_started_at": "2026-09-11T21:25:59.549154", + "tool_id": "b7747e76d7fb722e11ea267731099383:9", + "result_status": "success", + "duration_ms": 1.613 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T21:25:59.801832", + "execution_started_at": "2026-09-11T21:25:59.800660", + "tool_id": "b7747e76d7fb722e11ea267731099383:10", + "result_status": "success", + "duration_ms": 1.1720000000000002 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T21:26:00.087008", + "execution_started_at": "2026-09-11T21:26:00.086176", + "tool_id": "b7747e76d7fb722e11ea267731099383:11", + "result_status": "success", + "duration_ms": 0.832 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T21:26:00.395861", + "execution_started_at": "2026-09-11T21:26:00.394123", + "tool_id": "b7747e76d7fb722e11ea267731099383:12", + "result_status": "success", + "duration_ms": 1.738 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T21:26:02.768580", + "execution_started_at": "2026-09-11T21:26:00.725396", + "tool_id": "b7747e76d7fb722e11ea267731099383:13", + "result_status": "success", + "duration_ms": 2043.1840000000002 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T21:26:02.858151", + "execution_started_at": "2026-09-11T21:26:00.773556", + "tool_id": "b7747e76d7fb722e11ea267731099383:14", + "result_status": "success", + "duration_ms": 2084.5950000000003 + } + ] + } + ] +} diff --git a/tests/_fixtures/timing_runs/claude-code.json b/tests/_fixtures/timing_runs/claude-code.json new file mode 100644 index 000000000..1eff89cca --- /dev/null +++ b/tests/_fixtures/timing_runs/claude-code.json @@ -0,0 +1,225 @@ +{ + "_comment": "Scrubbed representative run for the timing residual report. Prompts, outputs, tokens and cost are removed; only the wall-clock fields decompose_run.py reads are kept. NOT a before/after instrument \u2014 see the header in this directory.", + "agent_type": "claude-code", + "iterations": [ + { + "duration_seconds": 17.885937000159174, + "iteration": 1, + "harness_startup_ms": 0.0, + "harness_teardown_ms": 857.9449999999999, + "crashed": false, + "messages": [ + { + "completed_at": "2026-09-11T07:13:59.899445", + "started_at": "2026-09-11T07:13:56.052088", + "generation_duration_ms": 3847.4723750259727, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:00.221777", + "started_at": "2026-09-11T07:13:59.899445", + "generation_duration_ms": 322.3441250156611, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:01.010903", + "started_at": "2026-09-11T07:14:00.221777", + "generation_duration_ms": 789.1439578961581, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:01.672766", + "started_at": "2026-09-11T07:14:01.032139", + "generation_duration_ms": 640.6468339264393, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:02.386421", + "started_at": "2026-09-11T07:14:01.676561", + "generation_duration_ms": 709.8735831677914, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:03.064864", + "started_at": "2026-09-11T07:14:02.403329", + "generation_duration_ms": 661.5535838063806, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:03.747205", + "started_at": "2026-09-11T07:14:03.070861", + "generation_duration_ms": 676.3597088865936, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:04.328511", + "started_at": "2026-09-11T07:14:03.768013", + "generation_duration_ms": 560.5133329518139, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:04.917330", + "started_at": "2026-09-11T07:14:04.338140", + "generation_duration_ms": 579.204916022718, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:05.485979", + "started_at": "2026-09-11T07:14:04.920393", + "generation_duration_ms": 565.5976661946625, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:06.071761", + "started_at": "2026-09-11T07:14:05.493927", + "generation_duration_ms": 577.8485001064837, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:06.671228", + "started_at": "2026-09-11T07:14:06.078218", + "generation_duration_ms": 593.0241669993848, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:07.253322", + "started_at": "2026-09-11T07:14:06.678189", + "generation_duration_ms": 575.1475831493735, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:07.734996", + "started_at": "2026-09-11T07:14:07.253322", + "generation_duration_ms": 481.6872498486191, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:12.089632", + "started_at": "2026-09-11T07:14:09.587005", + "generation_duration_ms": 2502.6892910245806, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:13.079613", + "started_at": "2026-09-11T07:14:12.089632", + "generation_duration_ms": 990.0077090132982, + "role": "assistant", + "parent_tool_use_id": null + } + ], + "commands": [ + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:14:01.032594", + "execution_started_at": "2026-09-11T07:14:01.011427", + "tool_id": "toolu_01BqWiy1hXCaepichHH9dnXC", + "result_status": "success", + "duration_ms": 21.16704103536904 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:14:01.676585", + "execution_started_at": "2026-09-11T07:14:01.672806", + "tool_id": "toolu_01YN7oVP3MEuUGhNgtoYXDhV", + "result_status": "success", + "duration_ms": 3.7791249342262745 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:14:02.403402", + "execution_started_at": "2026-09-11T07:14:02.386537", + "tool_id": "toolu_01JNtoHGHMzD4QamxrePBR9T", + "result_status": "success", + "duration_ms": 16.864625038579106 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:14:03.070898", + "execution_started_at": "2026-09-11T07:14:03.064935", + "tool_id": "toolu_01DaWJKb1Hm6bxcpiKjVzeYd", + "result_status": "success", + "duration_ms": 5.963291972875595 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:14:03.768071", + "execution_started_at": "2026-09-11T07:14:03.747296", + "tool_id": "toolu_014MD2sCpV9RhaF9TjAQccFT", + "result_status": "success", + "duration_ms": 20.77529113739729 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:14:04.338200", + "execution_started_at": "2026-09-11T07:14:04.328605", + "tool_id": "toolu_01M2q1dfDu3Vsxte9dSv6AhA", + "result_status": "success", + "duration_ms": 9.594874922186136 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:14:04.920425", + "execution_started_at": "2026-09-11T07:14:04.917406", + "tool_id": "toolu_01FueR7TLLvWTEWXEbfGErzE", + "result_status": "success", + "duration_ms": 3.019041148945689 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:14:05.493992", + "execution_started_at": "2026-09-11T07:14:05.486085", + "tool_id": "toolu_017wr4qR5X22aegAGbtx9PVE", + "result_status": "success", + "duration_ms": 7.906709099188447 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:14:06.078272", + "execution_started_at": "2026-09-11T07:14:06.071847", + "tool_id": "toolu_01HDYj2UeY1Nagd1ecjbs81V", + "result_status": "success", + "duration_ms": 6.425125058740377 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:14:06.678240", + "execution_started_at": "2026-09-11T07:14:06.671314", + "tool_id": "toolu_01UbZp7DRorw4cUbo5PB2Bia", + "result_status": "success", + "duration_ms": 6.9256669376045465 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:09.527272", + "execution_started_at": "2026-09-11T07:14:07.253341", + "tool_id": "toolu_0194Py8X1atsEPAA5WAWVjyT", + "result_status": "success", + "duration_ms": 2273.930750088766 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:09.587030", + "execution_started_at": "2026-09-11T07:14:07.734993", + "tool_id": "toolu_018Nbqg6YSdYk6fb9c4gWAGy", + "result_status": "success", + "duration_ms": 1852.037207921967 + } + ] + } + ] +} diff --git a/tests/_fixtures/timing_runs/codex.json b/tests/_fixtures/timing_runs/codex.json new file mode 100644 index 000000000..0042aaf67 --- /dev/null +++ b/tests/_fixtures/timing_runs/codex.json @@ -0,0 +1,109 @@ +{ + "_comment": "Scrubbed representative run for the timing residual report. Prompts, outputs, tokens and cost are removed; only the wall-clock fields decompose_run.py reads are kept. NOT a before/after instrument \u2014 see the header in this directory.", + "agent_type": "codex", + "iterations": [ + { + "duration_seconds": 19.704716542037204, + "iteration": 1, + "harness_startup_ms": 4209.975, + "harness_teardown_ms": 6.189, + "crashed": false, + "messages": [ + { + "completed_at": "2026-09-11T07:14:25.044000", + "started_at": "2026-09-11T07:14:22.104000", + "generation_duration_ms": 1593.815668, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:25.044000", + "started_at": "2026-09-11T07:14:22.104000", + "generation_duration_ms": 1337.184332, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:31.755000", + "started_at": "2026-09-11T07:14:25.044000", + "generation_duration_ms": 6436.0, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:14:37.592000", + "started_at": "2026-09-11T07:14:31.755000", + "generation_duration_ms": 4040.0, + "role": "assistant", + "parent_tool_use_id": null + } + ], + "commands": [ + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:14:25.044000", + "execution_started_at": "2026-09-11T07:14:25.035000", + "tool_id": "call_MFDN1SquQKjnPMAC7cdID6es", + "result_status": "success", + "duration_ms": 9.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:31.452000", + "execution_started_at": "2026-09-11T07:14:31.452000", + "tool_id": "call_XoyG0xCaUZkVh4gs4HscEQY6", + "result_status": "success", + "duration_ms": 0.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:31.474000", + "execution_started_at": "2026-09-11T07:14:31.474000", + "tool_id": "call_3xwdnJ9958Qws766mtNMUmum", + "result_status": "success", + "duration_ms": 0.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:31.480000", + "execution_started_at": "2026-09-11T07:14:31.480000", + "tool_id": "call_kQU4wxYIqLLd7MxdhVBXt1fz", + "result_status": "success", + "duration_ms": 0.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:33.552000", + "execution_started_at": "2026-09-11T07:14:31.480000", + "tool_id": "call_EHI6niBRHKpW9GSxYgbSGPQn", + "result_status": "success", + "duration_ms": 2072.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:31.483000", + "execution_started_at": "2026-09-11T07:14:31.483000", + "tool_id": "call_5UNrspGHKnOkwpcR5PedDt5B", + "result_status": "success", + "duration_ms": 0.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:14:31.487000", + "execution_started_at": "2026-09-11T07:14:31.487000", + "tool_id": "call_xOqzas8ctiA0ri1wIsfiJobK", + "result_status": "success", + "duration_ms": 0.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": null, + "execution_started_at": null, + "tool_id": "call_zAeVW6TDp7vxHy3HilRTAa3i", + "result_status": null, + "duration_ms": null + } + ] + } + ] +} diff --git a/tests/_fixtures/timing_runs/opencode.json b/tests/_fixtures/timing_runs/opencode.json new file mode 100644 index 000000000..749e62e37 --- /dev/null +++ b/tests/_fixtures/timing_runs/opencode.json @@ -0,0 +1,171 @@ +{ + "_comment": "Scrubbed representative run for the timing residual report. Prompts, outputs, tokens and cost are removed; only the wall-clock fields decompose_run.py reads are kept. NOT a before/after instrument \u2014 see the header in this directory.", + "agent_type": "opencode", + "iterations": [ + { + "duration_seconds": 22.314462833106518, + "iteration": 1, + "harness_startup_ms": 3018.525, + "harness_teardown_ms": 26.273, + "crashed": false, + "messages": [ + { + "completed_at": "2026-09-11T20:13:55.133594", + "started_at": "2026-09-11T20:13:53.488937", + "generation_duration_ms": 1640.657, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T20:13:59.771406", + "started_at": "2026-09-11T20:13:55.133594", + "generation_duration_ms": 4597.812, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T20:14:03.986747", + "started_at": "2026-09-11T20:13:59.771406", + "generation_duration_ms": 4171.340999999999, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T20:14:07.755801", + "started_at": "2026-09-11T20:14:03.986747", + "generation_duration_ms": 1705.054, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T20:14:10.259492", + "started_at": "2026-09-11T20:14:07.755801", + "generation_duration_ms": 2501.691, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T20:14:12.758747", + "started_at": "2026-09-11T20:14:10.259492", + "generation_duration_ms": 2499.2549999999997, + "role": "assistant", + "parent_tool_use_id": null + } + ], + "commands": [ + { + "tool_name": "TodoWrite", + "execution_completed_at": "2026-09-11T20:13:55.119000", + "execution_started_at": "2026-09-11T20:13:55.115000", + "tool_id": "toolu_01WfZGLMB5HLRS9E15jLYJq8", + "result_status": "success", + "duration_ms": 4.0 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T20:13:56.979000", + "execution_started_at": "2026-09-11T20:13:56.968000", + "tool_id": "toolu_01WLeJLQCihZjNtCgwCi1xpL", + "result_status": "success", + "duration_ms": 11.0 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T20:13:57.684000", + "execution_started_at": "2026-09-11T20:13:57.678000", + "tool_id": "toolu_019tjDMgudrqyntWUAh8Aupy", + "result_status": "success", + "duration_ms": 6.0 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T20:13:58.385000", + "execution_started_at": "2026-09-11T20:13:58.376000", + "tool_id": "toolu_013J2dP6wxbm4q1acY5PXr6s", + "result_status": "success", + "duration_ms": 9.0 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T20:13:59.080000", + "execution_started_at": "2026-09-11T20:13:59.075000", + "tool_id": "toolu_01DZAPiHZrcfEiDeUDVdur6H", + "result_status": "success", + "duration_ms": 5.0 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T20:13:59.760000", + "execution_started_at": "2026-09-11T20:13:59.751000", + "tool_id": "toolu_01S2Kd2KUueP11tuBZ3hX53j", + "result_status": "success", + "duration_ms": 9.0 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T20:14:01.518000", + "execution_started_at": "2026-09-11T20:14:01.506000", + "tool_id": "toolu_01RbNdCyWmL3XcWUZ1BZBMVs", + "result_status": "success", + "duration_ms": 12.0 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T20:14:02.089000", + "execution_started_at": "2026-09-11T20:14:02.079000", + "tool_id": "toolu_01SDGHLgG9HR9aCieXd81Wz3", + "result_status": "success", + "duration_ms": 10.0 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T20:14:02.660000", + "execution_started_at": "2026-09-11T20:14:02.654000", + "tool_id": "toolu_01XER7fGNDxwMaaucfPhKS7C", + "result_status": "success", + "duration_ms": 6.0 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T20:14:03.239000", + "execution_started_at": "2026-09-11T20:14:03.232000", + "tool_id": "toolu_01TFRCQq7bYkiBwHvSpc3Cxr", + "result_status": "success", + "duration_ms": 7.0 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T20:14:03.974000", + "execution_started_at": "2026-09-11T20:14:03.965000", + "tool_id": "toolu_01MHcPMKPGMihm8MkmhAWBtu", + "result_status": "success", + "duration_ms": 9.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T20:14:06.181000", + "execution_started_at": "2026-09-11T20:14:06.125000", + "tool_id": "toolu_014B6sD12tnYcpCAHbC7YJbS", + "result_status": "success", + "duration_ms": 56.0 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T20:14:07.745000", + "execution_started_at": "2026-09-11T20:14:05.681000", + "tool_id": "toolu_01KCeAUpoHtjG2ssT85e4o9c", + "result_status": "success", + "duration_ms": 2064.0 + }, + { + "tool_name": "TodoWrite", + "execution_completed_at": "2026-09-11T20:14:10.249000", + "execution_started_at": "2026-09-11T20:14:10.247000", + "tool_id": "toolu_01GMeN29huU3tN1BHpusugFh", + "result_status": "success", + "duration_ms": 2.0 + } + ] + } + ] +} diff --git a/tests/_fixtures/timing_runs/pi.json b/tests/_fixtures/timing_runs/pi.json new file mode 100644 index 000000000..e6a2a5924 --- /dev/null +++ b/tests/_fixtures/timing_runs/pi.json @@ -0,0 +1,141 @@ +{ + "_comment": "Scrubbed representative run for the timing residual report. Prompts, outputs, tokens and cost are removed; only the wall-clock fields decompose_run.py reads are kept. NOT a before/after instrument \u2014 see the header in this directory.", + "agent_type": "pi", + "iterations": [ + { + "duration_seconds": 14.706855208845809, + "iteration": 1, + "harness_startup_ms": 344.118, + "harness_teardown_ms": 22.406, + "crashed": false, + "messages": [ + { + "completed_at": "2026-09-11T07:42:08.410088", + "started_at": "2026-09-11T07:42:03.676385", + "generation_duration_ms": 4716.634, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:42:10.831543", + "started_at": "2026-09-11T07:42:08.410734", + "generation_duration_ms": 2409.315, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:42:14.599514", + "started_at": "2026-09-11T07:42:10.831791", + "generation_duration_ms": 1718.625, + "role": "assistant", + "parent_tool_use_id": null + }, + { + "completed_at": "2026-09-11T07:42:18.016455", + "started_at": "2026-09-11T07:42:14.600077", + "generation_duration_ms": 3416.3779999999997, + "role": "assistant", + "parent_tool_use_id": null + } + ], + "commands": [ + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:42:08.406411", + "execution_started_at": "2026-09-11T07:42:08.389754", + "tool_id": "toolu_016h4AfvKbh7JeGEBNGX9XXf", + "result_status": "success", + "duration_ms": 16.657 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:42:08.405764", + "execution_started_at": "2026-09-11T07:42:08.398691", + "tool_id": "toolu_01PBQpT5a3TeTqKhkyNP8x2t", + "result_status": "success", + "duration_ms": 7.073 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:42:08.406823", + "execution_started_at": "2026-09-11T07:42:08.399067", + "tool_id": "toolu_01G3VRUbZC8WUetG97MvmRML", + "result_status": "success", + "duration_ms": 7.756 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:42:08.406580", + "execution_started_at": "2026-09-11T07:42:08.399201", + "tool_id": "toolu_01C9J9jQya5aZZQvvARyi2Cn", + "result_status": "success", + "duration_ms": 7.3790000000000004 + }, + { + "tool_name": "Write", + "execution_completed_at": "2026-09-11T07:42:08.406723", + "execution_started_at": "2026-09-11T07:42:08.399309", + "tool_id": "toolu_01DitFddEYKRKVBhSzeeGqZB", + "result_status": "success", + "duration_ms": 7.414 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:42:10.824862", + "execution_started_at": "2026-09-11T07:42:10.813486", + "tool_id": "toolu_0171JeX8QfzobMv3YZxqafuM", + "result_status": "success", + "duration_ms": 11.376000000000001 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:42:10.824535", + "execution_started_at": "2026-09-11T07:42:10.814175", + "tool_id": "toolu_01PoWyWCXneLVWEJoBbRTiYQ", + "result_status": "success", + "duration_ms": 10.36 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:42:10.822806", + "execution_started_at": "2026-09-11T07:42:10.814507", + "tool_id": "toolu_01NVqJypM1gJY6To9KfihgJL", + "result_status": "success", + "duration_ms": 8.299000000000001 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:42:10.823747", + "execution_started_at": "2026-09-11T07:42:10.815282", + "tool_id": "toolu_01MgTWDsatAnY18osCK4qTTr", + "result_status": "success", + "duration_ms": 8.465 + }, + { + "tool_name": "Read", + "execution_completed_at": "2026-09-11T07:42:10.824980", + "execution_started_at": "2026-09-11T07:42:10.815562", + "tool_id": "toolu_01ACtUYSTWuGSkr34qH2Lvqd", + "result_status": "success", + "duration_ms": 9.418 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:42:14.598034", + "execution_started_at": "2026-09-11T07:42:12.548936", + "tool_id": "toolu_01SGnPjtpNCtQnCrxwKLvqQ4", + "result_status": "success", + "duration_ms": 2049.098 + }, + { + "tool_name": "Bash", + "execution_completed_at": "2026-09-11T07:42:12.626587", + "execution_started_at": "2026-09-11T07:42:12.549868", + "tool_id": "toolu_016kzccEcHbwwo1P9USYsXKM", + "result_status": "success", + "duration_ms": 76.719 + } + ] + } + ] +} diff --git a/tests/lint/rules/ce063_no_busy_ms_in_agents.py b/tests/lint/rules/ce063_no_busy_ms_in_agents.py new file mode 100644 index 000000000..6a6d901a2 --- /dev/null +++ b/tests/lint/rules/ce063_no_busy_ms_in_agents.py @@ -0,0 +1,105 @@ +"""CE063: a reducer may not compute its own tool subtraction. + +Tool execution comes out of a generation window in exactly ONE place: +``coder_eval.streaming.collector.subtract_tool_time``. Before that, five +reducers each did it themselves — four through ``close_window`` as they +flushed, claude-code once at finalization — while the head and the tail were +already computed centrally at the collector seam. That asymmetry is where every +timing defect on this branch actually lived, and none of them was in the +arithmetic: they were in the bookkeeping AROUND it. When to reset a per-step +span list (clearing it at ``step_start`` wiped a span before the flush could +subtract it — a 100% overstatement of that window). When to clear a spent start +stamp (a second flush with no intervening start republished the previous span — +3000 ms of generation for a 2000 ms turn). When to advance the mark. + +A sixth harness whose author reaches for ``busy_ms`` is rebuilding exactly that +bookkeeping, and its tool time would then be subtracted TWICE: once by the +reducer and once by the collector, which subtracts from every window it is +handed. The result is a silently under-reported generation figure on one +harness only — the shape that takes a corpus comparison to notice. + +Separate id from CE061 deliberately, and CE061 is NOT rebodied into this. +CE061 asks where a window's ARITHMETIC came from, and four reducers still call +``close_window``, so its property is still live and still worth guarding — it +is not superseded. This one asks a different question: whether a reducer +subtracts tool time at all. One invariant per id is what makes a ``# noqa`` +mean one thing. (Phase 5 did make CE061 exemption-free: claude-code now calls +the shrunken ``close_window`` like the other four, so its one permanent +suppression is gone.) + +WHY NOT ``_imports_the_helper``, which CE061 uses. That function deliberately +returns True for a bare module import (``from coder_eval import timing``), so +that ``timing.close_window(...)`` counts as reaching the helper — its own +comment says a rule that missed it "would tell an author to change a working +call site." Inverted into a BAN that branch flags any reducer importing the +module and calling ``timing.close_window(...)``, which after Phase 5 is four of +them. So this rule keys on the ``busy_ms`` NAME binding plus an +``ast.Attribute`` match for the ``timing.busy_ms`` spelling, and leaves the +module import alone. + +The name is taken from the function object rather than written here as a +string, the way CE061 takes ``close_window``: renaming it moves this rule too. + +BLIND SPOT: a reducer that re-implements the union inline, without importing +anything, is invisible — as is one reaching ``busy_ms`` through a re-export. +The sensor for the arithmetic itself is +``tests/test_timing_identity_contract.py``, which drives every harness off a +scripted clock and asserts the four buckets tile the turn to the millisecond; +this rule adds only the cheap structural half that a static check can reach. +""" + +import ast + +from coder_eval.timing import busy_ms +from tests.lint.rules._model_ctor import AGENTS_ROOT +from tests.lint.rules.base import BaseRule + + +_TIMING_MODULE = "coder_eval.timing" +_TIMING_TAIL = _TIMING_MODULE.rpartition(".")[2] + +# Taken from the function, never spelled here: a rename then moves the rule too. +_BANNED = busy_ms.__name__ + +_MESSAGE = ( + f"imports '{_BANNED}', but a reducer does not subtract tool time any more — " + "coder_eval.streaming.collector.subtract_tool_time does it once, for every harness, " + "at the single capture seam. Publish the RAW window (close_window gives you its bounds " + "and span) and let the collector clip the tool union out of it. Subtracting here too " + "takes it out twice and silently under-reports generation on this harness alone." +) + + +class NoBusyMsInAgents(BaseRule): + id = "CE063" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(AGENTS_ROOT.search(filepath)) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + """`from coder_eval.timing import busy_ms`, under any alias. + + Relative forms (`from ..timing import busy_ms`) count too: the module + is the same one whatever the path to it looks like. + """ + if not self._in_scope: + return + module = node.module or "" + reaches = module.startswith(_TIMING_MODULE) or ( + bool(node.level) and (module == _TIMING_TAIL or module.startswith(f"{_TIMING_TAIL}.")) + ) + if reaches and any(alias.name == _BANNED for alias in node.names): + self.violation(node, _MESSAGE) + + def visit_Attribute(self, node: ast.Attribute) -> None: + """The `timing.busy_ms` spelling. + + Defensive: no reducer uses it today (all five import plain names), but + a name-binding check alone would let it through, and it is one arm. + """ + if not self._in_scope: + return + if node.attr == _BANNED and isinstance(node.value, ast.Name) and node.value.id == _TIMING_TAIL: + self.violation(node, _MESSAGE) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 791275091..a79bd0faa 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -39,6 +39,7 @@ from tests.lint.rules.ce059_generation_window_is_two_reads import GenerationWindowIsTwoReads from tests.lint.rules.ce060_message_id_declared import MessageIdDeclared from tests.lint.rules.ce061_window_via_close_window import WindowViaCloseWindow +from tests.lint.rules.ce063_no_busy_ms_in_agents import NoBusyMsInAgents from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -101,6 +102,7 @@ GenerationWindowIsTwoReads, MessageIdDeclared, WindowViaCloseWindow, + NoBusyMsInAgents, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index e437ff66a..d5f017c6c 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -62,6 +62,15 @@ # subtraction in codex_agent._flush_message. "codex_d_cross_flush_is_error", # flush lands before the tool completes: zero-width window "codex_e_orphan_tool", # the tool never completes, so the window never opens + # Same shape, reached from the opposite direction. This scenario injects + # a 5 ms CLI tool interval into a replay whose whole turn is well under + # one millisecond, so the tool spans BOTH windows entirely and the + # central subtraction takes each down to a measured 0.0. It is the tool + # interval that is fictional, not the subtraction — which is why the + # scenario is also in FICTIONAL_DURATIONS. Its point is the TILING (the + # second window opens at the first `step_finish`), and the snapshot + # still records that. + "opencode_c_multi_step_tiling", } ) diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py index b79e06818..90f013c44 100644 --- a/tests/test_agent_telemetry.py +++ b/tests/test_agent_telemetry.py @@ -1457,13 +1457,11 @@ def test_seeding_twice_by_hand_is_a_no_op_the_second_time(self, monkeypatch): clock.at_ms = 800 state._seed_first_generation_window() seeded_wall = state.last_event_wall - seeded_monotonic = state.last_event_monotonic clock.at_ms = 5000 state._seed_first_generation_window() assert state.last_event_wall == seeded_wall - assert state.last_event_monotonic == seeded_monotonic def test_a_stream_with_no_message_start_still_clamps_to_zero(self, monkeypatch): """Partial streaming off, a mocked query(), or a crash before the first event. diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index e25db8637..e15b1dee2 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -2302,7 +2302,7 @@ async def test_generation_plus_tool_exec_does_not_exceed_the_window(self): assert gen_ms + tool_ms == pytest.approx(window_ms) async def test_the_published_window_reconciles_to_its_own_bounds(self): - """The reducer subtracted exactly the spans the record carries. + """The collector subtracted exactly the spans the record carries. `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell both recompute the tool UNION from the recorded command spans and @@ -2409,11 +2409,12 @@ class TestFlushMessageWindowBounds: """Where `_flush_message`'s window OPENS, driven at the reducer. The end-to-end cases above all describe a stream whose stamps advance, so - they cannot reach the two arguments the reducer hands `close_window` for - the awkward cases: the emission's own first stamp (`item_start`) and the - calls still open at the flush. Both moved from inline code into the shared - helper, so without these they are pinned only in the helper's own unit - tests — the wiring between the two would be free to rot. + they cannot reach the awkward case the reducer still hands `close_window`: + the emission's own first stamp (`item_start`), whose `min()` against the + mark is the backwards-clock defence. The tool-span arguments this class + also used to cover are gone — the subtraction moved to + `EventCollector.subtract_tool_time`, and + `tests/test_event_collector.py::TestSubtractToolTime` pins it there. """ @staticmethod @@ -2466,17 +2467,24 @@ def test_a_mark_later_than_the_first_item_does_not_invert_the_window(self): assert message.started_at == _ms_to_dt(_BOUNDS_EPOCH_MS + 500) assert message.generation_duration_ms == pytest.approx(600.0) - def test_a_call_still_open_at_the_flush_is_subtracted_bounded_at_the_end(self): - # It has no completion yet, so only [start, window end] is not model - # time. Its full interval joins the NEXT window's closed spans, where - # busy_ms clips it to the remainder — subtracted once, not twice. + def test_the_published_window_is_raw_and_ignores_a_call_still_open(self): + """The reducer publishes the RAW span; the collector subtracts. + + It used to bound a still-open call at the window's end and take that + slice out here. `EventCollector.subtract_tool_time` sees every span at + once, so a call is subtracted from the windows its REAL interval + overlaps once it resolves — no boundary approximation, and nothing for + this reducer to remember. A call that never resolves has no + `execution_completed_at` and contributes nothing, which is what "never + timed" should cost. + """ message = self._flush( gen_mark_ms=_BOUNDS_EPOCH_MS, open_start_ms=_BOUNDS_EPOCH_MS, open_end_ms=_BOUNDS_EPOCH_MS + 1000, open_tool_started_ms=_BOUNDS_EPOCH_MS + 700, ) - assert message.generation_duration_ms == pytest.approx(700.0) + assert message.generation_duration_ms == pytest.approx(1000.0) def test_a_call_opening_after_the_window_closes_is_ignored(self): message = self._flush( @@ -2609,3 +2617,99 @@ async def test_the_split_survives_end_to_end_through_communicate(self): # sub-message produced. assert [m.generation_duration_ms for m in assistant] == [400.0, 600.0] assert sum(m.generation_duration_ms or 0.0 for m in assistant) == 1000.0 + + +class TestTwoSpecGenerationContainingATool: + """A thinking+action window holding a tool: the case the split and the + subtraction have to survive TOGETHER. + + Codex is the only harness that cuts one window into several messages, and + a generation that calls a tool is a two-spec window by construction — the + thinking block plus the tool_use. `TestFlushMessageGenTimeSplit` drives two + specs with no tool; `TestGenerationWindowExcludesToolExecution` on the + other harnesses drives a tool into a single-spec window. Neither reaches + the interaction, which is where grouping by bounds earns its keep: subtract + per message and the overlap comes out twice, and the parts stop summing to + the window. + """ + + @staticmethod + def _published(*, window_ms: int, tool_from_ms: int, tool_to_ms: int, think_out: int, action_out: int): + from coder_eval.agents.codex_agent import _CodexTurnState, _ms_to_dt + from coder_eval.models import CommandTelemetry, ContentBlock, TokenUsage + from coder_eval.streaming.callbacks import CompositeStreamCallback + from coder_eval.streaming.collector import EventCollector + from coder_eval.streaming.events import AgentEndEvent, AgentEndStatus, AgentStartEvent, ToolEndEvent + + agent = CodexAgent(parse_agent_config(type=AgentKind.CODEX, model="gpt-5.5")) + collector = EventCollector() + state = _CodexTurnState( + agent, + emit=CompositeStreamCallback([collector]), + task_id="codex", + turn_id="codex-1", + collector=collector, + commands=[], + messages=[], + user_input="go", + iteration=1, + turn_start_time=0.0, + ) + command = CommandTelemetry( + tool_name="bash", + tool_id="c1", + timestamp=_ms_to_dt(_BOUNDS_EPOCH_MS + tool_from_ms), + execution_started_at=_ms_to_dt(_BOUNDS_EPOCH_MS + tool_from_ms), + execution_completed_at=_ms_to_dt(_BOUNDS_EPOCH_MS + tool_to_ms), + result_status="success", + ) + state.commands.append(command) + state.open_blocks = [ + ContentBlock(block_type="thinking", sequence=0, thinking="plan"), + ContentBlock(block_type="tool_use", sequence=0, tool_use_id="c1"), + ] + state.open_start_ms = _BOUNDS_EPOCH_MS + state.open_end_ms = _BOUNDS_EPOCH_MS + window_ms + state._flush_message( + SimpleNamespace( + input_tokens=100, + cached_input_tokens=0, + output_tokens=think_out + action_out, + reasoning_output_tokens=think_out, + ) + ) + + collector.on_event( + AgentStartEvent(task_id="codex", prompt="go", iteration=1, timestamp=_ms_to_dt(_BOUNDS_EPOCH_MS)) + ) + collector.on_event(ToolEndEvent(task_id="codex", turn_id="codex-1", tool=command)) + collector.on_event( + AgentEndEvent( + task_id="codex", + status=AgentEndStatus.COMPLETED, + messages=list(state.messages), + usage=TokenUsage(), + timestamp=_ms_to_dt(_BOUNDS_EPOCH_MS + window_ms), + ) + ) + record = collector.build_turn_record() + return [m for m in record.messages if m.role == "assistant"] + + def test_the_group_is_subtracted_once_and_the_parts_still_sum(self): + # A 1000 ms window, split 80/20 by output share, holding a 250 ms tool. + published = self._published(window_ms=1000, tool_from_ms=300, tool_to_ms=550, think_out=800, action_out=200) + assert len(published) == 2, "a thinking + tool_use window is two sub-messages" + total = sum(m.generation_duration_ms or 0.0 for m in published) + # ONCE: 1000 - 250. Subtracting per message would give 500. + assert total == pytest.approx(750.0) + assert [m.generation_duration_ms for m in published] == [pytest.approx(600.0), pytest.approx(150.0)] + + def test_both_sub_messages_still_share_one_window(self): + """The bounds are what the grouping keys on, so they must stay identical.""" + published = self._published(window_ms=1000, tool_from_ms=300, tool_to_ms=550, think_out=800, action_out=200) + assert published[0].started_at == published[1].started_at + assert published[0].completed_at == published[1].completed_at + + def test_a_window_entirely_covered_by_its_tool_splits_zero_two_ways(self): + published = self._published(window_ms=1000, tool_from_ms=0, tool_to_ms=1000, think_out=800, action_out=200) + assert [m.generation_duration_ms for m in published] == [0.0, 0.0] diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 4431d2a3b..2183098c6 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4034,11 +4034,17 @@ def test_the_real_agents_tree_is_clean(self): found = [v for path in sorted(root.glob("*.py")) for v in check_file(path, [WindowViaCloseWindow])] assert not found, found - def test_each_suppression_is_load_bearing(self): - # A noqa nobody needs is a noqa that outlives its reason, so the set is - # pinned rather than merely non-empty. It has already earned that: - # antigravity carried a TEMPORARY suppression until it moved onto - # `close_window`, and this test is what failed when the reason expired. + def test_the_rule_is_now_exemption_free(self): + """No reducer needs a `# noqa: CE061` any more, and the set is PINNED empty. + + A noqa nobody needs is a noqa that outlives its reason, so this asserts + the exact set rather than merely that it shrank. It has earned that + twice: antigravity carried a TEMPORARY suppression until it moved onto + `close_window`, and claude-code carried a permanent one until the tool + subtraction moved to `EventCollector.subtract_tool_time` — at which + point it could call the same shrunken helper as the other four. This + test is what failed each time the reason expired. + """ import ast import pathlib @@ -4050,7 +4056,7 @@ def test_each_suppression_is_load_bearing(self): for path in sorted(root.glob("*.py")) if WindowViaCloseWindow(str(path)).check(ast.parse(path.read_text(encoding="utf-8"))) } - assert suppressed == {"claude_code_agent.py"} + assert suppressed == set() class TestRuffExternalCoversEveryRule: @@ -4662,3 +4668,70 @@ def test_the_real_antigravity_flush_declares_its_id(self): path = SRC / "coder_eval/agents/antigravity_agent.py" assert path.is_file(), "the fixture file must exist or this test passes vacuously" assert not [v for v in check_file(path) if v.rule_id == "CE060"] + + +class TestCE063NoBusyMsInAgents: + """CE063 flags a reducer that would subtract tool time itself. + + The subtraction lives once, in + `coder_eval.streaming.collector.subtract_tool_time`. A reducer that also + does it has its tool time taken out TWICE — once by itself, once by the + collector — which under-reports generation on that harness alone. + """ + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/agents/pi_agent.py"): + import ast + + from tests.lint.rules.ce063_no_busy_ms_in_agents import NoBusyMsInAgents + + return NoBusyMsInAgents(filepath).check(ast.parse(src)) + + def test_flags_the_bare_name_import(self): + assert len(self._run("from coder_eval.timing import busy_ms")) == 1 + + def test_flags_it_under_an_alias(self): + # The import is what is banned, whatever it is bound to. + assert len(self._run("from coder_eval.timing import busy_ms as union")) == 1 + + def test_flags_it_alongside_an_allowed_import(self): + assert len(self._run("from coder_eval.timing import busy_ms, close_window")) == 1 + + def test_flags_a_relative_import(self): + # `agents/` uses relative imports; matching only the absolute path + # would leave the rule blind for a whole file. + assert len(self._run("from ..timing import busy_ms")) == 1 + + def test_flags_the_module_attribute_spelling(self): + assert len(self._run("from coder_eval import timing\nx = timing.busy_ms(s, lo, hi)")) == 1 + + def test_does_not_fire_on_close_window_through_the_module(self): + """The exact false positive a naive inversion of CE061's resolver gives. + + `_imports_the_helper` returns True for a bare module import so that + `timing.close_window(...)` counts as reaching the helper. Inverted into + a ban, that branch flags every reducer importing the module — which + after the migration is four of the five. + """ + assert not self._run("from coder_eval import timing\nx = timing.close_window(mark=m, now=n)") + + def test_does_not_fire_on_close_window_by_name(self): + assert not self._run("from coder_eval.timing import close_window\nx = close_window(mark=m, now=n)") + + def test_does_not_fire_on_an_unrelated_attribute_named_busy_ms(self): + # `self.busy_ms` is not `timing.busy_ms`; only the module spelling counts. + assert not self._run("x = self.busy_ms") + + def test_does_not_fire_outside_agents(self): + # The collector is where the subtraction belongs, so it must import it. + assert not self._run("from coder_eval.timing import busy_ms", filepath="src/coder_eval/streaming/collector.py") + + def test_is_suppressible(self, tmp_path): + from tests.lint.rules.ce063_no_busy_ms_in_agents import NoBusyMsInAgents + from tests.lint.runner import check_file + + agents = tmp_path / "src" / "coder_eval" / "agents" + agents.mkdir(parents=True) + target = agents / "pi_agent.py" + target.write_text("from coder_eval.timing import busy_ms # noqa: CE063\n", encoding="utf-8") + assert not check_file(target, [NoBusyMsInAgents]) diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 6b655eae0..10aab05cc 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -5,7 +5,7 @@ rules in ``coder_eval/streaming/collector.py``. """ -from datetime import datetime +from datetime import datetime, timedelta from typing import ClassVar import pytest @@ -18,13 +18,15 @@ TokenUsage, TurnRecord, ) -from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.collector import EventCollector, subtract_tool_time from coder_eval.streaming.events import ( AgentEndEvent, + AgentEndStatus, AgentStartEvent, ToolEndEvent, TurnStartEvent, ) +from coder_eval.timing import union_ms TASK_ID = "collector-test" @@ -725,3 +727,298 @@ def test_a_new_turn_clears_the_previous_turn_terminal_event(self): rec = collector.build_turn_record() assert rec.harness_startup_ms is None assert rec.harness_teardown_ms is None + + +class TestSubtractToolTime: + """The ONE tool subtraction, moved here from five reducers. + + Four of them did it inside `close_window` as they flushed; claude-code did + it once at finalization. Head and tail were already computed centrally, in + this module — that asymmetry was the complexity, and every timing defect + this branch fixed lived in the per-reducer bookkeeping around the + subtraction rather than in the subtraction itself. + """ + + BASE: ClassVar[datetime] = datetime(2026, 9, 11, 9, 0, 0) + + @classmethod + def _at(cls, ms: float) -> datetime: + return cls.BASE + timedelta(milliseconds=ms) + + @classmethod + def _msg(cls, lo: float, hi: float, gen: float | None, **kwargs) -> AssistantMessage: + return AssistantMessage(started_at=cls._at(lo), completed_at=cls._at(hi), generation_duration_ms=gen, **kwargs) + + def test_a_contained_tool_is_subtracted_exactly_once(self): + out = subtract_tool_time([self._msg(0, 1000, 1000.0)], [(self._at(200), self._at(700))]) + assert out[0].generation_duration_ms == pytest.approx(500.0) + + def test_the_input_messages_are_not_mutated(self): + """Non-mutating because of ALIASING, not because of repeated calls. + + Every agent builds its terminal event as + `AgentEndEvent(messages=list(...))`, which copies the LIST and not the + message objects — so an in-place write would reach back into the + agent's own live state from the collector. + """ + messages = [self._msg(0, 1000, 1000.0)] + subtract_tool_time(messages, [(self._at(200), self._at(700))]) + assert messages[0].generation_duration_ms == pytest.approx(1000.0) + + def test_a_group_sharing_bounds_is_subtracted_once_and_the_parts_still_sum(self): + """Codex splits one window across two sub-messages by output share. + + Subtracting the group's overlap from each part separately would take it + twice and stop the parts summing to the window. Grouping is on the + BOUNDS, not on `message_id` — OpenCode and Pi can carry `None` there. + """ + # A 1000 ms window split 25/75, with a 250 ms tool inside it. + out = subtract_tool_time( + [self._msg(0, 1000, 250.0, message_id="m"), self._msg(0, 1000, 750.0, message_id="m")], + [(self._at(300), self._at(550))], + ) + assert [m.generation_duration_ms for m in out] == [pytest.approx(187.5), pytest.approx(562.5)] + assert sum(m.generation_duration_ms or 0.0 for m in out) == pytest.approx(750.0) + + def test_a_group_with_no_message_id_is_still_grouped_by_its_bounds(self): + """The case keying on `message_id` would break. + + Two id-less messages sharing a window must be one group; keying on the + id would instead collapse every id-less message of the turn into one. + """ + out = subtract_tool_time( + [self._msg(0, 1000, 500.0), self._msg(0, 1000, 500.0), self._msg(2000, 3000, 1000.0)], + [(self._at(200), self._at(400))], + ) + assert sum(m.generation_duration_ms or 0.0 for m in out[:2]) == pytest.approx(800.0) + assert out[2].generation_duration_ms == pytest.approx(1000.0), "a different window is a different group" + + def test_concurrent_tools_subtract_their_union_not_their_sum(self): + """Summing would clamp a real generation to zero. + + The expectation is DERIVED from `union_ms` rather than written as a + literal, so this cannot drift from the rule the rest of the codebase + applies — and the sum is asserted separately to be the wrong answer. + """ + spans = [ + (self._at(100), self._at(500)), + (self._at(150), self._at(550)), + (self._at(200), self._at(600)), + (self._at(250), self._at(650)), + ] + out = subtract_tool_time([self._msg(0, 1000, 1000.0)], spans) + assert out[0].generation_duration_ms == pytest.approx(1000.0 - union_ms(spans)) + assert sum((e - s).total_seconds() * 1000.0 for s, e in spans) > 1000.0, ( + "the fixture must actually over-subtract when summed, or this proves nothing" + ) + assert out[0].generation_duration_ms > 0.0 + + def test_a_window_entirely_covered_by_tools_is_a_measured_zero(self): + out = subtract_tool_time([self._msg(0, 1000, 1000.0)], [(self._at(0), self._at(1000))]) + assert out[0].generation_duration_ms == 0.0, "a measurement, not an absence" + + def test_a_none_duration_stays_none(self): + """`None` means no window was ever measured, and CE058 keeps it distinct.""" + out = subtract_tool_time([self._msg(0, 1000, None)], [(self._at(0), self._at(500))]) + assert out[0].generation_duration_ms is None + + def test_a_zero_group_does_not_divide_by_zero(self): + out = subtract_tool_time([self._msg(0, 1000, 0.0)], [(self._at(0), self._at(500))]) + assert out[0].generation_duration_ms == 0.0 + + def test_a_sub_agent_generation_is_skipped(self): + """Its own tools are not in this span set, and the spawning Agent call + already covers its whole run.""" + out = subtract_tool_time([self._msg(0, 1000, 900.0, parent_tool_use_id="t1")], [(self._at(0), self._at(500))]) + assert out[0].generation_duration_ms == pytest.approx(900.0) + + def test_non_assistant_entries_pass_through_by_identity(self): + reconciliation = ReconciliationMessage( + input_tokens=1, output_tokens=1, cache_creation_tokens=0, cache_read_tokens=0, note="n" + ) + out = subtract_tool_time([self._msg(0, 1000, 1000.0), reconciliation], [(self._at(0), self._at(200))]) + assert out[1] is reconciliation + + +class TestBuildTurnRecordIsIdempotent: + """Building the record twice must give the same numbers. + + `EventCollector` is not built once and read once. `EarlyStopWatcher` holds + ONE across a turn's tool-call rounds and calls `build_turn_record()` on + every one, and the crash path builds it again from `Agent._finalize`. The + tool subtraction now happens inside that method, so a version of it that + mutated would subtract again on every call — and the numbers would depend + on how many times something happened to look. + """ + + BASE: ClassVar[datetime] = datetime(2026, 9, 11, 9, 0, 0) + + def _collector(self) -> EventCollector: + at = lambda ms: self.BASE + timedelta(milliseconds=ms) # noqa: E731 + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=at(0))) + collector.on_event( + ToolEndEvent( + task_id="t", + turn_id="t1", + tool=CommandTelemetry( + tool_name="bash", + tool_id="c1", + timestamp=at(700), + execution_started_at=at(700), + execution_completed_at=at(1200), + result_status="success", + ), + ) + ) + collector.on_event( + AgentEndEvent( + task_id="t", + status=AgentEndStatus.COMPLETED, + messages=[ + AssistantMessage( + started_at=at(500), completed_at=at(2000), generation_duration_ms=1500.0, output_tokens=5 + ) + ], + usage=TokenUsage(output_tokens=5), + timestamp=at(2500), + ) + ) + return collector + + def test_two_builds_agree_on_every_timing_figure(self): + collector = self._collector() + first, second = collector.build_turn_record(), collector.build_turn_record() + + assert [m.generation_duration_ms for m in first.messages if m.role == "assistant"] == [ + m.generation_duration_ms for m in second.messages if m.role == "assistant" + ] + assert first.harness_startup_ms == second.harness_startup_ms + assert first.harness_teardown_ms == second.harness_teardown_ms + + def test_the_first_build_already_subtracted_once(self): + """Guards the other direction: identical-but-wrong would also pass above.""" + record = self._collector().build_turn_record() + generation = [m.generation_duration_ms for m in record.messages if m.role == "assistant"] + # A 1500 ms window holding a 500 ms tool. + assert generation == [pytest.approx(1000.0)] + + def test_the_agents_own_message_objects_are_not_written_through(self): + """The aliasing case, which is the real reason for `model_copy`. + + `AgentEndEvent(messages=list(...))` copies the LIST, not the messages, + so the objects the collector receives are the agent's own live state. + """ + at = lambda ms: self.BASE + timedelta(milliseconds=ms) # noqa: E731 + message = AssistantMessage(started_at=at(0), completed_at=at(1000), generation_duration_ms=1000.0) + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=at(0))) + collector.on_event( + ToolEndEvent( + task_id="t", + turn_id="t1", + tool=CommandTelemetry( + tool_name="bash", + tool_id="c1", + timestamp=at(200), + execution_started_at=at(200), + execution_completed_at=at(700), + result_status="success", + ), + ) + ) + collector.on_event( + AgentEndEvent(task_id="t", status=AgentEndStatus.COMPLETED, messages=[message], timestamp=at(1000)) + ) + collector.build_turn_record() + + assert message.generation_duration_ms == pytest.approx(1000.0), "the agent's own object must be untouched" + + +class TestOverheadExcludesSubAgentTools: + """The head and tail are bracketed on the MAIN thread, commands included. + + `_overhead_ms` filtered its GENERATIONS to the main thread and then passed + EVERY command as a tool span, so its own claim to keep all four buckets + measuring one thread was true only by luck: a child nests inside the parent + Agent call, whose interval the union already covers. Codex's recovered + child tools carry the CHILD's clock, so nothing made it true by + construction — and the evalboard's twin DOES filter, so the two agreed by + accident. + """ + + BASE: ClassVar[datetime] = datetime(2026, 9, 11, 9, 0, 0) + + def test_a_sub_agent_tool_inside_the_head_does_not_shrink_it(self): + at = lambda ms: self.BASE + timedelta(milliseconds=ms) # noqa: E731 + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=at(0))) + # A sub-agent's own tool call, sitting inside what is otherwise head. + collector.on_event( + ToolEndEvent( + task_id="t", + turn_id="t1", + tool=CommandTelemetry( + tool_name="Bash", + tool_id="child-1", + timestamp=at(100), + execution_started_at=at(100), + execution_completed_at=at(400), + result_status="success", + ), + ) + ) + collector.on_event( + AgentEndEvent( + task_id="t", + status=AgentEndStatus.COMPLETED, + messages=[ + AssistantMessage(started_at=at(500), completed_at=at(1000), generation_duration_ms=500.0), + # The child generation that OWNS child-1. + AssistantMessage( + started_at=at(100), + completed_at=at(400), + generation_duration_ms=300.0, + parent_tool_use_id="agent-call", + tool_use_ids=["child-1"], + ), + ], + timestamp=at(1500), + ) + ) + record = collector.build_turn_record() + + # 500 ms of head, all of it. Counting the child's tool would book 300 ms + # of it as tool execution that no main-thread bucket claims. + assert record.harness_startup_ms == pytest.approx(500.0) + assert record.harness_teardown_ms == pytest.approx(500.0) + + def test_a_main_thread_tool_inside_the_head_still_shrinks_it(self): + """The control: the filter must exclude children, not all commands.""" + at = lambda ms: self.BASE + timedelta(milliseconds=ms) # noqa: E731 + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=at(0))) + collector.on_event( + ToolEndEvent( + task_id="t", + turn_id="t1", + tool=CommandTelemetry( + tool_name="Bash", + tool_id="main-1", + timestamp=at(100), + execution_started_at=at(100), + execution_completed_at=at(400), + result_status="success", + ), + ) + ) + collector.on_event( + AgentEndEvent( + task_id="t", + status=AgentEndStatus.COMPLETED, + messages=[AssistantMessage(started_at=at(500), completed_at=at(1000), generation_duration_ms=500.0)], + timestamp=at(1500), + ) + ) + record = collector.build_turn_record() + assert record.harness_startup_ms == pytest.approx(200.0), "500 ms of head minus a 300 ms tool" diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 1d2f9e762..605563fab 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -32,8 +32,9 @@ _unwrap, ) from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.models import AssistantMessage, CommandTelemetry, OpenCodeAgentConfig, PermissionMode +from coder_eval.models import AssistantMessage, CommandTelemetry, OpenCodeAgentConfig, PermissionMode, TokenUsage from coder_eval.pricing import calculate_cost +from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -1784,24 +1785,35 @@ def test_orphan_result_is_never_dropped(self): class TestGenerationWindowExcludesToolExecution: - """A tool running inside a step is not model time. - - OpenCode marks the window at `step_start` and closes it at - `step_finish`, and every tool call executes INSIDE it while also - publishing its own measured `duration_ms`. Publishing the raw span as - generation time counted the same milliseconds twice, which the task - page's Unaccounted cell renders as a ~-100% residual. - - Driven at the reducer rather than through `communicate()`: the window is - two `datetime.now()` reads and the tool interval comes from the event - payload, so only setting both explicitly makes the arithmetic - deterministic. + """A tool running inside a step is not model time — asserted where it is now DECIDED. + + The reducer no longer subtracts anything. It publishes the RAW window, and + `EventCollector.subtract_tool_time` takes the tool union back out of it + once, for all five harnesses. So these cases drive the reducer and then a + real collector, and assert the PUBLISHED number — the one that reaches + `task.json` — rather than an intermediate the reducer used to own. + + They are not duplicates of + `tests/test_event_collector.py::TestSubtractToolTime`: those pin the + arithmetic, these pin that THIS reducer hands the collector a window and a + span set the arithmetic can be right about. """ WINDOW_START = datetime(2026, 1, 1, 12, 0, 0) WINDOW_END = datetime(2026, 1, 1, 12, 0, 1) # a 1000ms step def _finish_step(self, monkeypatch, spans, open_starts=()): + """Drive the reducer, then publish through a real collector. + + `spans` are RESOLVED calls (both bounds); `open_starts` are calls that + never returned. An unresolved call now contributes NO span — it has no + `execution_completed_at`, and inventing one is what `None` exists to + prevent — where the reducer used to bound it at the window's end. That + is a real change and a better one: the collector sees every span at + once, so a call straddling a boundary is clipped to each window it + actually overlapped instead of approximated at the boundary. + """ + class _Clock(datetime): @staticmethod def now(tz=None): @@ -1809,37 +1821,56 @@ def now(tz=None): state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="do it", model="deepseek/deepseek-v4-pro") state.step_started_at = self.WINDOW_START - state.step_tool_spans = list(spans) - for i, started in enumerate(open_starts): - state.open_tools[f"open-{i}"] = CommandTelemetry( + commands = [ + CommandTelemetry( tool_name="bash", - tool_id=f"open-{i}", + tool_id=f"closed-{i}", timestamp=started, execution_started_at=started, + execution_completed_at=completed, + result_status="success", ) + for i, (started, completed) in enumerate(spans) + ] + commands += [ + CommandTelemetry(tool_name="bash", tool_id=f"open-{i}", timestamp=st, execution_started_at=st) + for i, st in enumerate(open_starts) + ] monkeypatch.setattr(agent_module, "datetime", _Clock) state.on_step_finish({"reason": "stop", "tokens": {"input": 100, "output": 20}}) - assistant = [m for m in state.messages if m.role == "assistant"] - assert len(assistant) == 1 - return assistant[0] + + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t1", prompt="do it", iteration=1, timestamp=self.WINDOW_START)) + for command in commands: + collector.on_event(ToolEndEvent(task_id="t1", turn_id="s1", tool=command)) + collector.on_event( + AgentEndEvent( + task_id="t1", + status=AgentEndStatus.COMPLETED, + messages=list(state.messages), + usage=TokenUsage(), + timestamp=self.WINDOW_END, + ) + ) + published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] + assert len(published) == 1 + return published[0] def test_tool_time_inside_the_step_is_subtracted(self, monkeypatch): - # A 500ms tool squarely inside the 1000ms step. message = self._finish_step( monkeypatch, [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], ) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - assert span_ms == pytest.approx(1000.0) + assert span_ms == pytest.approx(1000.0), "the reducer still publishes the whole window as its bounds" assert message.generation_duration_ms == pytest.approx(500.0) def test_a_step_with_no_tools_keeps_its_whole_window(self, monkeypatch): - message = self._finish_step(monkeypatch, []) - assert message.generation_duration_ms == pytest.approx(1000.0) + assert self._finish_step(monkeypatch, []).generation_duration_ms == pytest.approx(1000.0) def test_concurrent_tools_are_subtracted_once(self, monkeypatch): - # Two overlapping 500ms tools occupy 600ms of wall clock, not 1000ms. - # Summing them would leave 0 generation for a step that generated 400. + # Two overlapping 500ms tools occupy 600ms, not 1000ms. Summing them + # would leave 0 generation for a step that generated 400. message = self._finish_step( monkeypatch, [ @@ -1850,34 +1881,30 @@ def test_concurrent_tools_are_subtracted_once(self, monkeypatch): assert message.generation_duration_ms == pytest.approx(400.0) def test_the_window_never_goes_negative(self, monkeypatch): - # A tool whose recorded interval straddles the step is clipped to it. message = self._finish_step( monkeypatch, [(self.WINDOW_START - timedelta(seconds=30), self.WINDOW_END + timedelta(seconds=30))], ) assert message.generation_duration_ms == 0.0 - def test_a_tool_still_open_at_the_boundary_is_subtracted(self, monkeypatch): - # The windows tile from the previous step's finish, so a call that - # opens inside this step and closes inside the NEXT one straddles the - # boundary. Counting only closed intervals published the pre-boundary - # 400ms as generation while the call's own duration_ms counted it - # again — the exact double-count `busy_ms` exists to prevent. - message = self._finish_step( - monkeypatch, - [], - open_starts=[self.WINDOW_START + timedelta(milliseconds=600)], - ) - assert message.generation_duration_ms == pytest.approx(600.0) + def test_a_tool_still_open_at_the_boundary_contributes_no_span(self, monkeypatch): + """The behaviour that CHANGED with the move, stated rather than implied. - def test_an_open_tool_overlapping_a_closed_one_is_counted_once(self, monkeypatch): - # Union, not sum, across the closed and still-open sets alike. + The reducer used to bound a still-open call at the window's end and + subtract that slice. The collector cannot: a call with no + `execution_completed_at` was never timed. Its time is subtracted when it + RESOLVES, from whichever windows its real interval overlaps. + """ + message = self._finish_step(monkeypatch, [], open_starts=[self.WINDOW_START + timedelta(milliseconds=600)]) + assert message.generation_duration_ms == pytest.approx(1000.0) + + def test_a_resolved_tool_overlapping_an_unresolved_one_counts_only_the_resolved(self, monkeypatch): message = self._finish_step( monkeypatch, [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], open_starts=[self.WINDOW_START + timedelta(milliseconds=500)], ) - assert message.generation_duration_ms == pytest.approx(200.0) + assert message.generation_duration_ms == pytest.approx(500.0) def test_a_mark_later_than_the_step_start_does_not_invert_the_window(self, monkeypatch): """The backwards-clock defence, pinned at the reducer, not in isolation. @@ -1886,7 +1913,9 @@ def test_a_mark_later_than_the_step_start_does_not_invert_the_window(self, monke step's own start as `item_start`. Drop that argument and the window opens at the (later) mark instead, so the span shrinks — or inverts and clamps to 0.0, publishing a fabricated instant generation. Nothing else - in this file fails when it is dropped. + in this file fails when it is dropped, which is the whole reason it is + here: the mark is what the reducer still owns after the tool + subtraction moved to the collector. """ state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="do it", model="m") state.step_started_at = self.WINDOW_START @@ -1907,28 +1936,20 @@ def now(tz=None): assert message.generation_duration_ms == pytest.approx(1000.0) def test_the_published_window_reconciles_to_its_own_bounds(self, monkeypatch): - """The reducer subtracted exactly the spans the record carries. + """The collector subtracted exactly the spans the record carries. `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell both recompute the tool UNION from the recorded command spans and - subtract it from the recorded window bounds. This asserts the reducer - fed the window the same set, so a span silently added or dropped on - the way in shows up here. - - It is deliberately the narrow half: `expected` is derived from the - PUBLISHED bounds, so it cannot see a wrong mark, and both sides call - `busy_ms`, so it cannot see a union bug. Those are pinned by the cases - above and by tests/test_timing_close_window.py. + subtract it from the recorded window bounds. This asserts the published + record is internally consistent under that recomputation, so a span + silently added or dropped on the way in shows up here. """ from coder_eval.timing import busy_ms closed = [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))] - open_start = self.WINDOW_START + timedelta(milliseconds=500) - message = self._finish_step(monkeypatch, closed, open_starts=[open_start]) - - spans = [*closed, (open_start, message.completed_at)] + message = self._finish_step(monkeypatch, closed) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - expected = span_ms - busy_ms(spans, message.started_at, message.completed_at) + expected = span_ms - busy_ms(closed, message.started_at, message.completed_at) assert message.generation_duration_ms == pytest.approx(expected) @@ -1957,7 +1978,6 @@ def now(tz=None): return now state.step_started_at = step_start - state.step_tool_spans = [] monkeypatch.setattr(agent_module, "datetime", _Clock) state.on_step_finish({"reason": "stop", "tokens": {"input": 100, "output": 20}}) @@ -2022,11 +2042,15 @@ def now(tz=None): class TestToolSpansSurviveTheStepBoundary: """A tool that closes BETWEEN two steps still belongs to the next window. - `step_tool_spans` used to be cleared at `step_start`, which is after the - window it feeds has already opened at `gen_mark`. A call closing in that - gap had its span wiped before the next `step_finish` could subtract it, so - the window published the call's execution as model time while the call's - own `duration_ms` counted the same milliseconds again. + This used to be a bookkeeping problem: a per-step span list, cleared at + `step_start` — after the window it feeds had already opened at `gen_mark` — + so a call closing in the gap had its span wiped before the next + `step_finish` could subtract it. That list is gone. + `EventCollector.subtract_tool_time` sees every span at once and clips each + to the windows it overlaps, so the property now holds by construction + rather than by a reset rule. Kept, and re-pointed at the collector, because + the property is what matters: a future reducer change could still break it + by moving a mark or failing to emit the ToolEnd the collector reduces. It needs the NON-TERMINAL tool path to reach: the CLI normally emits one already-`completed` event per call, which closes inside the step that @@ -2061,7 +2085,24 @@ def tool(status, *, end_ms=None): state.on_step_start({"messageID": "m2"}) _SteppedClock.at_ms = 2000 state.on_step_finish({"reason": "stop", "tokens": {"input": 10, "output": 5}}) - return resolved, [m for m in state.messages if m.role == "assistant"] + + # Published through the real collector: the reducer hands over raw + # windows, and the tool subtraction happens once, there. + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t1", prompt="go", iteration=1, timestamp=_SPAN_BASE)) + for command in resolved: + collector.on_event(ToolEndEvent(task_id="t1", turn_id="s1", tool=command)) + collector.on_event( + AgentEndEvent( + task_id="t1", + status=AgentEndStatus.COMPLETED, + messages=list(state.messages), + usage=TokenUsage(), + timestamp=_SPAN_BASE + timedelta(milliseconds=2000), + ) + ) + published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] + return resolved, published def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self, monkeypatch): _, messages = self._run(monkeypatch) @@ -2123,7 +2164,14 @@ def test_a_duplicate_step_finish_does_not_republish_the_previous_window(self, mo assert messages[1].started_at == messages[0].completed_at assert sum(m.generation_duration_ms or 0.0 for m in messages) == pytest.approx(2000.0) - def test_a_step_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self, monkeypatch): + def test_a_step_that_never_finishes_does_not_advance_the_mark(self, monkeypatch): + """The half of this that is still the reducer's job. + + There is no span list to preserve any more — the collector reduces the + ToolEnd stream itself. What the reducer still owns is the MARK: a step + that published nothing must not advance it, or its time is handed to + whichever step finishes next. + """ monkeypatch.setattr(agent_module, "datetime", _SteppedClock) state = _OpenCodeTurnState(task_id="t1", iteration=1, user_input="go", model="m") _SteppedClock.at_ms = 0 @@ -2145,10 +2193,4 @@ def test_a_step_that_never_finishes_neither_advances_the_mark_nor_clears_the_spa _SteppedClock.at_ms = 1900 state.close_open_tools() # crash/timeout orphan sweep — no message appended - # Published nothing, so tiling past it would hand its time to whichever - # step finishes next, and wiping the spans would publish c2's execution - # as that step's model time. assert state.gen_mark == mark_after_flush - assert state.step_tool_spans == [ - (_SPAN_BASE + timedelta(milliseconds=1700), _SPAN_BASE + timedelta(milliseconds=1900)) - ] diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index baa2ce952..f9e6f5dd6 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -25,8 +25,9 @@ from coder_eval.agents.pi_agent import PiAgent, _PiTurnState, _result_text from coder_eval.errors import AgentCrashError, TurnTimeoutError -from coder_eval.models import AgentKind, AssistantMessage, CommandTelemetry, PiAgentConfig +from coder_eval.models import AgentKind, AssistantMessage, CommandTelemetry, PiAgentConfig, TokenUsage from coder_eval.pricing import calculate_cost +from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -1104,52 +1105,78 @@ def now(self) -> datetime: class TestGenerationWindowExcludesToolExecution: - """A tool running inside a turn is not model time. - - Pi marks the window at `turn_start` and closes it at `turn_end`, and - every tool call executes INSIDE it while also publishing its own - measured `duration_ms`. Publishing the raw span as generation time - counted the same milliseconds twice, which the task page's Unaccounted - cell renders as a ~-100% residual. - - Driven at the reducer with an injected clock frozen at `WINDOW_END`: the - window's end and the tool intervals both have to be set explicitly for the - arithmetic to be deterministic. + """A tool running inside a turn is not model time — asserted where it is now DECIDED. + + The reducer no longer subtracts anything. It publishes the RAW window, and + `EventCollector.subtract_tool_time` takes the tool union back out of it + once, for all five harnesses. So these cases drive the reducer and then a + real collector, and assert the PUBLISHED number — the one that reaches + `task.json` — rather than an intermediate the reducer used to own. + + They are not duplicates of + `tests/test_event_collector.py::TestSubtractToolTime`: those pin the + arithmetic, these pin that THIS reducer hands the collector a window and a + span set the arithmetic can be right about. """ WINDOW_START = datetime(2026, 1, 1, 12, 0, 0) WINDOW_END = datetime(2026, 1, 1, 12, 0, 1) # a 1000ms turn def _finish_turn(self, spans, open_starts=()): - state = _PiTurnState( - task_id="t", - iteration=1, - user_input="x", - model="m", - clock=_FixedClock(self.WINDOW_END), - ) + """Drive the reducer, then publish through a real collector. + + `spans` are RESOLVED calls (both bounds); `open_starts` are calls that + never returned. An unresolved call now contributes NO span — it has no + `execution_completed_at`, and inventing one is what `None` exists to + prevent — where the reducer used to bound it at the window's end. That + is a real change and a better one: the collector sees every span at + once, so a call straddling a boundary is clipped to each window it + actually overlapped instead of approximated at the boundary. + """ + state = _PiTurnState(task_id="t", iteration=1, user_input="x", model="m", clock=_FixedClock(self.WINDOW_END)) state.turn_started_at = self.WINDOW_START - state.turn_tool_spans = list(spans) - for i, started in enumerate(open_starts): - state.open_tools[f"open-{i}"] = CommandTelemetry( + commands = [ + CommandTelemetry( tool_name="bash", - tool_id=f"open-{i}", + tool_id=f"closed-{i}", timestamp=started, execution_started_at=started, + execution_completed_at=completed, + result_status="success", ) + for i, (started, completed) in enumerate(spans) + ] + commands += [ + CommandTelemetry(tool_name="bash", tool_id=f"open-{i}", timestamp=s, execution_started_at=s) + for i, s in enumerate(open_starts) + ] state.on_turn_end( {"message": {"role": "assistant", "usage": {"input": 100, "output": 20}, "stopReason": "stop"}} ) - assistant = [m for m in state.messages if m.role == "assistant"] - assert len(assistant) == 1 - return assistant[0] + + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="x", iteration=1, timestamp=self.WINDOW_START)) + for command in commands: + collector.on_event(ToolEndEvent(task_id="t", turn_id="t1", tool=command)) + collector.on_event( + AgentEndEvent( + task_id="t", + status=AgentEndStatus.COMPLETED, + messages=list(state.messages), + usage=TokenUsage(), + timestamp=self.WINDOW_END, + ) + ) + published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] + assert len(published) == 1 + return published[0] def test_tool_time_inside_the_turn_is_subtracted(self): message = self._finish_turn( [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], ) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - assert span_ms == pytest.approx(1000.0) + assert span_ms == pytest.approx(1000.0), "the reducer still publishes the whole window as its bounds" assert message.generation_duration_ms == pytest.approx(500.0) def test_a_turn_with_no_tools_keeps_its_whole_window(self): @@ -1172,48 +1199,39 @@ def test_the_window_never_goes_negative(self): ) assert message.generation_duration_ms == 0.0 - def test_a_tool_still_open_at_the_boundary_is_subtracted(self): - # A call that opens inside this turn and closes inside the NEXT one - # straddles the boundary. Counting only closed intervals published the - # pre-boundary 400ms as generation while the call's own duration_ms - # counted it again. - message = self._finish_turn( - [], - open_starts=[self.WINDOW_START + timedelta(milliseconds=600)], - ) - assert message.generation_duration_ms == pytest.approx(600.0) + def test_a_tool_still_open_at_the_boundary_contributes_no_span(self): + """The behaviour that CHANGED with the move, stated rather than implied. - def test_an_open_tool_overlapping_a_closed_one_is_counted_once(self): - # Union, not sum, across the closed and still-open sets alike. + The reducer used to bound a still-open call at the window's end and + subtract that slice. The collector cannot: a call with no + `execution_completed_at` was never timed. Its time is subtracted when it + RESOLVES, from whichever windows its real interval overlaps. + """ + message = self._finish_turn([], open_starts=[self.WINDOW_START + timedelta(milliseconds=600)]) + assert message.generation_duration_ms == pytest.approx(1000.0) + + def test_a_resolved_tool_overlapping_an_unresolved_one_counts_only_the_resolved(self): message = self._finish_turn( [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))], open_starts=[self.WINDOW_START + timedelta(milliseconds=500)], ) - assert message.generation_duration_ms == pytest.approx(200.0) + assert message.generation_duration_ms == pytest.approx(500.0) def test_the_published_window_reconciles_to_its_own_bounds(self): - """The reducer subtracted exactly the spans the record carries. + """The collector subtracted exactly the spans the record carries. `scripts/timing/decompose_run.py` and the evalboard's Unaccounted cell both recompute the tool UNION from the recorded command spans and - subtract it from the recorded window bounds. This asserts the reducer - fed the window the same set, so a span silently added or dropped on - the way in shows up here. - - It is deliberately the narrow half: `expected` is derived from the - PUBLISHED bounds, so it cannot see a wrong mark, and both sides call - `busy_ms`, so it cannot see a union bug. Those are pinned by the cases - above and by tests/test_timing_close_window.py. + subtract it from the recorded window bounds. This asserts the published + record is internally consistent under that recomputation, so a span + silently added or dropped on the way in shows up here. """ from coder_eval.timing import busy_ms closed = [(self.WINDOW_START + timedelta(milliseconds=200), self.WINDOW_START + timedelta(milliseconds=700))] - open_start = self.WINDOW_START + timedelta(milliseconds=500) - message = self._finish_turn(closed, open_starts=[open_start]) - - spans = [*closed, (open_start, message.completed_at)] + message = self._finish_turn(closed) span_ms = (message.completed_at - message.started_at).total_seconds() * 1000.0 - expected = span_ms - busy_ms(spans, message.started_at, message.completed_at) + expected = span_ms - busy_ms(closed, message.started_at, message.completed_at) assert message.generation_duration_ms == pytest.approx(expected) @@ -1282,12 +1300,16 @@ def test_the_inter_turn_gap_is_inside_a_window_rather_than_unaccounted(self): class TestToolSpansSurviveTheTurnBoundary: """A tool that closes BETWEEN two turns still belongs to the next window. - `turn_tool_spans` used to be cleared at `turn_start`, which is after the - window it feeds has opened at the mark. Pi was protected from that only by - NOT tiling: its window opened at `turn_start`, so a call that ended before - then fell outside it anyway. Tiling without moving the reset therefore - takes a correct harness and introduces the double-count — which is why both - changes land in one commit, reset first. + This used to be a bookkeeping problem: a per-turn span list, cleared at + `turn_start` — after the window it feeds had already opened at the mark — + so a call closing in the gap had its span wiped before the flush could + subtract it. That list is gone. `EventCollector.subtract_tool_time` sees + every span at once and clips each to the windows it overlaps, so the + property now holds by construction rather than by a reset rule. + + Kept, and re-pointed at the collector, because the property itself is what + matters and a future reducer change could still break it — by moving a + mark, or by failing to emit the ToolEnd the collector reduces. """ def _run(self): @@ -1309,7 +1331,24 @@ def _run(self): state.on_turn_start() clock.at_ms = 2000 state.on_turn_end(_turn_end_payload()) - return resolved, [m for m in state.messages if m.role == "assistant"] + + # Published through the real collector: the reducer hands over raw + # windows, and the tool subtraction happens once, there. + collector = EventCollector() + collector.on_event(AgentStartEvent(task_id="t", prompt="go", iteration=1, timestamp=_SPAN_BASE)) + for command in resolved: + collector.on_event(ToolEndEvent(task_id="t", turn_id="t1", tool=command)) + collector.on_event( + AgentEndEvent( + task_id="t", + status=AgentEndStatus.COMPLETED, + messages=list(state.messages), + usage=TokenUsage(), + timestamp=_SPAN_BASE + timedelta(milliseconds=2000), + ) + ) + published = [m for m in collector.build_turn_record().messages if m.role == "assistant"] + return resolved, published def test_the_gap_slice_of_a_straddling_call_is_not_published_as_generation(self): _, messages = self._run() @@ -1371,7 +1410,14 @@ def test_a_duplicate_turn_end_does_not_republish_the_previous_window(self): assert messages[1].started_at == messages[0].completed_at assert sum(m.generation_duration_ms or 0.0 for m in messages) == pytest.approx(2000.0) - def test_a_turn_that_never_finishes_neither_advances_the_mark_nor_clears_the_spans(self): + def test_a_turn_that_never_finishes_does_not_advance_the_mark(self): + """The half of this that is still the reducer's job. + + There is no span list to preserve any more — the collector reduces the + ToolEnd stream itself. What the reducer still owns is the MARK: a turn + that published nothing must not advance it, or its time is handed to + whichever turn finishes next. + """ clock = _SteppedClock() state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) state.on_turn_start() @@ -1386,13 +1432,7 @@ def test_a_turn_that_never_finishes_neither_advances_the_mark_nor_clears_the_spa clock.at_ms = 1900 state.close_open_tools() # crash/timeout orphan sweep — no message appended - # Published nothing, so tiling past it would hand its time to whichever - # turn finishes next, and wiping the spans would publish c2's execution - # as that turn's model time. assert state.gen_mark == mark_after_flush - assert [(s, e) for s, e in state.turn_tool_spans] == [ - (_SPAN_BASE + timedelta(milliseconds=1700), _SPAN_BASE + timedelta(milliseconds=1900)) - ] class TestClockIsFreshPerTurn: diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py index b47abe22c..31dc5dc2d 100644 --- a/tests/test_timing_close_window.py +++ b/tests/test_timing_close_window.py @@ -33,113 +33,58 @@ def _load_decompose_run(): class TestCloseWindow: - def test_no_tools_keeps_the_whole_window(self): - started, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[]) + """The RAW window: where it opens, where it ends, and the clamp. + + The tool subtraction these cases used to cover moved to + `streaming/collector.py::subtract_tool_time`, where it happens once for all + five harnesses instead of five times in five reducers — see + `tests/test_event_collector.py::TestSubtractToolTime`, which carries the + union, grouping, clamping and non-mutation cases. What is left here is the + part that is genuinely per-reducer: the mark. + """ + + def test_the_window_is_the_whole_span_from_the_mark(self): + started, span_ms = close_window(mark=MARK, now=_at(1000)) assert started == MARK - assert generation_ms == pytest.approx(1000.0) - - def test_a_contained_closed_tool_is_subtracted_once(self): - _, generation_ms = close_window( - mark=MARK, - now=_at(1000), - closed_spans=[(_at(200), _at(700))], - open_started_ats=[], - ) - assert generation_ms == pytest.approx(500.0) - - def test_overlapping_closed_tools_subtract_their_union_not_their_sum(self): - # Two 500 ms calls overlapping by 400 ms occupy 600 ms of wall clock. - # Summing them would leave 0 generation for a window that generated 400. - _, generation_ms = close_window( - mark=MARK, - now=_at(1000), - closed_spans=[(_at(100), _at(600)), (_at(200), _at(700))], - open_started_ats=[], - ) - assert generation_ms == pytest.approx(400.0) - - def test_an_open_tool_is_bounded_at_now(self): - # Still running when the window closes: it owns [300, 1000], not nothing. - _, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[_at(300)]) - assert generation_ms == pytest.approx(300.0) - - def test_a_tool_straddling_the_mark_is_clipped_to_the_post_mark_part(self): - # The pre-mark half belongs to the PREVIOUS window, which already - # subtracted it. Counting it again here would over-subtract. - _, generation_ms = close_window( - mark=MARK, - now=_at(1000), - closed_spans=[(_at(-400), _at(300))], - open_started_ats=[], - ) - assert generation_ms == pytest.approx(700.0) + assert span_ms == pytest.approx(1000.0) def test_item_start_before_the_mark_wins(self): # A stamp that went backwards: the window must cover the item, so the # min() moves the start back rather than inverting the span. - started, generation_ms = close_window( - mark=MARK, - now=_at(1000), - item_start=_at(-200), - closed_spans=[], - open_started_ats=[], - ) + started, span_ms = close_window(mark=MARK, now=_at(1000), item_start=_at(-200)) assert started == _at(-200) - assert generation_ms == pytest.approx(1200.0) + assert span_ms == pytest.approx(1200.0) def test_item_start_after_the_mark_keeps_the_mark(self): # The normal tiling case: the gap between the previous close and this # item's first stamp IS model time and belongs inside the window. - started, generation_ms = close_window( - mark=MARK, - now=_at(1000), - item_start=_at(400), - closed_spans=[], - open_started_ats=[], - ) + started, span_ms = close_window(mark=MARK, now=_at(1000), item_start=_at(400)) assert started == MARK - assert generation_ms == pytest.approx(1000.0) + assert span_ms == pytest.approx(1000.0) def test_an_inverted_window_clamps_to_zero_rather_than_going_negative(self): - started, generation_ms = close_window(mark=_at(1000), now=MARK, closed_spans=[], open_started_ats=[]) + started, span_ms = close_window(mark=_at(1000), now=MARK) assert started == _at(1000) - assert generation_ms == 0.0 - - def test_an_open_tool_starting_after_now_is_ignored(self): - _, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[_at(1500)]) - assert generation_ms == pytest.approx(1000.0) - - def test_an_open_tool_starting_exactly_at_now_is_ignored(self): - _, generation_ms = close_window(mark=MARK, now=_at(1000), closed_spans=[], open_started_ats=[_at(1000)]) - assert generation_ms == pytest.approx(1000.0) - - def test_closed_and_open_spans_are_unioned_together(self): - # A closed [100, 400] and an open from 300 bounded at 1000 union to - # [100, 1000] — 900 ms busy, 100 ms of generation. - _, generation_ms = close_window( - mark=MARK, - now=_at(1000), - closed_spans=[(_at(100), _at(400))], - open_started_ats=[_at(300)], - ) - assert generation_ms == pytest.approx(100.0) - - def test_tools_covering_the_whole_window_leave_zero_not_a_negative(self): - _, generation_ms = close_window( - mark=MARK, - now=_at(1000), - closed_spans=[(_at(-500), _at(1500))], - open_started_ats=[], - ) - assert generation_ms == 0.0 + assert span_ms == 0.0 def test_mark_is_keyword_only_and_has_no_default(self): # A reducer cannot open a window without STATING what it tiles from. # The value is still the caller's to get right — see the docstring. with pytest.raises(TypeError): - close_window(MARK, _at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[misc] + close_window(MARK, _at(1000)) # type: ignore[misc] + with pytest.raises(TypeError): + close_window(now=_at(1000)) # type: ignore[call-arg] + + def test_it_no_longer_accepts_the_span_arguments_that_moved(self): + """The subtraction moved; the parameters must not linger as no-ops. + + A reducer still passing `closed_spans=` would otherwise keep compiling + while its tool time was silently subtracted a second time centrally. + """ + with pytest.raises(TypeError): + close_window(mark=MARK, now=_at(1000), closed_spans=[]) # type: ignore[call-arg] with pytest.raises(TypeError): - close_window(now=_at(1000), closed_spans=[], open_started_ats=[]) # type: ignore[call-arg] + close_window(mark=MARK, now=_at(1000), open_started_ats=[]) # type: ignore[call-arg] class TestNaiveAwareMix: @@ -340,3 +285,98 @@ def test_a_fresh_clock_anchors_on_its_own_pair(self): assert second._mono0 > first._mono0 assert second._wall0 >= first._wall0 + + +class TestTheThreeToolUnionsAgree: + """Three implementations recompute the turn's tool union. They must agree. + + * `EventCollector._main_thread_tool_spans` — what the harness subtracts + from the generation windows and measures the head and tail against. + * `tests/_fixtures/golden_streams/_scrub.py::_tool_union_ms` — the golden + corpus's identity check. + * `scripts/timing/decompose_run.py::_tool_ms` — the LIVE two-sided residual + gate, which `.github/workflows/pr-checks.yml` runs against a real run. + + They agreed by luck once and it cost a defect: the collector filtered its + GENERATIONS to the main thread and then passed EVERY command as a tool + span. A child nests inside the parent Agent call, whose interval the union + already covers, so nothing failed — but Codex's recovered child tools carry + the CHILD's clock, so the nesting is not guaranteed. When the collector + started filtering, the other two did not, and a gate computing a different + tool total than the harness reports a residual that is an artifact of the + disagreement rather than a bucket error. That is the worst possible place + for a divergence, because this is the only live sensor for the identity. + """ + + @staticmethod + def _record() -> dict: + """A turn with a sub-agent whose own tool sits OUTSIDE the parent call. + + Inside, the three agree whatever they filter, so the fixture has to put + the child's tool where the parent's interval does not cover it. + """ + return { + "duration_seconds": 3.0, + "commands": [ + { + "tool_id": "agent-call", + "execution_started_at": _at(1000).isoformat(), + "execution_completed_at": _at(1500).isoformat(), + }, + { + "tool_id": "child-tool", + "execution_started_at": _at(2000).isoformat(), + "execution_completed_at": _at(2400).isoformat(), + }, + ], + "messages": [ + {"role": "assistant", "parent_tool_use_id": None, "tool_use_ids": ["agent-call"]}, + {"role": "assistant", "parent_tool_use_id": "agent-call", "tool_use_ids": ["child-tool"]}, + ], + } + + def test_the_two_recomputing_readers_exclude_the_sub_agent_tool(self): + from tests._fixtures.golden_streams._scrub import _tool_union_ms + + tool_ms = _load_decompose_run()._tool_ms + record = self._record() + # Only the parent Agent call's own 500 ms. Counting the child's 400 ms + # books time no main-thread bucket claims. + assert _tool_union_ms(record) == pytest.approx(500.0) + assert tool_ms(record) == pytest.approx(500.0) + + def test_the_collector_excludes_it_too(self): + from coder_eval.models import AssistantMessage, CommandTelemetry + from coder_eval.streaming.collector import EventCollector + from coder_eval.streaming.events import ToolEndEvent + + collector = EventCollector() + for tool_id, lo, hi in (("agent-call", 1000, 1500), ("child-tool", 2000, 2400)): + collector.on_event( + ToolEndEvent( + task_id="t", + turn_id="t1", + tool=CommandTelemetry( + tool_name="Agent", + tool_id=tool_id, + timestamp=_at(lo), + execution_started_at=_at(lo), + execution_completed_at=_at(hi), + result_status="success", + ), + ) + ) + messages = [ + AssistantMessage( + started_at=_at(0), completed_at=_at(1000), generation_duration_ms=1000.0, tool_use_ids=["agent-call"] + ), + AssistantMessage( + started_at=_at(2000), + completed_at=_at(2400), + generation_duration_ms=400.0, + parent_tool_use_id="agent-call", + tool_use_ids=["child-tool"], + ), + ] + spans = collector._main_thread_tool_spans(messages) + assert union_ms(spans) == pytest.approx(500.0), "the same 500 ms the other two report" diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index a3354a66c..64c8c8a0a 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -33,9 +33,10 @@ * a ``datetime`` SUBCLASS monkeypatched onto the module — opencode, which also calls ``datetime.fromtimestamp`` through the same global (see ``tests/test_opencode_agent.py``'s ``_SteppedClock`` for why a stub breaks); -* ``time.monotonic`` AND ``datetime`` both patched — claude-code, which derives - the DURATION from the monotonic clock and the BOUNDS from the wall clock, so - patching one leaves the other real and the test measures nothing. +* ``time.monotonic`` AND ``datetime`` both patched — claude-code. Its window is + wall-derived now, but ``turn_start_time`` and the turn deadline still read + ``time.monotonic()``, so patching only one leaves the reducer straddling a + real clock and a scripted one. Codex is the fifth and takes its stamps from SDK epoch milliseconds rather than from any host clock, so its case scripts those stamps directly. @@ -419,10 +420,10 @@ def _codex_turn() -> Turn: def _claude_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: """A tool call between two emissions, with a real head and a real tail. - This reducer derives the window's DURATION from ``time.monotonic()`` and - its BOUNDS from ``datetime.now()``, so both module globals are patched off - one counter. Patching either alone leaves the other reading the real clock, - and the case would then assert a measured span against an unmeasured one. + Both module globals are patched off one counter. The window itself is + wall-derived, but ``turn_start_time`` and the deadline still read + ``time.monotonic()``, so patching only one leaves the reducer straddling a + real clock and a scripted one. The first `message_start` re-seeds the window, so the CLI spawn and the query build before it are head rather than msg0's generation. That a LATER From 8ab9d247d34a194c34b513da874c71e2e70750cd Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 07:00:44 -0700 Subject: [PATCH 33/54] =?UTF-8?q?feat(reports):=206-7/7=20=E2=80=94=20the?= =?UTF-8?q?=20offline=20report=20carries=20the=20buckets;=20a=20TS=20None-?= =?UTF-8?q?vs-0=20guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Phase 6.** `reports_html.py` is described in CLAUDE.md as the evalboard's static twin, and it rendered only Total Latency / Turns / Avg Turn Latency — so anyone reading the artifact rather than the dashboard got none of the wall-clock accounting this branch added. The card now shows Startup / Generation / Tool exec / Teardown / Unaccounted. The arithmetic is in `reports_stats.turn_time_buckets` and the renderer only formats, because putting the sums in `_render_generation_metrics` would make it the fourth place these buckets are aggregated. For the same reason the main-thread span rule is no longer restated there: `main_thread_tool_spans` moves out of `EventCollector` to module level and both consume it. A second typed copy of that rule is exactly how two surfaces come to publish two different tool totals for one run. Three None-vs-0 distinctions the first draft got wrong, each measured: * `tool_ms` returned `0.0` for a run that recorded no bounded span at all, rendering `0ms` — "measured and instant" — where nobody measured anything. It is `None` unless some turn recorded a span. * `unaccounted_ms` was computed from a `duration_seconds` that is a non-optional float defaulting to `0.0`, so an untimed run rendered a fabricated negative residual instead of a dash. The evalboard keeps its own null for this case. * The docstring claimed every bucket went `None` when nothing measured it, while two of five could not. Display and arithmetic differ on purpose and say so: an unmeasured bucket shows as an em dash and sums as `0.0`, so its time surfaces in Unaccounted rather than vanishing — the rule `decompose_run.py::_turn_buckets` already applies. The Unaccounted label states that it includes sandbox setup and grading, so it is not comparable with the per-turn residual. **Phase 7.** `no-zero-coalesce.test.ts` is the TypeScript counterpart to CE058. There is no eslint in `evalboard/`, so it is a vitest source scan. An ALLOWLIST rather than a ban, because the residual arithmetic uses `?? 0` correctly — subtracting only what was measured is the whole point — so a blanket ban fires on right code. It scans for timing names (`Ms`, `Seconds`, `duration`) rather than every `?? 0`, and that narrowing is deliberate: a blanket scan matches 58 occurrences, about half token and cache buckets where zero is a fine answer because tokens are counted rather than measured. An allowlist that long is one nobody reads. Blind spots are declared in the file. Two meta-tests keep it honest — a negative control, so the scan cannot pass by matching nothing, and an assertion that every allowlist entry is still present, so an entry cannot outlive its reason. Both caught real problems in the allowlist before it landed. `AssistantMessage.message_id` no longer names one harness of five. Its census is taken from the agents rather than from the plan, which had it off by one: three schemes, not two — passed through on claude-code, opencode and pi; synthesized on codex and antigravity; and claude-code synthesizes in exactly one place, the sub-agent terminal message that is never streamed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- .claude/harness-candidates.md | 63 +++-- .../lib/__tests__/no-zero-coalesce.test.ts | 179 +++++++++++++ src/coder_eval/models/telemetry.py | 17 +- src/coder_eval/reports_html.py | 41 ++- src/coder_eval/reports_stats.py | 115 +++++++- src/coder_eval/streaming/collector.py | 83 +++--- src/coder_eval/timing.py | 12 +- tests/test_reports_html.py | 253 +++++++++++++++++- 8 files changed, 695 insertions(+), 68 deletions(-) create mode 100644 evalboard/lib/__tests__/no-zero-coalesce.test.ts diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index f5048417a..a0efc75d5 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -565,21 +565,32 @@ divergences, so the deferred-work record is one place. Measurements in replay runs in well under one, so closing that last gap needs the agent's own clock faked, not the fixtures' rebased. -- [ ] **No TypeScript counterpart to CE058.** `evalboard/lib/runs.ts` and - `_sections.tsx` carry the same None-vs-0 contract as the Python side, and - `sumMeasured` implements it correctly, but nothing stops the next author - writing `?? 0` where an unmeasured value must stay null. Not a simple lint - rule: the residual arithmetic in `_sections.tsx` uses `?? 0` *correctly* - (subtract only what was measured), so a blanket ban fires on right code and - the rule needs a way to tell "publishing a value" from "consuming one". +- [x] ~~No TypeScript counterpart to CE058.~~ **DONE.** + `evalboard/lib/__tests__/no-zero-coalesce.test.ts` is a vitest source scan + (there is no eslint in `evalboard/`) over `lib/runs.ts`, `lib/timing.ts` and + `_sections.tsx`. It is an ALLOWLIST rather than a ban, exactly because the + residual arithmetic uses `?? 0` correctly — each of the 13 entries carries a + one-line reason, and a new occurrence fails until its author justifies it or + keeps the value null. It is keyed on the codebase's own `…Ms` naming + convention rather than on every `?? 0`: a blanket scan matches 60 + occurrences, ~40 of them token buckets where zero is a fine answer, and an + allowlist that long is one nobody reads. Blind spots are declared in the + file. Two meta-tests keep it honest — a negative control (so the scan cannot + pass by matching nothing) and an assertion that every allowlist entry is + still present, so an entry cannot outlive its reason. Caught in: turn head/tail timing final review. -- [ ] **`timing.py::decompose_turn` raises an uncaught `TypeError` on a - naive/aware datetime mix**, straight out of `EventCollector.build_turn_record`, - killing the turn. Unreachable today — every stamp in `agents/` and - `streaming/` is a naive `datetime.now()` (verified by grep: zero hits for - `timezone.utc` / `utcnow` / `astimezone`) — but nothing pins that invariant, - so the first agent to record an aware stamp discovers it at runtime. +- [x] ~~**`timing.py::decompose_turn` raises an uncaught `TypeError` on a + naive/aware datetime mix**~~ **DONE.** `timing.py::_require_same_awareness` + now raises from five call sites (`decompose_turn`'s head and tail, + `busy_ms`'s window bounds and each span's two ends) with one message template + naming the field and which side is aware. Deliberately a GUARD and not a lint + rule: the invariant is still unviolated in-tree, and the exposure that + actually matters is a third-party agent registered through the + `coder_eval.plugins` SPI, which lives outside `src/coder_eval/agents/` and + which a rule scoped to that directory could never see — so the message + addresses that reader directly. An empty span list is checked not at all, + bounds included: nothing is compared, so there is no pair to be about. Caught in: turn head/tail timing final review. - [x] ~~**`claude-code` does not subtract tool execution from its generation @@ -661,18 +672,20 @@ divergences, so the deferred-work record is one place. Measurements in wider than any one capture fix. Caught in: the CE060 / antigravity `message_id` final review. -- [ ] **`AssistantMessage.message_id`'s field description names one harness of - five** (`models/telemetry.py:283`: "Anthropic API message_id … when the Claude - Code CLI splits one API response"). Five backends now write the field and four - synthesize it, so `docs/agents/HARNESS_PARITY.md`'s new row is the real SSOT - while the model — which this project's DRY principle designates as - authoritative — describes claude-code only. Not fixed here because the plan - scoped out every model change (the field already existed, so touching it would - have put a schema file in a golden-regeneration diff for prose). No mechanical - guard is obvious either: "a field description must not name a single harness - when the union has five writers" needs a writer census per field, which is - CE054-shaped but over a `str` description rather than a key. The cheap version - is to fix the sentence in the next change that touches the model. +- [x] ~~**`AssistantMessage.message_id`'s field description names one harness of + five**~~ **DONE.** It said "Anthropic API message_id … when the Claude Code + CLI splits one API response", while five backends write the field and four + synthesize it — so `docs/agents/HARNESS_PARITY.md`'s row was the real SSOT + and the model, which this project's DRY principle designates as + authoritative, described claude-code only. Rewritten agent-agnostically: what + the id MEANS (the generation an emission belongs to), that all five write it + and four synthesize it, each scheme named, a pointer to the per-harness row, + and the fact that it is a WITHIN-TURN identity that repeats across retry + attempts. No mechanical guard was added and none is obvious — "a field + description must not name a single harness when the union has five writers" + needs a writer census per field, which is CE054-shaped but over a `str` + description rather than a key; the cheap version was exactly this, fixing the + sentence in the next change that touches the model. Caught in: the CE060 / antigravity `message_id` final review. - [ ] **The golden corpus pins that a timing value EXISTS, never what it is.** diff --git a/evalboard/lib/__tests__/no-zero-coalesce.test.ts b/evalboard/lib/__tests__/no-zero-coalesce.test.ts new file mode 100644 index 000000000..ed82280af --- /dev/null +++ b/evalboard/lib/__tests__/no-zero-coalesce.test.ts @@ -0,0 +1,179 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "vitest"; + +// The TypeScript counterpart to CE058. +// +// CE058 stops an unmeasured TIMING value becoming a numeric literal in `src/`: +// `durationMs == null` means *never timed* and `0` means *timed and instant*, +// so writing the literal publishes the second while meaning the first. The +// evalboard carries the same contract — `sumMeasured` (lib/runs.ts) implements +// it correctly, returning null when nothing was measured — but nothing stopped +// the next author writing `?? 0` where an unmeasured value must stay null. +// +// There is no eslint in evalboard/ (package.json has only typecheck / test / +// build), so this is a vitest source scan rather than a lint plugin. +// +// WHY AN ALLOWLIST RATHER THAN A BAN. The residual arithmetic in _sections.tsx +// uses `?? 0` *correctly*: it subtracts only what was measured, which is the +// whole point. A blanket ban fires on right code. So every occurrence must be +// listed with a reason, and a NEW one fails until its author either justifies +// it here or uses null. +// +// WHY IT KEYS ON TIMING NAMES, and this is a deliberate narrowing. A scan for +// every `?? 0` in these files matches 58 occurrences, roughly half of them +// token or cache buckets (`inputTokens ?? 0`, `prev?.input ?? 0`) where zero is +// a perfectly good answer — tokens are COUNTED, not measured, so there is no +// None-vs-0 ambiguity to protect. An allowlist that long, much of it saying +// "token bucket, fine", is one nobody reads and everybody appends to. CE058's +// contract is about TIMING, so this matches the codebase's own naming +// conventions for a measured interval: an identifier ending in `Ms`, `Seconds`, +// or `duration`/`Duration`. +// +// DECLARED BLIND SPOTS, stated the way the Python rules state theirs: +// * only the three files below are covered; anything else gains no protection; +// * a timing field named by NONE of those conventions is invisible — the +// convention IS the rule, and it is a convention rather than a type; +// * `?? 0.0`, `|| 0.0` and an `if (x == null) x = 0` assignment are not matched. + +const here = dirname(fileURLToPath(import.meta.url)); +const root = resolve(here, "../.."); + +const COVERED = [ + "lib/runs.ts", + "lib/timing.ts", + "app/runs/[id]/[...task]/_sections.tsx", +]; + +// Keyed on the trimmed source LINE, not a line number: numbers move on every +// edit and the test would fail for unrelated reasons. +const ALLOWED = new Map([ + [ + "? executed.reduce((a, t) => a + (t.duration ?? 0), 0)", + "Guarded: the enclosing branch only runs when `allHaveDuration` is true, so every term was measured.", + ], + [ + "const totalGenMs = mainThread.reduce((s, m) => s + (m.generationMs ?? 0), 0);", + "Summing a breakdown: an unmeasured emission contributes nothing to the total, which is what a sum of the measured means.", + ], + [ + "const thinkingMs = mainThread.reduce((s, m) => s + (m.thinkingMs ?? 0), 0);", + "Summing a breakdown: an unmeasured emission contributes nothing, which is what a sum of the measured means.", + ], + [ + "const textMs = mainThread.reduce((s, m) => s + (m.textMs ?? 0), 0);", + "Summing a breakdown: an emission with no text time contributes nothing to the text total.", + ], + [ + "const toolGenMs = mainThread.reduce((s, m) => s + (m.toolGenMs ?? 0), 0);", + "Summing a breakdown: an emission with no tool-gen time contributes nothing to that total.", + ], + [ + "const mixedMs = mainThread.reduce((s, m) => s + (m.mixedGenMs ?? 0), 0);", + "Summing a breakdown: an emission with no mixed-block time contributes nothing to that total.", + ], + [ + "(m) => (m.generationMs ?? 0) >= SLOW_GEN_MS,", + "A threshold comparison: an unmeasured generation is not a slow one, and 0 is the right answer to the question asked.", + ], + [ + "s + m.toolUses.filter((t) => (t.durationMs ?? 0) >= SLOW_TOOL_MS).length,", + "A threshold comparison: an untimed call is not a slow call, so 0 answers the question asked.", + ], + [ + "(harnessStartupMs ?? 0) -", + "The residual. Subtracting only what was measured is the whole point; an unmeasured head leaves its time IN the residual rather than silently claiming it.", + ], + [ + "(harnessTeardownMs ?? 0)", + "The residual's tail half: subtracting only what was measured leaves unmeasured time IN the residual.", + ], + [ + "const slowExec = (execMs ?? 0) >= SLOW_TOOL_MS;", + "A threshold comparison: an untimed execution is not a slow one, so 0 answers the question asked.", + ], + [ + "const slowGen = (m.generationMs ?? 0) >= SLOW_GEN_MS;", + "A threshold comparison: an unmeasured generation is not a slow one.", + ], + [ + "const slowTool = m.toolUses.some((t) => (t.durationMs ?? 0) >= SLOW_TOOL_MS);", + "A threshold comparison: an untimed call is not a slow call.", + ], + [ + "const execMs = m.toolUses.reduce((a, t) => a + (t.durationMs ?? 0), 0);", + "Summing the measured calls of one message; an untimed call adds nothing to that sum.", + ], +]); + +// `x ?? 0` / `x || 0` where the coalesced identifier names a measured interval. +// Not anchored to the line start, so several on one line are all found. +const COALESCE = /\b[\w$]*(?:Ms|Seconds|[Dd]uration)\s*(?:\?\?|\|\|)\s*0(?![.\d\w])/g; + +export function findCoalesces(source: string): string[] { + const hits: string[] = []; + for (const raw of source.split("\n")) { + // Strip line comments so prose about `?? 0` is not a violation. + const line = raw.replace(/\/\/.*$/, ""); + if (COALESCE.test(line)) hits.push(raw.trim()); + COALESCE.lastIndex = 0; + } + return hits; +} + +describe("no unguarded zero-coalesce on a timing value", () => { + for (const relative of COVERED) { + test(relative, () => { + const source = readFileSync(resolve(root, relative), "utf8"); + const unlisted = findCoalesces(source).filter((line) => !ALLOWED.has(line)); + expect( + unlisted, + `${relative} coalesces an unmeasured timing value to 0 without a reason. ` + + `\`x ?? 0\` on a \`…Ms\` field publishes "measured, and instant" where null means ` + + `"never measured" — the contract CE058 enforces on the Python side. If the zero is ` + + `correct here (a sum, or a threshold comparison), add the line to ALLOWED with a ` + + `one-line reason; otherwise keep it null.`, + ).toEqual([]); + }); + } + + test("the scanner actually matches something (a scan that finds nothing proves nothing)", () => { + // The failure mode that makes a source scanner worthless: a regex that + // silently stops matching, after which every file "passes". + const found = COVERED.flatMap((relative) => + findCoalesces(readFileSync(resolve(root, relative), "utf8")), + ); + expect(found.length).toBeGreaterThan(10); + }); + + test("every allowlist entry is still present in a covered file", () => { + // An entry nobody needs is an entry that outlived its reason. + const all = new Set( + COVERED.flatMap((relative) => + findCoalesces(readFileSync(resolve(root, relative), "utf8")), + ), + ); + expect([...ALLOWED.keys()].filter((line) => !all.has(line))).toEqual([]); + }); + + test("every allowlist entry carries a non-empty reason", () => { + expect([...ALLOWED.entries()].filter(([, why]) => why.trim().length < 20)).toEqual([]); + }); + + test("NEGATIVE CONTROL: an un-allowlisted occurrence is reported", () => { + // Without this the suite could pass by matching nothing at all. + const hits = findCoalesces("const x = someTimingMs ?? 0;\n"); + expect(hits).toEqual(["const x = someTimingMs ?? 0;"]); + expect(ALLOWED.has(hits[0])).toBe(false); + }); + + test("NEGATIVE CONTROL: prose and non-timing fields are not matched", () => { + expect(findCoalesces("// the residual uses `?? 0` deliberately\n")).toEqual([]); + expect(findCoalesces("const n = inputTokens ?? 0;\n")).toEqual([]); + expect(findCoalesces("const n = count ?? 0;\n")).toEqual([]); + // The widened conventions, so a regression to `Ms`-only is caught here. + expect(findCoalesces("const s = taskSeconds ?? 0;\n")).toEqual(["const s = taskSeconds ?? 0;"]); + expect(findCoalesces("const d = t.duration ?? 0;\n")).toEqual(["const d = t.duration ?? 0;"]); + }); +}); diff --git a/src/coder_eval/models/telemetry.py b/src/coder_eval/models/telemetry.py index a29d12f92..7079b2dac 100644 --- a/src/coder_eval/models/telemetry.py +++ b/src/coder_eval/models/telemetry.py @@ -290,9 +290,20 @@ class AssistantMessage(BaseModel): message_id: str | None = Field( default=None, description=( - "Anthropic API message_id. Multiple AssistantMessage records can share this id " - "when the Claude Code CLI splits one API response into per-block-kind events. " - "Downstream tooling can group by this id to recover one logical generation." + "Identity of the generation this emission belongs to. Several AssistantMessage " + "records can share one id, and downstream tooling groups by it to recover a single " + "logical generation — the evalboard's timeline renders one row per group. " + "ALL FIVE backends write it, by three different schemes. Passed THROUGH from the " + "harness: claude-code (a real Anthropic API message_id, which repeats because the CLI " + "splits one API response into per-block-kind events — the case this field was named " + "for), and opencode and pi (the CLI's own id, so they can legitimately leave this None " + "when the payload omits it). SYNTHESIZED: codex, which deliberately repeats one id " + "across the sub-messages of a generation, and antigravity, which mints a distinct one " + "per generation because its Step stream carries none. claude-code also synthesizes in " + "ONE place — the sub-agent terminal message, which is delivered as a tool result and " + "never streamed. See docs/agents/HARNESS_PARITY.md for the per-harness row — and note " + "the id is a WITHIN-TURN identity only: it repeats across retry attempts of one turn " + "on every synthesizing harness." ), ) diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index 7f8fcade4..c0c1c8ebf 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -20,7 +20,7 @@ from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs from .reports import early_stop_gate_note -from .reports_stats import format_score, is_env_table_key +from .reports_stats import format_score, is_env_table_key, turn_time_buckets if TYPE_CHECKING: @@ -932,8 +932,20 @@ def _render_token_usage(result: EvaluationResult) -> str: """ +def _format_signed_ms(ms: float | None) -> str: + """Like `_format_ms`, but keeps a NEGATIVE residual visible and signed. + + A negative Unaccounted is real and means generation and tool execution + overlapped, so it is rendered rather than clamped — the evalboard does the + same. Clamping would turn a measurable inconsistency into a clean zero. + """ + if ms is None: + return "—" + return f"-{_format_ms(-ms)}" if ms < 0 else _format_ms(ms) + + def _render_generation_metrics(result: EvaluationResult) -> str: - """Render Generation Metrics — latency, turns.""" + """Render Generation Metrics — latency, turns, and the four wall-clock buckets.""" from .reports import count_partials_by_outcome, group_consecutive_by_iteration turns = result.iterations or [] @@ -951,6 +963,22 @@ def _render_generation_metrics(result: EvaluationResult) -> str: f'
Crashed Partials
' f'
{_esc(breakdown)}
' ) + # The four wall-clock buckets. The arithmetic is in reports_stats; this + # only formats it. An unmeasured bucket renders as an em dash, never 0ms — + # a run predating the head/tail capture measured nothing, and a zero would + # claim it measured instantly (CE058). + buckets = turn_time_buckets(result) + startup = _format_ms(buckets.startup_ms) + generation = _format_ms(buckets.generation_ms) + tool_exec = _format_ms(buckets.tool_ms) + teardown = _format_ms(buckets.teardown_ms) + unaccounted = _format_signed_ms(buckets.unaccounted_ms) + unaccounted_title = _esc( + "the task's wall clock minus the four buckets. Measured against the whole task, so it " + + "legitimately includes sandbox setup and grading — it is LARGER than the per-turn residual " + + "scripts/timing/decompose_run.py reports, and the two are not comparable. Negative means " + + "generation and tool execution overlapped." + ) return f"""

Generation Metrics

@@ -961,6 +989,15 @@ def _render_generation_metrics(result: EvaluationResult) -> str:
Avg Turn Latency
{avg_latency}
{crashed_stat}
+
+
Startup
{startup}
+
Generation
{generation}
+
Tool exec
{tool_exec}
+
Teardown
{teardown}
+
+
Unaccounted (incl. setup + grading)
{unaccounted}
+
+
""" diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index ba9f9de37..5bba846f5 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -11,10 +11,20 @@ import math import random import statistics as _stats +from collections.abc import Iterable from pathlib import Path from typing import NamedTuple -from coder_eval.models import EvaluationResult, ExperimentResult, ExperimentVariant, TaskExperimentSummary +from coder_eval.models import ( + AssistantMessage, + EvaluationResult, + ExperimentResult, + ExperimentVariant, + TaskExperimentSummary, + TurnRecord, +) +from coder_eval.streaming.collector import main_thread_tool_spans +from coder_eval.timing import union_ms from .path_utils import TASK_JSON_FILENAME @@ -336,6 +346,109 @@ def format_score(score: float | None) -> str: return UNGRADED_SCORE_TEXT if score is None else f"{score:.3f}" +class TurnTimeBuckets(NamedTuple): + """The four wall-clock buckets of a whole run, plus what they leave over. + + Each is ``None`` when NOTHING in the run measured it — a run recorded before + the head and tail were captured has no startup at all, a run that recorded + no bounded tool span has no tool total, and a run with no duration has no + residual. Rendering any of those as ``0ms`` claims a measurement nobody + took (CE058, and the reason the evalboard's ``sumMeasured`` returns + ``null``). A MEASURED zero stays ``0.0`` and renders as ``0ms``. + + DISPLAY AND ARITHMETIC DIFFER HERE, on purpose. An unmeasured bucket renders + as a dash and counts as ``0.0`` toward ``unaccounted``, so the missing time + surfaces as residual rather than vanishing. That is the rule + ``scripts/timing/decompose_run.py::_turn_buckets`` already applies, and + keeping the two the same is what lets a reader compare them. + """ + + startup_ms: float | None + generation_ms: float | None + tool_ms: float | None + teardown_ms: float | None + unaccounted_ms: float | None + + +def turn_time_buckets(result: EvaluationResult) -> TurnTimeBuckets: + """Sum the four timing buckets across a run's turns, and the residual. + + The arithmetic lives HERE rather than in the renderer because this module is + the designated home for shared report statistics: the evalboard, the + markdown report and the HTML report must not each grow their own version. + ``reports_html`` formats what this returns and decides nothing. + + ``unaccounted`` is measured against ``EvaluationResult.duration_seconds`` — + the TASK's wall clock, which is what the card's existing Total Latency uses + and what the evalboard's own Unaccounted cell uses. It therefore legitimately + contains sandbox setup and grading, and is LARGER than the per-turn residual + ``decompose_run.py`` reports. The two are not comparable and the label says + so. + """ + turns = result.iterations or [] + startup = _sum_measured(t.harness_startup_ms for t in turns) + teardown = _sum_measured(t.harness_teardown_ms for t in turns) + # MAIN THREAD ONLY, the same filter the collector and the evalboard apply: + # a sub-agent's generations bubble into the same stream, and the spawning + # Agent call's own interval already spans them. + generation = _sum_measured( + m.generation_duration_ms + for t in turns + for m in t.messages + if isinstance(m, AssistantMessage) and m.parent_tool_use_id is None + ) + # `None` only when NO turn recorded a bounded tool span. A turn that ran + # tools and timed none is indistinguishable from a turn that ran none, so + # the presence of a SPAN — not the presence of a turn — is what decides + # measured-versus-not. `_sum_measured` over a list of plain floats could + # never return None, which made this read `0ms` ("measured and instant") + # for a run nobody timed. + per_turn = [_turn_tool_union_ms(t) for t in turns] + tool = _sum_measured(per_turn) if any(ms is not None for ms in per_turn) else None + + # `duration_seconds` is a non-optional float defaulting to 0.0, so there is + # no None arm to write — but a 0.0 duration is a run that was never timed, + # and subtracting real buckets from it renders a fabricated negative + # residual. The evalboard keeps that null for the same reason; so do we. + unaccounted = ( + result.duration_seconds * 1000.0 - (startup or 0.0) - (generation or 0.0) - (tool or 0.0) - (teardown or 0.0) + if result.duration_seconds > 0.0 + else None + ) + return TurnTimeBuckets(startup, generation, tool, teardown, unaccounted) + + +def _sum_measured(values: Iterable[float | None]) -> float | None: + """Sum what was measured, or ``None`` when nothing was. + + The Python twin of the evalboard's ``sumMeasured``: a run with no measured + value anywhere returns ``None`` (never measured), while a run that measured + a genuine zero returns ``0.0``. + """ + total: float | None = None + for value in values: + if isinstance(value, (int, float)) and math.isfinite(value): + total = (total or 0.0) + value + return total + + +def _turn_tool_union_ms(turn: TurnRecord) -> float | None: + """One turn's tool execution — the UNION of its main-thread command spans. + + ``None`` when the turn recorded no bounded span at all, which is different + from a turn whose tools took no time. Never the sum: concurrent calls + occupy the wall clock once, and summing them books the overlap twice. + + The span SELECTION is ``streaming.collector.main_thread_tool_spans``, not a + copy of it. That rule (which commands count, and the sub-agent exclusion) + is what the collector measures the generation subtraction and the head and + tail against, so a second typed implementation here is how two surfaces + come to publish two different tool totals for one run. + """ + spans = main_thread_tool_spans(turn.messages, turn.commands) + return union_ms(spans) if spans else None + + def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries]: """Per-variant (scores, durations, tokens, assistant-turns) series, keyed by variant id. diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 07a0ef4c4..9fa5bfeb0 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -22,6 +22,7 @@ from __future__ import annotations +from collections.abc import Iterable from datetime import datetime from coder_eval.models import ( @@ -42,6 +43,54 @@ from coder_eval.timing import busy_ms, decompose_turn +def main_thread_tool_spans( + messages: Iterable[TranscriptMessage], commands: Iterable[CommandTelemetry] +) -> list[tuple[datetime, datetime]]: + """Bounded execution intervals of the MAIN THREAD's tool calls. + + The span set the generation subtraction, the head and the tail are all + measured against, so they cannot disagree about which calls exist. Shared + with ``reports_stats.turn_time_buckets``, which answers the same question + about a finished ``TurnRecord`` — a second typed copy of this rule is how + two report surfaces come to publish two different tool totals for one run. + (``scripts/timing/decompose_run.py`` keeps its own, over raw ``task.json`` + dicts rather than models; that is the sanctioned third reader, and + ``tests/test_timing_close_window.py::TestTheThreeToolUnionsAgree`` pins all + three together.) + + Sub-agent tools are excluded, and that used to be the gap: ``_overhead_ms`` + filtered its GENERATIONS to the main thread and then passed EVERY command, + so its claim to keep all four buckets measuring one thread was true only by + luck. It held because a child nests inside the parent Agent call, whose own + interval the union already covers — but Codex's recovered child tools carry + the CHILD's clock, so nothing made it true by construction. The evalboard's + twin (``toolExecutionMs``) does filter, so the two agreed by accident. + + A sub-agent's tool ids are reachable only through the messages that own + them: a child generation carries ``parent_tool_use_id``, and its + ``tool_use_ids`` are the calls it made. + + An inverted pair (``end`` before ``start``) is dropped here rather than + passed on. ``busy_ms`` would discard it anyway, but ``timing.union_ms`` + documents that it does NOT filter them because its callers do — so this is + the caller keeping that true. + """ + sub_agent_tool_ids = { + tool_id + for m in messages + if isinstance(m, AssistantMessage) and m.parent_tool_use_id is not None + for tool_id in m.tool_use_ids + } + return [ + (c.execution_started_at, c.execution_completed_at) + for c in commands + if c.execution_started_at is not None + and c.execution_completed_at is not None + and c.execution_completed_at >= c.execution_started_at + and c.tool_id not in sub_agent_tool_ids + ] + + def subtract_tool_time( messages: list[TranscriptMessage], spans: list[tuple[datetime, datetime]], @@ -250,38 +299,8 @@ def _overhead_ms( ) def _main_thread_tool_spans(self, messages: list[TranscriptMessage]) -> list[tuple[datetime, datetime]]: - """Bounded execution intervals of the MAIN THREAD's tool calls. - - The span set both the head/tail decomposition and the generation - subtraction are measured against, so they cannot disagree about which - calls exist. - - Sub-agent tools are excluded, and this used to be the gap: ``_overhead_ms`` - filtered its GENERATIONS to the main thread and then passed EVERY - command, so its docstring's claim to keep all four buckets measuring one - thread was true only by luck. It held because a child nests inside the - parent Agent call, whose own interval the union already covers — but - Codex's recovered child tools carry the CHILD's clock, so nothing made - it true by construction. The evalboard's twin (``toolExecutionMs``) does - filter, so the two implementations agreed by accident. - - A sub-agent's tool ids are reachable only through the messages that own - them: a child generation carries ``parent_tool_use_id``, and its - ``tool_use_ids`` are the calls it made. - """ - sub_agent_tool_ids = { - tool_id - for m in messages - if isinstance(m, AssistantMessage) and m.parent_tool_use_id is not None - for tool_id in m.tool_use_ids - } - return [ - (c.execution_started_at, c.execution_completed_at) - for c in self._commands.values() - if c.execution_started_at is not None - and c.execution_completed_at is not None - and c.tool_id not in sub_agent_tool_ids - ] + """This turn's main-thread tool spans, from the reduced ToolEnd stream.""" + return main_thread_tool_spans(messages, self._commands.values()) @staticmethod def _reconciled_messages(messages: list[TranscriptMessage], usage: TokenUsage) -> list[TranscriptMessage]: diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index e89c4161f..e764b5550 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -181,10 +181,14 @@ def union_ms(spans: list[tuple[datetime, datetime]]) -> float: and span building, because their input shapes genuinely differ; only this tail is shared. - It does NOT filter ``end < start``. Both callers already drop those while - building their span lists, so guarding again here would be a second rule - about the same input in a second place; keeping it at the caller preserves - today's behaviour exactly. + It does NOT filter ``end < start``. EVERY caller drops those while building + its span list — ``streaming.collector.main_thread_tool_spans`` (shared by + the collector and the report layer), ``_scrub.py`` and + ``decompose_run.py`` — so guarding again here would be a second rule about + the same input in a second place. That reasoning holds only while it stays + true of every caller: a new one that skips the check gets whatever + ``busy_ms`` does with an inverted pair, which is to discard it, but + silently rather than by this function's stated contract. """ if not spans: return 0.0 diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py index b468d8a0e..fcc2ade08 100644 --- a/tests/test_reports_html.py +++ b/tests/test_reports_html.py @@ -2,13 +2,14 @@ from __future__ import annotations -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path import pytest from coder_eval.models import ( AgentKind, + AssistantMessage, CommandStatistics, CommandTelemetry, CriterionResult, @@ -1337,3 +1338,253 @@ def test_no_surface_publishes_a_fabricated_zero(self): html = HTMLReportGenerator.generate_experiment_html(self._all_ungraded(["v1"]), None) assert "n/a" in html assert "0.0%" not in html + + +class TestGenerationMetricsBuckets: + """The offline report carries the same four buckets as the evalboard. + + `reports_html` is described in CLAUDE.md as the evalboard's static twin, and + it rendered only Total Latency / Turns / Avg Turn Latency — so anyone + reading the artifact rather than the dashboard got none of the wall-clock + accounting. The arithmetic lives in `reports_stats.turn_time_buckets`; this + asserts the rendering AND, through it, that arithmetic. + """ + + BASE = datetime(2026, 1, 1, 12, 0, 0) + + @classmethod + def _at(cls, ms: float) -> datetime: + return cls.BASE + timedelta(milliseconds=ms) + + @staticmethod + def _stat(html: str, label: str) -> str: + """The rendered VALUE of one stat card, by its label.""" + import re + + match = re.search(rf'
{re.escape(label)}[^<]*
\s*
([^<]*)
', html) + assert match is not None, f"no stat card labelled {label!r}" + return match.group(1) + + @classmethod + def _turn( + cls, + *, + startup: float | None, + teardown: float | None, + generations: list[tuple[float, float, float | None]], + tools: tuple[float, float] | None = None, + sub_agent: tuple[float, float, float] | None = None, + ) -> TurnRecord: + messages: list = [ + AssistantMessage(started_at=cls._at(lo), completed_at=cls._at(hi), generation_duration_ms=gen) + for lo, hi, gen in generations + ] + commands: list[CommandTelemetry] = [] + if tools is not None: + lo, hi = tools + commands.append( + CommandTelemetry( + tool_name="Bash", + tool_id="main-1", + timestamp=cls._at(lo), + execution_started_at=cls._at(lo), + execution_completed_at=cls._at(hi), + result_status="success", + ) + ) + if sub_agent is not None: + lo, hi, gen = sub_agent + messages.append( + AssistantMessage( + started_at=cls._at(lo), + completed_at=cls._at(hi), + generation_duration_ms=gen, + parent_tool_use_id="agent-call", + tool_use_ids=["child-1"], + ) + ) + commands.append( + CommandTelemetry( + tool_name="Bash", + tool_id="child-1", + timestamp=cls._at(lo), + execution_started_at=cls._at(lo), + execution_completed_at=cls._at(hi), + result_status="success", + ) + ) + return TurnRecord( + iteration=1, + user_input="go", + agent_output="done", + commands=commands, + messages=messages, + harness_startup_ms=startup, + harness_teardown_ms=teardown, + ) + + def test_each_bucket_is_summed_across_turns(self): + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result( + iterations=[ + self._turn(startup=500.0, teardown=100.0, generations=[(500, 1500, 800.0)], tools=(600, 800)), + self._turn(startup=300.0, teardown=50.0, generations=[(2000, 3000, 1000.0)], tools=(2100, 2400)), + ] + ) + buckets = turn_time_buckets(result) + assert buckets.startup_ms == pytest.approx(800.0) + assert buckets.teardown_ms == pytest.approx(150.0) + assert buckets.generation_ms == pytest.approx(1800.0) + assert buckets.tool_ms == pytest.approx(500.0), "200ms + 300ms, each turn's own union" + + def test_each_label_renders_its_own_value(self): + """Pins the label-to-value WIRING, not just that five cards exist. + + Asserting presence alone would pass if `Startup` rendered + `buckets.teardown_ms` — and the unmeasured-bucket test below compares + two `None`s, so a swap is invisible there too. These are five distinct + numbers precisely so a mix-up cannot hide. + """ + result = _make_result( + iterations=[self._turn(startup=500.0, teardown=100.0, generations=[(500, 1500, 800.0)], tools=(600, 800))] + ) + html = HTMLReportGenerator().generate_task_html(result) + assert self._stat(html, "Startup") == "500ms" + assert self._stat(html, "Generation") == "800ms" + assert self._stat(html, "Tool exec") == "200ms" + assert self._stat(html, "Teardown") == "100ms" + # 90s task minus 1.6s of measured buckets. + assert self._stat(html, "Unaccounted") == "88.40s" + + def test_an_unmeasured_bucket_renders_a_dash_not_zero(self): + """A run recorded before the head/tail existed measured nothing. + + `0ms` would claim a measurement nobody took — the same distinction + CE058 enforces in `src/`, and the reason the evalboard's `sumMeasured` + returns null. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result(iterations=[self._turn(startup=None, teardown=None, generations=[(500, 1500, 800.0)])]) + buckets = turn_time_buckets(result) + assert buckets.startup_ms is None + assert buckets.teardown_ms is None + + html = HTMLReportGenerator().generate_task_html(result) + assert self._stat(html, "Startup") == "—" + assert self._stat(html, "Teardown") == "—" + + def test_a_measured_zero_still_renders_as_zero(self): + """The control for the dash: `None` and `0.0` must stay distinguishable. + + Asserted through the RENDERER, not just the arithmetic — the dash is a + rendering decision, so its counterexample has to be one too. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result(iterations=[self._turn(startup=0.0, teardown=0.0, generations=[(500, 1500, 800.0)])]) + assert turn_time_buckets(result).startup_ms == 0.0 + + html = HTMLReportGenerator().generate_task_html(result) + assert self._stat(html, "Startup") == "0ms" + assert self._stat(html, "Teardown") == "0ms" + + def test_an_unmeasured_bucket_still_counts_as_zero_in_the_residual(self): + """Display and arithmetic differ on purpose. + + A bucket nobody measured shows as a dash but sums as 0.0, so the + missing time surfaces in Unaccounted rather than vanishing. That is the + rule `scripts/timing/decompose_run.py::_turn_buckets` already applies. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result(iterations=[self._turn(startup=None, teardown=None, generations=[(500, 1500, 800.0)])]) + # 90s task, 800ms of generation, nothing else measured. + assert turn_time_buckets(result).unaccounted_ms == pytest.approx(90_000.0 - 800.0) + + def test_a_negative_residual_is_rendered_signed_not_clamped(self): + """Real, and it means generation and tool execution OVERLAPPED. + + The fixture has to PRODUCE the overlap rather than manufacture the sign + some other way, or it tests the formatter and not the condition the + message names: a 60 s generation and a 60 s tool inside a 90 s task sum + past the task's own wall clock, which is what overlapping looks like in + the buckets. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result( + iterations=[self._turn(startup=0.0, teardown=0.0, generations=[(0, 60_000, 60_000.0)], tools=(0, 60_000))] + ) + buckets = turn_time_buckets(result) + assert buckets.generation_ms == pytest.approx(60_000.0) + assert buckets.tool_ms == pytest.approx(60_000.0), "the two overlap in wall clock" + assert buckets.unaccounted_ms is not None and buckets.unaccounted_ms < 0 + + html = HTMLReportGenerator().generate_task_html(result) + assert self._stat(html, "Unaccounted").startswith("-"), "a negative residual must keep its sign" + + def test_sub_agent_generations_and_their_tools_are_excluded(self): + """The same main-thread filter the collector and the evalboard apply. + + The spawning Agent call's own interval already spans the child's run, + so counting either books it twice. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result( + iterations=[ + self._turn( + startup=500.0, + teardown=100.0, + generations=[(500, 1500, 800.0)], + tools=(600, 800), + sub_agent=(3000, 3400, 400.0), + ) + ] + ) + buckets = turn_time_buckets(result) + assert buckets.generation_ms == pytest.approx(800.0), "the child's 400ms is not main-thread generation" + assert buckets.tool_ms == pytest.approx(200.0), "and its tool is not a main-thread span" + + def test_the_existing_four_stats_are_unchanged(self): + result = _make_result(iterations=[self._turn(startup=500.0, teardown=100.0, generations=[(500, 1500, 800.0)])]) + html = HTMLReportGenerator().generate_task_html(result) + for label in ("Total Latency", "Turns", "Assistant Turns", "Avg Turn Latency"): + assert self._stat(html, label), f"missing or empty {label} stat" + + def test_a_turn_that_recorded_no_tool_span_has_no_tool_total(self): + """`0ms` would claim the tools were measured and took no time. + + A turn that ran tools none of which were timed is indistinguishable + from one that ran none, so the presence of a SPAN decides — the same + None-vs-0 distinction CE058 enforces in `src/`. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result(iterations=[self._turn(startup=500.0, teardown=100.0, generations=[(500, 1500, 800.0)])]) + assert turn_time_buckets(result).tool_ms is None + assert self._stat(HTMLReportGenerator().generate_task_html(result), "Tool exec") == "—" + + def test_a_run_with_no_duration_has_no_residual(self): + """Subtracting real buckets from an untimed run fabricates a negative. + + `duration_seconds` defaults to 0.0 rather than None, so the guard has to + be on the value. The evalboard keeps its own residual null for exactly + this case. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result(iterations=[self._turn(startup=500.0, teardown=100.0, generations=[(500, 1500, 800.0)])]) + result.duration_seconds = 0.0 + assert turn_time_buckets(result).unaccounted_ms is None + assert self._stat(HTMLReportGenerator().generate_task_html(result), "Unaccounted") == "—" + + def test_a_run_with_no_turns_does_not_raise(self): + from coder_eval.reports_stats import turn_time_buckets + + buckets = turn_time_buckets(_make_result(iterations=[])) + assert buckets.generation_ms is None + assert buckets.tool_ms is None + HTMLReportGenerator().generate_task_html(_make_result(iterations=[])) From 73c3b1171bf62f8ad2f1486c97c59f1efd79c789 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 07:20:18 -0700 Subject: [PATCH 34/54] fix: code review fixes for turn-timing-p0-p3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent final reviews over the whole 7-phase change. No Critical and no High: the Phase 4 x Phase 5 interaction was attacked directly (a re-seeded mark landing inside a tool span; an open call clipped differently now that the collector subtracts) and the algebra holds on every harness. The findings that mattered were all the same shape — a claim that had stopped being true: * `tests/test_timing_identity_contract.py` was a FIFTH tool-union implementation that disagreed with the other four. It filtered generations to the main thread and then unioned every command, so the sensor built to police this identity was asserting a different one. Latent only because no case has a sub-agent command yet — the first one added would have reported a false regression. It now calls production's own `main_thread_tool_spans`. * CE061's docstring and violation MESSAGE still described the architecture Phase 5 deleted: a permanent claude-code suppression that no longer exists, and an instruction to subtract the tool union inside the reducer, which CE063 now forbids and which would recreate double subtraction. Both models flagged it independently. It now states what it owns and points at CE063 for the rest. * `HARNESS_PARITY.md`'s `[^identity]` footnote still said the only committed sensor is one-sided, in the same file that gained 269 lines describing the two-sided one. All three sensors are now named with what each can and cannot see. * The `opencode_c_multi_step_tiling` exemption claimed "the snapshot still records [the tiling]". It does not: `SCRUB_KEYS` masks both bounds and the duration, so nothing about where a window opened survives into the JSON. The comment now says what the snapshot actually pins (structure, blocks, tokens) and where the tiling IS asserted. Also fixed, from the same pass: antigravity's signal is the first MODEL-source `Step` and the table said "the first `Step`"; claude-code's seed docstring still said "the two marks" after Phase 5 deleted the monotonic one; the seed's degradation list did not mention that `include_partial_messages=false` reaches it through `-D`; the TS scanner's comment-stripping blind spot was undeclared; and two counts in `harness-candidates.md` disagreed with the file they describe. CE062 is now documented as deliberately unused. The ids jump 061 to 063, and an id is a permanent anchor — a suppression carrying 062 in an older branch must never start meaning something new. One test was removed rather than repaired. `test_generation_and_tool_time_account_for_the_turn` asserted the buckets cover at least half the turn, on the REAL clock. Phase 4 added the head to that sum and kept the bound; under `-n auto` the denominator inflates while the measured buckets do not, so it failed as a scheduler-noise detector. The share it reached for is asserted exactly, on a scripted clock, in the contract test. NOT fixed, deliberately: a reviewer flagged `EventCollector` retaining `_commands` and `_turn_starts` across a retry's `AgentStartEvent` as High. It is pre-existing and untouched here, and the claimed blast radius is wrong — the persisted record, the reports and `max_turns` all read the agent's OWN collector, which is fresh per `communicate()`. Only `EarlyStopWatcher`'s long-lived collector accumulates, where carrying a turn's whole engagement across retries is arguably what a live verdict wants. Recorded as a follow-up rather than changed blind. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- .claude/harness-candidates.md | 4 +- docs/agents/HARNESS_PARITY.md | 37 +++++++++++++------ .../lib/__tests__/no-zero-coalesce.test.ts | 5 ++- src/coder_eval/agents/claude_code_agent.py | 18 ++++++--- .../rules/ce061_window_via_close_window.py | 36 +++++++++++------- tests/lint/runner.py | 5 +++ tests/test_agent_golden_master.py | 17 +++++++-- tests/test_antigravity_agent.py | 16 +++++++- tests/test_event_collector.py | 11 ++++++ tests/test_timing_identity_contract.py | 23 ++++++------ 10 files changed, 124 insertions(+), 48 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index a0efc75d5..2b709c29a 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -569,10 +569,10 @@ divergences, so the deferred-work record is one place. Measurements in `evalboard/lib/__tests__/no-zero-coalesce.test.ts` is a vitest source scan (there is no eslint in `evalboard/`) over `lib/runs.ts`, `lib/timing.ts` and `_sections.tsx`. It is an ALLOWLIST rather than a ban, exactly because the - residual arithmetic uses `?? 0` correctly — each of the 13 entries carries a + residual arithmetic uses `?? 0` correctly — each of its 14 entries carries a one-line reason, and a new occurrence fails until its author justifies it or keeps the value null. It is keyed on the codebase's own `…Ms` naming - convention rather than on every `?? 0`: a blanket scan matches 60 + convention rather than on every `?? 0`: a blanket scan matches 58 occurrences, ~40 of them token buckets where zero is a fine answer, and an allowlist that long is one nobody reads. Blind spots are declared in the file. Two meta-tests keep it honest — a negative control (so the scan cannot diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 5c539447c..84064a1c8 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -26,7 +26,7 @@ wall clock its numbers account for. |---|---|---|---|---|---| | `generation_duration_ms` RAW window (the reducer's part) | harness clock: previous SDK event → this message | SDK item stamps | harness clock: previous flush → this flush | harness clock: previous `step_finish` → this one | harness clock: previous `turn_end` → this one | | tool time subtracted from it | centrally | centrally | centrally | centrally | centrally | -| what the **first** window covers | the first `message_start`, so CLI boot + TTFT are OUTSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | the first `Step`, so dispatch + TTFT are OUTSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | +| what the **first** window covers | the first `message_start`, so CLI boot + TTFT are OUTSIDE it | the first SDK item's own start, so CLI boot + TTFT are OUTSIDE it | the first MODEL-source `Step`, so dispatch + TTFT are OUTSIDE it | the first `step_start`, so CLI boot + TTFT are OUTSIDE it | the first `turn_start`, so CLI boot + TTFT are OUTSIDE it | | `harness_startup_ms` (turn head) | ~3.6 s — CLI boot fused with TTFT | ~3.1 s — CLI boot fused with TTFT | ~4.7 s — dispatch fused with TTFT (its harness process is spawned once at startup, not per turn) | ~2.5 s — CLI boot fused with TTFT | ~0.23 s — CLI boot fused with TTFT | | `harness_teardown_ms` (turn tail) | ~1.3 s | ~13 ms | ~7 ms | ~26 ms | ~19 ms | | tool `duration_ms` source | measured around the tool result | SDK `completed_at_ms − started_at_ms`; the item's own `duration_ms` only as a fallback | measured ACTIVE → DONE | measured around the tool event | measured around the tool event | @@ -37,16 +37,29 @@ wall clock its numbers account for. | clock basis for recorded stamps | wall bounds, wall duration (raw `datetime.now()`) | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | | window built by `timing.py::close_window` | yes | yes | yes | yes | yes | -[^identity]: "yes" is load-bearing but the committed sensor is one-sided. -`tests/_fixtures/golden_streams/_scrub.py` asserts only `overshoot <= …`, so it -catches a bucket claiming MORE time than the turn contains and says nothing -about one claiming less — an unmeasured bucket passes every test in the suite. -Worse, that suite cannot see the magnitudes at all: `SCRUB_KEYS` masks -`generation_duration_ms` and both bounds to a placeholder, so a golden snapshot -records that a window was measured, never what it measured. The two-sided check -is `scripts/timing/decompose_run.py --max-residual-pct N`, which gates on each -turn's `|residual|` as a share of its own wall clock. It is report-only and -nothing runs it on a schedule; run it by hand against real `task.json` files. +[^identity]: "yes" is load-bearing, and THREE sensors check it, each seeing +something the others cannot. + +`tests/test_timing_identity_contract.py` is the committed two-sided one: it +drives every built-in reducer off a scripted clock, through a real +`EventCollector`, and asserts the four buckets tile the turn to the +MILLISECOND. Magnitudes are only real where a scripted clock makes them real, +which is why it is not in the golden corpus. + +`tests/_fixtures/golden_streams/_scrub.py` replays recorded streams but asserts +only `overshoot <= …` — it catches a bucket claiming MORE time than the turn +contains and says nothing about one claiming less. It cannot be made two-sided +either: those replays run in ~0.3 ms of synthetic wall clock, where a relative +bound is vacuous. Nor can it see magnitudes at all — `SCRUB_KEYS` masks +`generation_duration_ms` and both bounds to a placeholder, so a snapshot records +that a window was measured, never what it measured. That is not a gap to close; +it is why the contract test exists. + +`scripts/timing/decompose_run.py --max-residual-pct N` is the two-sided check on +LIVE runs, gating each turn's `|residual|` as a share of its own wall clock. +`.github/workflows/pr-checks.yml` runs it over the `smoke-pass` bucket's real +`task.json` files, which covers claude-code only (`experiments/default.yaml` +sets that type); run it by hand for the others. **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** All five harnesses can have tool execution inside a generation window, and it is @@ -204,7 +217,7 @@ The per-harness first-output signal: |---|---| | claude-code | the first `message_start` stream event | | codex | the first SDK item's own start | -| antigravity | the first `Step` | +| antigravity | the first MODEL-source `Step` (a SYSTEM/USER Step does not seed) | | opencode | the first `step_start` | | pi | the first `turn_start` | diff --git a/evalboard/lib/__tests__/no-zero-coalesce.test.ts b/evalboard/lib/__tests__/no-zero-coalesce.test.ts index ed82280af..444ef86f9 100644 --- a/evalboard/lib/__tests__/no-zero-coalesce.test.ts +++ b/evalboard/lib/__tests__/no-zero-coalesce.test.ts @@ -35,7 +35,10 @@ import { describe, expect, test } from "vitest"; // * only the three files below are covered; anything else gains no protection; // * a timing field named by NONE of those conventions is invisible — the // convention IS the rule, and it is a convention rather than a type; -// * `?? 0.0`, `|| 0.0` and an `if (x == null) x = 0` assignment are not matched. +// * `?? 0.0`, `|| 0.0` and an `if (x == null) x = 0` assignment are not matched; +// * comment stripping cuts from the FIRST `//` on a line, so a coalesce that +// follows a URL or a `//` inside a string literal is invisible. Cheap to +// hit only on purpose, and the alternative is parsing TypeScript. const here = dirname(fileURLToPath(import.meta.url)); const root = resolve(here, "../.."); diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index e63821a69..114f054d2 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -514,7 +514,7 @@ def _seed_first_generation_window(self) -> None: the head and the generation disjoint so the four-bucket identity still closes. - Without this the two marks are stamped in ``__init__``, BEFORE + Without this the mark is stamped in ``__init__``, BEFORE ``AgentStartEvent`` is emitted, so the head is a small negative that ``decompose_turn`` clamps to ``0.0`` — a clamped inversion published as "measured, and instant", which is the exact confusion CE058 exists to @@ -538,10 +538,18 @@ def _seed_first_generation_window(self) -> None: per-attempt by construction. If a future harness reuses a turn state, the reset belongs there and not here. - A turn with no ``message_start`` — partial streaming off, a mocked - ``query()``, a crash before the first event — never calls this, keeps - the turn-entry mark and clamps to ``0.0`` exactly as before. That is the - correct degradation rather than a gap. + A turn with no ``message_start`` — a mocked ``query()``, a crash before + the first event — never calls this, keeps the turn-entry mark and clamps + to ``0.0`` exactly as before. That is the correct degradation rather + than a gap. + + One route to it is OPERATOR-REACHABLE and worth knowing: this harness + sets ``include_partial_messages=True`` BEFORE spreading + ``**self.config.sdk_options``, so + ``-D agent.sdk_options.include_partial_messages=false`` turns the raw + stream off, and with it this re-seed — the head silently returns to the + clamped ``0.0`` it used to publish. Nothing warns; the degradation is + safe but the number changes meaning. """ if self.first_output_seen: return diff --git a/tests/lint/rules/ce061_window_via_close_window.py b/tests/lint/rules/ce061_window_via_close_window.py index 4a27d5cce..eacd99bb3 100644 --- a/tests/lint/rules/ce061_window_via_close_window.py +++ b/tests/lint/rules/ce061_window_via_close_window.py @@ -21,6 +21,11 @@ where the arithmetic came from. One invariant per id is what makes a ``# noqa`` mean one thing. +NOTE what this rule no longer covers, and deliberately: the tool SUBTRACTION is +not part of a window's geometry any more, so "did this reducer subtract +correctly" is not a question here. CE063 owns it — no module in ``agents/`` may +import ``busy_ms`` at all. + BLIND SPOT, and it is the whole weakness of the chosen shape: this proves the module IMPORTS the helper, never that any particular call used it. The value passed to ``generation_duration_ms=`` is always a local (``generation_ms``, @@ -29,11 +34,16 @@ per-reducer window tests; this rule adds only the cheap structural half that neither can reach — a sixth harness rolling its own. -It costs exactly one permanent suppression. ``claude_code_agent.py`` computes -its window from a monotonic delta and subtracts tool time ONCE at finalization -across every emission, because a call issued by an earlier emission is still -running when the next window closes. Forcing that into ``close_window`` means a -mode flag on a helper whose whole value is having one shape. +It costs NO suppression. It used to cost exactly one: ``claude_code_agent.py`` +computed its window from a monotonic delta and subtracted tool time once at +finalization, because a call issued by an earlier emission is still running when +the next window closes — and forcing that into ``close_window`` would have meant +a mode flag on a helper whose whole value is having one shape. Moving the +subtraction to ``EventCollector.subtract_tool_time`` dissolved the exception: +the collector is already the place where every span is known, so claude-code +needs no separate pass and calls the same shrunken helper as the other four. +``tests/test_custom_lint.py::TestCE061WindowViaCloseWindow::test_the_rule_is_now_exemption_free`` +pins the suppression set EMPTY, so a new exemption has to be argued for. EXEMPT, because both are honest claims that no window was measured: an explicit ``generation_duration_ms=None`` (codex's rollout rebuild, claude-code's @@ -119,13 +129,13 @@ def visit_Call(self, node: ast.Call) -> None: node, f"{name}(...) publishes a measured 'generation_duration_ms' but this module " f"never imports {_TIMING_MODULE}.{_HELPER} — so it is computing a generation " - "window of its own. Every window is the same arithmetic: tile from the mark, " - "keep a backwards stamp from inverting the span, bound the calls still open at " - "the boundary, subtract the UNION of the tool intervals clipped to the window, " - "clamp at zero. Pi got that wrong by measuring from its own turn start, and " - "nothing caught it because the four-bucket identity is only asserted as an " - f"upper bound. Call {_HELPER} instead. If this harness genuinely cannot use it " - "— claude-code subtracts once at finalization across every emission — add " - "'# noqa: CE061' with a comment saying why.", + "window of its own. Every window is the same geometry: tile from the mark, and " + "keep a backwards item stamp from inverting the span. Publish that RAW span; do " + "NOT subtract tool time here — EventCollector.subtract_tool_time does it once, " + "for every harness, and doing it in the reducer too takes it out twice (CE063 " + "guards that half). Pi got the mark wrong by measuring from its own turn start, " + "and nothing caught it because the golden identity check is one-sided; " + "tests/test_timing_identity_contract.py is the two-sided one. " + f"Call {_HELPER} instead.", ) self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index a79bd0faa..2014c7bd5 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -56,6 +56,11 @@ from tests.lint.violation import Violation +# CE062 IS DELIBERATELY UNUSED and must stay that way — the ids above jump 061 +# to 063. It was claimed during the turn-timing work and then folded into CE063 +# rather than shipped. An id is a permanent documentation anchor: a suppression +# comment carrying 062 in an older branch, review or commit message must never +# start meaning something new. Claim 064 next. type RuleClass = type[BaseRule] ALL_RULES: list[RuleClass] = [ diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index d5f017c6c..c250bb0e6 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -67,9 +67,20 @@ # one millisecond, so the tool spans BOTH windows entirely and the # central subtraction takes each down to a measured 0.0. It is the tool # interval that is fictional, not the subtraction — which is why the - # scenario is also in FICTIONAL_DURATIONS. Its point is the TILING (the - # second window opens at the first `step_finish`), and the snapshot - # still records that. + # scenario is in FICTIONAL_DURATIONS too. + # + # BE HONEST ABOUT WHAT IS LEFT. With both exemptions on, this snapshot + # asserts neither the identity nor a positive window, and it does NOT + # record the tiling the scenario is named for — `SCRUB_KEYS` masks + # `started_at`, `completed_at` and `generation_duration_ms`, so nothing + # about where a window opened survives into the JSON. What it still + # pins is the STRUCTURE: two assistant messages, their content blocks, + # their token buckets, and one resolved command. OpenCode's tiling is + # asserted where it can be — `tests/test_timing_identity_contract.py` + # (scripted clock, ms-exact) and + # `tests/test_opencode_agent.py::TestGenerationWindowsTileTheTurn`. + # `pi_c_multi_turn_tiling` is the same scenario shape on a harness whose + # stamps come from its own clock, and it needs neither exemption. "opencode_c_multi_step_tiling", } ) diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index d9528c252..54763f2ee 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -1908,7 +1908,21 @@ async def test_generation_and_tool_time_account_for_the_turn(): assert gen_ms > 0 assert head_ms > 0, "the dispatch before the first Step is now a measured bucket, not 0.0" assert gen_ms + tool_ms + head_ms + tail_ms <= turn_ms - assert gen_ms + tool_ms + head_ms + tail_ms >= 0.5 * turn_ms + + # NO relative LOWER bound. This case runs on the REAL clock, and the fake + # conversation's own overhead is the residual — under parallel load the + # denominator (`duration_seconds`, the agent's monotonic span) inflates + # while the measured buckets do not, so any `>= share * turn_ms` assertion + # is a scheduler-noise detector. It was one: a `>= 0.5 *` bound survived + # here only while the sum excluded the head, and failed under `-n auto` + # once the head joined it. + # + # The share this test was reaching for IS asserted, exactly, in + # tests/test_timing_identity_contract.py — on a scripted clock, where the + # magnitudes are real and the identity closes to the millisecond. What is + # left here is what an end-to-end run can honestly claim: the buckets are + # measured, the head is no longer the clamped 0.0, and nothing overflows + # the turn. async def test_timing_change_moves_no_token_bucket(): diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 10aab05cc..6072c970b 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -832,6 +832,17 @@ def test_a_sub_agent_generation_is_skipped(self): out = subtract_tool_time([self._msg(0, 1000, 900.0, parent_tool_use_id="t1")], [(self._at(0), self._at(500))]) assert out[0].generation_duration_ms == pytest.approx(900.0) + def test_a_crash_partial_with_no_messages_and_live_spans_is_safe(self): + """The shape a crashed turn actually produces. + + `Agent._finalize` builds a record from whatever the collector saw, and + a turn that died before its first emission has resolved tool calls but + NO messages. Nothing to group, nothing to subtract — and no + ZeroDivisionError, no IndexError, and no invented entry. + """ + out = subtract_tool_time([], [(self._at(0), self._at(500))]) + assert out == [] + def test_non_assistant_entries_pass_through_by_identity(self): reconciliation = ReconciliationMessage( input_tokens=1, output_tokens=1, cache_creation_tokens=0, cache_read_tokens=0, note="n" diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index 64c8c8a0a..027a44cf8 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -61,7 +61,7 @@ parse_agent_config, ) from coder_eval.streaming.callbacks import CompositeStreamCallback -from coder_eval.streaming.collector import EventCollector +from coder_eval.streaming.collector import EventCollector, main_thread_tool_spans from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -134,9 +134,9 @@ def assert_identity_closes(turn: Turn) -> None: scripted, so the only slack is float representation. A bound wide enough to absorb a real defect is the sensor this module exists to replace. - Main thread only on the generation side, mirroring the collector and - ``decompose_run.py``: a sub-agent's generations bubble into the same stream - and the spawning Agent call's own interval already spans them. + MAIN THREAD ONLY on both sides, and both through production's own helpers: + a sub-agent's generations bubble into the same stream, and the spawning + Agent call's own interval already spans them and their tools. """ record = _record(turn) span_ms = turn.ended_ms - turn.started_ms @@ -146,13 +146,14 @@ def assert_identity_closes(turn: Turn) -> None: for m in record.messages if isinstance(m, AssistantMessage) and m.parent_tool_use_id is None ) - tool_ms = union_ms( - [ - (c.execution_started_at, c.execution_completed_at) - for c in record.commands - if c.execution_started_at is not None and c.execution_completed_at is not None - ] - ) + # The PRODUCTION selector, not a re-derivation of it. Unioning every command + # would assert a different identity than the collector computes: production, + # the golden sensor, the live residual gate and the HTML report all exclude + # a sub-agent's own tools (the spawning Agent call's interval already spans + # them). No case here has a child command yet, so a local copy stayed green + # while quietly testing something else — and the first sub-agent case added + # would have reported a false regression. + tool_ms = union_ms(main_thread_tool_spans(record.messages, record.commands)) assert record.harness_startup_ms is not None, "a turn that generated has a measured head" assert record.harness_teardown_ms is not None, "a turn that generated has a measured tail" bucket_sum = record.harness_startup_ms + generation_ms + tool_ms + record.harness_teardown_ms From 3b9338c3135cfa39c193569319733aab6b1b2ca1 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 07:20:55 -0700 Subject: [PATCH 35/54] docs(harness): register what the turn-timing run could not guard Six entries, each with why it is not a rule today rather than just what it is. Two are prose-vs-artifact defects a lint rule would have to parse English to catch; one needs a decision about intent before any guard could be right; three are code defects the golden corpus now captures but that were out of the plan's scope to fix. The three-way tool-union divergence this run also surfaced is NOT here: it was guarded the same day by TestTheThreeToolUnionsAgree, which is the point of the promote-or-defer split. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01DLBDYGjbKkJ4Xg9a2QtabU --- .claude/harness-candidates.md | 62 +++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 2b709c29a..3e95de17a 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -709,3 +709,65 @@ divergences, so the deferred-work record is one place. Measurements in added on pi and opencode (`test_the_four_bucket_identity_closes_exactly_across_the_boundary`). Caught in: the timing-architecture-standardization final review. + +## From the turn-timing P0–P3 run (2026-09-12) + +- [ ] **A golden scenario's justification comment can contradict its own + snapshot, and nothing notices.** Three did in this run: two orphan-tool + comments asserted bounds the committed JSON plainly carries (`pi_d`, + `opencode_d`), and `opencode_c`'s exemption claimed "the snapshot still + records the tiling" while `SCRUB_KEYS` masks both bounds and the duration. + Each was found by a human/model reading the JSON beside the prose — nothing + mechanically ties an exemption's stated reason to what its snapshot contains. + A rule would have to parse prose, so this is probably not guardable; the cheap + substitute is the review instruction that already exists ("read every new + snapshot before committing") plus the habit of quoting the actual JSON in the + comment. Caught in: turn-timing P0–P3, phases 2 and 5. + +- [ ] **A rationale comment asserting a now-false premise survives a ripple that + updated its siblings.** The "in-process SDK" claim was corrected in six files + and left standing in two (`test_event_collector.py`, + `message-timeline.test.tsx`), one of them directly beside a sibling that WAS + updated. Same shape as CE026/CE047 (doc-surface parity) but over a PHRASE + rather than a symbol, so a rule would be a phrase blocklist with an + ever-growing allowlist. Deferred on cost, not on value — a grep for the retired + phrase in the acceptance criteria is what actually caught these, and that is + cheap to write into a plan. + +- [ ] **`EventCollector` retains `_commands` and `_turn_starts` across a retry's + `AgentStartEvent`**, which resets only `_agent_end`. Pre-existing and NOT + introduced by the timing work. Blast radius is narrower than it first looks: + the persisted record, the reports and `max_turns` all read the AGENT's + collector, which is fresh per `communicate()`. Only `EarlyStopWatcher`'s + long-lived collector accumulates — where carrying a turn's whole engagement + across retry attempts is arguably what a live "did it engage the skill" + verdict wants, and `_check_round`'s docstring already reasons about crashed + attempts. Needs a decision on intent before any guard. Caught in: turn-timing + P0–P3 final review. + +- [ ] **claude-code has no `TurnClock`.** Its window bounds and span now share + one basis (raw `datetime.now()`), so they cannot disagree with each other — + but both carry the naive-local exposure `TurnClock` exists to remove: a DST + transition or NTP step inside a turn lands directly in a generation window, + and nightly runs are hours long. antigravity and pi already derive wall stamps + from monotonic; codex and opencode cannot (their spans are the CLI's epoch + stamps). claude-code is the one that could and does not. Caught in: turn-timing + P0–P3, phase 5. + +- [ ] **`pi_agent` publishes a `duration_ms` and a subtracted tool SPAN for an + UNRESOLVED orphan.** `_close_tool` guards on `execution_started_at is not + None` while its own comment claims it guards on "resolved", and the + `execution_completed_at` it stamps is only the instant the orphan sweep ran. + claude-code's `_finalize_commands` deliberately leaves the field `None` here, + for the reason CE058 exists. Captured in `pi_d_orphaned_tool.json`. Caught in: + turn-timing P0–P3, phase 2. + +- [ ] **`pi_agent` republishes a turn's content on a duplicate `turn_end`.** + `turn_text_parts` / `turn_tool_ids` are cleared only in `on_turn_start`, so a + second `turn_end` with no intervening start emits the previous turn's text as + its own assistant message and re-lists the same `tool_use_ids`. The TIMING + half of that same reset was deliberately fixed (`turn_started_at` moved into + `on_turn_end`, with a comment making exactly this argument); the content half + was not. Pi retries internally, so a replayed `turn_end` is a transport hiccup + rather than a hypothetical. Captured in `pi_f_duplicate_turn_end.json`. + Caught in: turn-timing P0–P3, phase 2. From 588dea6c0948a16a4668735775117c2b47086932 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 08:36:58 -0700 Subject: [PATCH 36/54] fix(timing): a TurnClock for claude-code, and pi's two captured defects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clears the three entries registered under "From the turn-timing P0–P3 run" in .claude/harness-candidates.md. claude-code now derives every wall stamp a turn records from one injected `TurnClock`: both window bounds, the fallback tool timestamp, and the tool span. Sharing raw `datetime.now()` had already removed the bounds-vs-span disagreement; it left both sides naive-local, where a DST transition or an NTP step inside a turn lands directly in a generation window — an hour-long jump in a millisecond field, on nightly runs that start at 04:18 and last hours. `_resolve_pending_command` takes the reading as an argument rather than reading a clock of its own: it stamps the span that is clipped against those bounds, so a second basis at that one call site would put two clocks inside one subtraction. `turn_start_time` and the turn deadline stay raw monotonic — a deadline must not move when the wall clock steps. One raw `datetime.now()` is left deliberately, on the synthesized sub-agent terminal message, and the code says why: those bounds are an admitted placeholder that `subtract_tool_time` and `_overhead_ms`'s head/tail bracket both exclude, so no arithmetic reads them and there is no basis to share. The clock is INJECTED, not read from a module global. That is load-bearing for the sensor rather than cosmetic: a derived stamp escapes a monkeypatched `datetime`, so the old patch would have left tests/test_timing_identity_contract.py measuring the real clock and passing by accident. It is re-pointed at the injected clock, keeps `time.monotonic` patched (the tool duration is still monotonic-measured), and reverting the conversion now fails it by ~10^7 ms. pi `_close_tool` stamps `execution_completed_at` and derives `duration_ms` only when the status is not UNRESOLVED — the guard the old comment claimed and the code did not have (it tested `execution_started_at is not None`, which an orphan passes). The sweep's instant is not a completion anybody observed, and the manufactured pair read as a measured span the collector took back out of a generation window the tool never occupied. `execution_started_at` is kept: the CLI really did emit that start, and one bound alone forms no span. pi `on_turn_end` clears `turn_text_parts` / `turn_tool_ids` beside `turn_started_at`, on the argument that comment already made — all three have been SPENT into the message just appended. The timing half of that reset had a unit test that stayed green while the content half republished the previous turn's text as its own assistant message and re-listed the same `tool_use_ids`, so the two are now asserted separately. Both pi defects were captured in committed goldens. Regenerated with GOLDEN_REGEN=1, and the run before it failed on exactly those two scenarios: pi_d loses a `duration_ms` and an `execution_completed_at` to `null`, pi_f's second message loses the republished text block. Nothing else moved. Registered but NOT fixed: antigravity stamps a completion on its own orphan sweep the same way (no `duration_ms`). `timing.decompose_turn`'s docstring reasons about that stamp landing in the tail and the antigravity_d residual was measured against it, so it needs its own fixture re-derivation rather than a ride-along. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FpDo37ypvLjLiWXFsEkg6k --- .claude/harness-candidates.md | 66 ++++++++++------ docs/agents/HARNESS_PARITY.md | 63 ++++++++------- src/coder_eval/agents/claude_code_agent.py | 71 +++++++++++------ src/coder_eval/agents/pi_agent.py | 31 +++++++- src/coder_eval/timing.py | 20 +++-- .../expected/pi_d_orphaned_tool.json | 4 +- .../expected/pi_f_duplicate_turn_end.json | 12 +-- tests/_fixtures/golden_streams/pi_fixtures.py | 33 ++++---- tests/test_agent_telemetry.py | 25 +++--- tests/test_command_telemetry_result_data.py | 9 +++ tests/test_pi_agent.py | 79 +++++++++++++++++++ tests/test_timing_identity_contract.py | 50 ++++++------ 12 files changed, 306 insertions(+), 157 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 3e95de17a..94744d3f9 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -745,29 +745,43 @@ divergences, so the deferred-work record is one place. Measurements in attempts. Needs a decision on intent before any guard. Caught in: turn-timing P0–P3 final review. -- [ ] **claude-code has no `TurnClock`.** Its window bounds and span now share - one basis (raw `datetime.now()`), so they cannot disagree with each other — - but both carry the naive-local exposure `TurnClock` exists to remove: a DST - transition or NTP step inside a turn lands directly in a generation window, - and nightly runs are hours long. antigravity and pi already derive wall stamps - from monotonic; codex and opencode cannot (their spans are the CLI's epoch - stamps). claude-code is the one that could and does not. Caught in: turn-timing - P0–P3, phase 5. - -- [ ] **`pi_agent` publishes a `duration_ms` and a subtracted tool SPAN for an - UNRESOLVED orphan.** `_close_tool` guards on `execution_started_at is not - None` while its own comment claims it guards on "resolved", and the - `execution_completed_at` it stamps is only the instant the orphan sweep ran. - claude-code's `_finalize_commands` deliberately leaves the field `None` here, - for the reason CE058 exists. Captured in `pi_d_orphaned_tool.json`. Caught in: - turn-timing P0–P3, phase 2. - -- [ ] **`pi_agent` republishes a turn's content on a duplicate `turn_end`.** - `turn_text_parts` / `turn_tool_ids` are cleared only in `on_turn_start`, so a - second `turn_end` with no intervening start emits the previous turn's text as - its own assistant message and re-lists the same `tool_use_ids`. The TIMING - half of that same reset was deliberately fixed (`turn_started_at` moved into - `on_turn_end`, with a comment making exactly this argument); the content half - was not. Pi retries internally, so a replayed `turn_end` is a transport hiccup - rather than a hypothetical. Captured in `pi_f_duplicate_turn_end.json`. - Caught in: turn-timing P0–P3, phase 2. +- [x] ~~**claude-code has no `TurnClock`.**~~ **RESOLVED.** `_ClaudeTurnState` + now takes an injected `TurnClock` and every wall stamp the turn records + derives from it — both window bounds, the tool span + `_resolve_pending_command` stamps (which takes the reading as an argument, so + the span and the bounds it is clipped against cannot end up on two clocks), + and the fallback tool timestamp. One raw `datetime.now()` is deliberately + left, on the synthesized sub-agent terminal message: those bounds are an + admitted placeholder that `subtract_tool_time` and the head/tail bracket both + exclude, so no arithmetic reads them and there is no basis to share. The + ms-exact sensor was re-pointed at the injected clock rather than the module + `datetime` — a derived stamp escapes a monkeypatch, so the old patch would + have left `tests/test_timing_identity_contract.py` measuring the real clock + and passing by accident; reverting the conversion now fails it by ~10^7 ms. + +- [x] ~~**`pi_agent` publishes a `duration_ms` and a subtracted tool SPAN for an + UNRESOLVED orphan.**~~ **RESOLVED.** `_close_tool` now stamps + `execution_completed_at` and derives `duration_ms` only when the status is + not `UNRESOLVED`; the guard the old comment claimed is the guard the code + has. `execution_started_at` is kept (the CLI really did emit that start) and + one bound alone forms no span, so the orphan no longer has time subtracted + from a generation window it never occupied. `pi_d_orphaned_tool.json` now + records both fields as `null`. + + **Sibling, NOT fixed:** `antigravity_agent` does the same thing at its own + orphan sweep (`tel.model_copy(update={..., "execution_completed_at": + self.clock.now()})`), though it stops short of a `duration_ms`. Deliberately + left: `timing.decompose_turn`'s docstring reasons about that stamped + completion landing inside the tail, and the `antigravity_d_orphaned_tool` + residual was measured against it, so changing it is a separate piece of work + with its own fixture to re-derive — not a ride-along. + +- [x] ~~**`pi_agent` republishes a turn's content on a duplicate `turn_end`.**~~ + **RESOLVED.** `turn_text_parts` / `turn_tool_ids` are now cleared in + `on_turn_end` beside `turn_started_at`, on the argument that comment already + made: all three have been SPENT into the message just appended. + `pi_f_duplicate_turn_end.json` now records the second message with an empty + `content_blocks` and no `tool_use_ids` — it books the duplicate's own usage + and nothing else. The timing half had a unit test that stayed green while the + content half was broken, so the two are now asserted separately + (`test_a_duplicate_turn_end_does_not_republish_the_previous_content`). diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 84064a1c8..4ca8e5f56 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -34,7 +34,7 @@ wall clock its numbers account for. | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | | `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID`; `None` when absent | CLI `responseId`; `None` when absent | | `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | -| clock basis for recorded stamps | wall bounds, wall duration (raw `datetime.now()`) | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | +| clock basis for recorded stamps | one `TurnClock` per turn | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | | window built by `timing.py::close_window` | yes | yes | yes | yes | yes | [^identity]: "yes" is load-bearing, and THREE sensors check it, each seeing @@ -125,14 +125,23 @@ Two consequences worth stating, because both are behaviour changes: (not on `message_id`, which OpenCode and Pi can legitimately leave `None`), subtracts the overlap once, and re-apportions so the parts still sum. -**Three clock bases remain, and the row above says which.** Antigravity and Pi -derive every recorded wall stamp from one `TurnClock` per turn, so a turn's -bounds and the tool spans subtracted from them cannot disagree. Antigravity -needed it: its span was monotonic while its tool intervals were wall, which is -the only reason its window could go negative, and the clamp that caught it was -indistinguishable from a real instant generation. Pi needed it for a different -reason — its stamps were naive-local, so a DST transition or an NTP step inside -a turn lands directly in a generation window. +**Two clock bases remain, and the row above says which.** Antigravity, Pi and +claude-code derive every recorded wall stamp from one `TurnClock` per turn, so +a turn's bounds and the tool spans subtracted from them cannot disagree, and +neither can be moved by a DST transition or an NTP step inside the turn. +Antigravity needed it first: its span was monotonic while its tool intervals +were wall, which is the only reason its window could go negative, and the clamp +that caught it was indistinguishable from a real instant generation. Pi and +claude-code needed it for the other reason — their stamps were naive-local, and +nightly runs start at 04:18 and last hours, so an hour-long jump landing in a +millisecond field is reachable rather than theoretical. + +claude-code has exactly one raw `datetime.now()` left, on the synthesized +sub-agent terminal message. Those bounds are an admitted placeholder for a +generation that arrives as a tool result and is never streamed +(`generation_duration_ms is None`, `parent_tool_use_id` set), which is what +excludes the message from `subtract_tool_time` and from the head/tail bracket. +A stamp no bucket reads has no basis to share. Codex and OpenCode are **not** converted and the hazard is narrowed rather than removed. Their tool spans are the CLI's own epoch-millisecond stamps @@ -141,15 +150,6 @@ cannot be re-derived host-side; converting only the window bounds would put two bases inside one `busy_ms` subtraction, relocating the defect instead of removing it. Both therefore keep the naive-local exposure. -claude-code is the third case and the newest. Its window duration used to be a -monotonic delta while its bounds were wall stamps — the split `TurnClock` -exists to remove — and central subtraction made that untenable, because it -clips WALL tool spans against those WALL bounds. It now measures the span from -the bounds, so the two agree; but the bounds are still raw `datetime.now()`, -so it keeps the same naive-local exposure as codex and opencode, for a -different reason: no epoch-stamp constraint, it simply has not been converted. -That conversion is the remaining improvement here and is not done. - Deadlines on every harness stay on raw `time.monotonic()` and must — a deadline may not move when the wall clock steps. @@ -174,16 +174,23 @@ own. Central subtraction dissolves the special case: the collector is *already* the place where every span is known, so claude-code needs no separate pass and no exemption. -Its window is also now measured on ONE clock. The duration used to be a -monotonic delta while the bounds were wall stamps, which is exactly the split -`TurnClock` exists to eliminate — and it became load-bearing with central -subtraction, which clips WALL tool spans against those WALL bounds. A -monotonic-measured duration would have had the two disagreeing inside one -subtraction, which is the defect that let Antigravity's window go negative. -`turn_start_time` stays monotonic and is untouched: `duration_seconds` and the -turn deadline read it, and a deadline must not move when the wall clock steps. -Adopting a full `TurnClock` here (deriving the wall stamps from monotonic, as -antigravity and pi do) is the remaining improvement and is not done. +Its window is also now measured on ONE clock, and that clock is a `TurnClock`. +The duration used to be a monotonic delta while the bounds were wall stamps, +which is exactly the split `TurnClock` exists to eliminate — and it became +load-bearing with central subtraction, which clips WALL tool spans against +those WALL bounds. A monotonic-measured duration would have had the two +disagreeing inside one subtraction, which is the defect that let Antigravity's +window go negative. Sharing raw `datetime.now()` fixed the disagreement and +left both sides naive-local; deriving both from the turn's monotonic anchor +removes that too. The clock is INJECTED into `_ClaudeTurnState` rather than +read from a module global, because a derived stamp escapes a monkeypatched +`datetime` — a test that patched one would quietly measure the real clock and +pass. `_resolve_pending_command` takes the reading as an argument for the same +reason: it stamps the tool span that is clipped against those bounds, so a +second basis at that one call site would put two clocks inside one subtraction. +`turn_start_time` stays raw monotonic and is untouched: `duration_seconds` and +the turn deadline read it, and a deadline must not move when the wall clock +steps. **The head and tail are measured, not normalized.** Generation and tool are only two of the four buckets. The turn's **head** (turn start → first diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 114f054d2..cd78f649c 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -76,7 +76,7 @@ TurnEndStatus, TurnStartEvent, ) -from coder_eval.timing import close_window +from coder_eval.timing import TurnClock, close_window from coder_eval.utils import dump_dataclass, process_plugins @@ -213,6 +213,7 @@ def __init__( log: PrefixedAdapter, turn_start_time: float, deadline: float | None, + clock: TurnClock | None = None, ) -> None: self._agent = agent self.emit = emit @@ -244,16 +245,23 @@ def __init__( self.sequence_number = 0 self.last_assistant_message_index: int | None = None - # ONE clock basis for the window. The duration used to be a MONOTONIC - # delta while these bounds were wall, which is the split - # `timing.TurnClock` exists to eliminate: the central subtraction clips - # WALL tool spans to these WALL bounds, so a monotonic-measured - # duration would have the two disagreeing inside one subtraction — - # exactly the defect that let antigravity's window go negative. - # `turn_start_time` stays monotonic and is untouched: `duration_seconds` - # and the turn deadline read it, and a deadline must not move when the - # wall clock steps. - self.last_event_wall: datetime = datetime.now() + # ONE clock per turn, and every wall stamp this turn records derives + # from it — the window bounds below, the tool spans + # `_resolve_pending_command` stamps, the fallback tool timestamp. The + # central subtraction clips those WALL tool spans to these WALL window + # bounds, so the two sharing one basis is what keeps the arithmetic + # meaningful; before `TurnClock` they shared only naive-LOCAL + # `datetime.now()`, which a DST transition or an NTP step inside a turn + # lands directly in a generation window — an hour-long jump in a + # millisecond field, on runs that start at 04:18 and last hours. + # Injectable so a test supplies a fake rather than monkeypatching this + # module's `datetime` global, which a derived stamp silently escapes — + # leaving the test passing against the real clock. + # `turn_start_time` stays raw monotonic and is untouched: + # `duration_seconds` and the turn deadline read it, and a deadline must + # not move when the wall clock steps. + self.clock = clock or TurnClock() + self.last_event_wall: datetime = self.clock.now() # Re-seeded ONCE, at the first observed model output. See # `_seed_first_generation_window`. self.first_output_seen: bool = False @@ -323,7 +331,7 @@ def dispatch(self, message: Message) -> None: def on_assistant_message(self, message: Message) -> None: """Capture ToolUseBlocks + build the AssistantMessage telemetry record.""" - message_arrival_wall = datetime.now() + message_arrival_wall = self.clock.now() generation_started_wall = self.last_event_wall current_turn_index = len(self.sdk_messages) @@ -554,7 +562,7 @@ def _seed_first_generation_window(self) -> None: if self.first_output_seen: return self.first_output_seen = True - self.last_event_wall = datetime.now() + self.last_event_wall = self.clock.now() def on_stream_event(self, message: Message) -> None: """Recover cumulative output_tokens from raw ``message_start`` / @@ -585,7 +593,7 @@ def on_user_message(self, message: Message) -> None: """Process tool results (and a sub-agent's terminal generation) from a tool-result UserMessage. The sub-agent message is appended BEFORE the tool-result loop — its position in ``sdk_messages`` is observable.""" - self.last_event_wall = datetime.now() + self.last_event_wall = self.clock.now() sub_msg = self._agent._synthesize_subagent_terminal_message(message, self.sdk_model_used) if sub_msg is not None: @@ -604,13 +612,14 @@ def on_user_message(self, message: Message) -> None: block.content, self.pending_commands, self.processed_results, + now=self.clock.now(), ) is_error_flag = getattr(block, "is_error", False) or False resolved = self.pending_commands.get(block.tool_use_id, {}).get("telemetry") tool_for_event = resolved or CommandTelemetry( tool_name=tool_name or "unknown", tool_id=block.tool_use_id, - timestamp=datetime.now(), + timestamp=self.clock.now(), result_status="error" if is_error_flag else "success", result_summary=format_payload(block.content), ) @@ -1825,6 +1834,16 @@ def _int(value: Any) -> int: # This generation arrives as a tool result and is never streamed, so no # window exists to measure — None (unknown), not 0.0 (instant). + # + # Deliberately NOT on the turn's `TurnClock`, and the only wall stamp in + # this harness that is not. These two bounds are an admitted + # PLACEHOLDER, not a measurement: `generation_duration_ms is None` and + # `parent_tool_use_id` is set, which is exactly what excludes this + # message from `subtract_tool_time` and from `_overhead_ms`'s + # head/tail bracket. A stamp no arithmetic reads has no basis to share, + # and threading a clock into a `@staticmethod` to produce one would + # claim otherwise. Codex's rollout rebuild stamps the same placeholder + # the same way, for the same reason. now = datetime.now() return AssistantMessageTelemetry( started_at=now, @@ -1849,6 +1868,8 @@ def _resolve_pending_command( content: Any, pending_commands: dict[str, dict[str, Any]], processed_results: set[str], + *, + now: datetime, ) -> None: """Match a tool result back to its pending command and update status/duration. @@ -1858,6 +1879,10 @@ def _resolve_pending_command( content: The result content (string or structured) pending_commands: Map of tool_id -> {telemetry, command_start_time} processed_results: Set of already-processed tool IDs (for duplicate detection) + now: This turn's ``TurnClock`` reading, passed in rather than read + here. The span stamped below is clipped against the window + bounds the same clock produced, so a second basis at this one + call site would put two clocks inside one subtraction. """ # Normalize content to string for storage content_str = str(content) if content is not None else "" @@ -1877,13 +1902,15 @@ def _resolve_pending_command( cmd.result_summary = content_str if content_str else None cmd.result_data = ClaudeCodeAgent._try_parse_json_value(content) - # Wall-clock execution bounds. `execution_completed_at` is now; - # `execution_started_at` is reconstructed by subtracting the - # measured monotonic duration. This avoids storing a separate - # wall-clock start (we don't have one without restructuring - # pending_commands further) while still giving consumers two - # explicit timestamps with the right delta. - cmd.execution_completed_at = datetime.now() + # Wall-clock execution bounds. `execution_completed_at` is the + # turn clock's reading; `execution_started_at` is reconstructed by + # subtracting the measured monotonic duration. This avoids storing + # a separate wall-clock start (we don't have one without + # restructuring pending_commands further) while still giving + # consumers two explicit timestamps with the right delta — and the + # reconstruction is now exact rather than approximate, since the + # turn clock is itself monotonic-derived. + cmd.execution_completed_at = now cmd.execution_started_at = cmd.execution_completed_at - timedelta(milliseconds=duration_ms) if is_error: diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index f53af2ca9..da02b0476 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -457,10 +457,25 @@ def _close_tool( timestamp=self.clock.now(), sequence_number=self.sequence, ) - completed = self.clock.now() - telemetry.execution_completed_at = completed - if telemetry.execution_started_at is not None: - telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 + # Only a RESOLVED tool is timed. An orphan force-closed by + # `close_open_tools` was never observed finishing, so the instant the + # sweep runs is not a completion — stamping it manufactures both an + # `execution_completed_at` and the `duration_ms` derived from it, and + # the pair then reads as a measured span that + # `EventCollector.subtract_tool_time` takes back out of a generation + # window it never actually occupied. `execution_started_at` IS kept: + # the CLI really did emit that start, and one bound alone forms no + # span (`main_thread_tool_spans` requires both). This is the guard the + # old comment here claimed and the code did not have — it tested + # `execution_started_at is not None`, which an orphan passes. + # claude-code's `_finalize_commands` leaves the same field `None` for + # the same reason: unknown status and unknown duration are one fact + # (CE058). + if status is not ToolEndStatus.UNRESOLVED: + completed = self.clock.now() + telemetry.execution_completed_at = completed + if telemetry.execution_started_at is not None: + telemetry.duration_ms = (completed - telemetry.execution_started_at).total_seconds() * 1000 telemetry.result_status = _RESULT_STATUS[status] # Stored untruncated by design (sub-agent returns must survive whole). telemetry.result_summary = summary @@ -637,6 +652,14 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: # at the previous turn's start and publish that whole span a second # time. Reproduced: 3000 ms of generation for a 2000 ms turn. self.turn_started_at = None + # The CONTENT half of the same reset, and the same argument: both + # lists have now been SPENT into the message appended above. + # Cleared only in `on_turn_start`, a second `turn_end` with no + # intervening start re-emitted the previous turn's text as its own + # assistant message and re-listed the same `tool_use_ids`, so one + # tool call appeared to belong to two generations. + self.turn_text_parts = [] + self.turn_tool_ids = [] self.emit( TurnEndEvent( task_id=self.task_id, diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index e764b5550..e5a902134 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -39,11 +39,11 @@ class TurnClock: unioning WALL-clock tool intervals and subtracting one from the other. That is the only reason its window could go negative at all, and the clamp that hid it was indistinguishable from a real instant generation. - * Pi stamped with naive-LOCAL ``datetime.now()``. A DST transition or an - NTP step inside a turn lands directly in a generation window — an - hour-long jump in a millisecond field. Nightly runs start at 04:18 and - run for hours, so it is reachable rather than theoretical. A - monotonic-derived stamp cannot express it. + * Pi stamped with naive-LOCAL ``datetime.now()``, and claude-code did the + same. A DST transition or an NTP step inside a turn lands directly in a + generation window — an hour-long jump in a millisecond field. Nightly + runs start at 04:18 and run for hours, so it is reachable rather than + theoretical. A monotonic-derived stamp cannot express it. It is an EXTRACTION, not an invention: antigravity already captured this exact pair at the top of ``communicate`` and simply did not use it for @@ -70,12 +70,10 @@ class TurnClock: converting only the window bounds would put two bases inside one ``busy_ms`` subtraction — relocating the defect instead of removing it. - claude-code does not use it either, but for no good reason: it has no - epoch-stamp constraint, it simply has not been converted. Its window bounds - and its span now share one basis (raw ``datetime.now()``), so the two cannot - disagree with each other — but both carry the naive-local exposure this - class removes. Converting it is the remaining work; see - docs/agents/HARNESS_PARITY.md. + claude-code DOES use it, and is the third of the three that can. Its one + remaining raw ``datetime.now()`` is the synthesized sub-agent terminal + message, whose bounds are an admitted placeholder that no bucket reads — + see the comment at that call site. """ def __init__(self) -> None: diff --git a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json index e2efaaad4..87f6426ec 100644 --- a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json @@ -4,9 +4,9 @@ "commands": [ { "assistant_turn_index": 1, - "duration_ms": "", + "duration_ms": null, "error_message": "no result observed", - "execution_completed_at": "", + "execution_completed_at": null, "execution_started_at": "", "generation_completed_at": null, "parameters": { diff --git a/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json index 66c7e39dc..c7d047901 100644 --- a/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json +++ b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json @@ -41,17 +41,7 @@ "cache_creation_tokens": 0, "cache_read_tokens": 0, "completed_at": "", - "content_blocks": [ - { - "block_type": "text", - "is_error": false, - "sequence": 0, - "signature": null, - "text": "First.", - "thinking": null, - "tool_use_id": null - } - ], + "content_blocks": [], "generation_duration_ms": "", "input_tokens": 10, "message_id": null, diff --git a/tests/_fixtures/golden_streams/pi_fixtures.py b/tests/_fixtures/golden_streams/pi_fixtures.py index 3dfb54998..2d26d7d6d 100644 --- a/tests/_fixtures/golden_streams/pi_fixtures.py +++ b/tests/_fixtures/golden_streams/pi_fixtures.py @@ -273,16 +273,15 @@ def _build_catalogue() -> list[PiScenario]: # (d) a tool the CLI opens and never resolves — force-closed as `unresolved` # by the orphan sweep at finalization. # - # READ THE SNAPSHOT: it carries a `duration_ms` and BOTH execution bounds, - # and that span is subtracted from the generation window. Its - # `execution_completed_at` is the instant the sweep ran, not a completion - # anybody observed, so the duration is manufactured — and `_close_tool`'s - # own comment ("Only a RESOLVED tool contributes: one force-closed without - # a result was never timed") describes a guard it does not have: the test - # is `execution_started_at is not None`, which an orphan passes. - # claude-code's `_finalize_commands` deliberately leaves `duration_ms` - # None in exactly this case, and says why. Captured rather than fixed: - # this scenario is what makes it visible. + # READ THE SNAPSHOT: the command carries `execution_started_at` (the CLI + # really did emit that start) and NEITHER `execution_completed_at` NOR + # `duration_ms`. Nothing observed this call finishing, so the instant the + # sweep runs is not a completion; stamping it used to manufacture both, and + # the pair then read as a measured span that the collector subtracted from + # a generation window the tool never occupied. One bound alone forms no + # span (`main_thread_tool_spans` requires both), so the window is left + # whole. Same rule as claude-code's `_finalize_commands`: unknown status + # and unknown duration are one fact (CE058). scenarios.append( PiScenario( name="d_orphaned_tool", @@ -321,13 +320,13 @@ def _build_catalogue() -> list[PiScenario]: # reproduced as 3000 ms of generation for a 2000 ms turn. It had a unit test # and no golden. # - # READ THE SNAPSHOT: it records that the TIMING half of that reset is fixed - # and the CONTENT half is not. `turn_text_parts` / `turn_tool_ids` are - # cleared in `on_turn_start` only, so the second `turn_end` publishes the - # first turn's text a second time, as its own assistant message. The - # argument `on_turn_end`'s comment makes for moving `turn_started_at` out of - # `on_turn_start` applies to those two lists unchanged. Captured here rather - # than fixed: this scenario is what makes it visible at all. + # READ THE SNAPSHOT: both halves of that reset are now in `on_turn_end`. + # The second assistant message carries NO content block and an empty + # `tool_use_ids` — it booked the duplicate's own usage and nothing else. + # `turn_text_parts` / `turn_tool_ids` used to be cleared in `on_turn_start` + # only, so the replayed line published the first turn's text a second time + # as its own message; the argument `on_turn_end`'s comment makes for + # `turn_started_at` applies to those two lists unchanged. scenarios.append( PiScenario( name="f_duplicate_turn_end", diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py index 90f013c44..50582537c 100644 --- a/tests/test_agent_telemetry.py +++ b/tests/test_agent_telemetry.py @@ -1361,20 +1361,24 @@ def test_plugin_resolution_does_not_change_that(self, tmp_path): class TestClaudeFirstWindowReseed: """The first `message_start` moves the window mark; a later one must not. - Driven at `_ClaudeTurnState` with both clocks patched off one counter. - claude-code derives the window's DURATION from `time.monotonic()` and its - BOUNDS from `datetime.now()`, so patching one leaves the other real and - these tests would measure nothing while still passing. + Driven at `_ClaudeTurnState` with both clocks moved off one counter. The + window's BOUNDS come from the turn's `TurnClock`, which is INJECTED — a + derived stamp escapes a monkeypatched module `datetime` entirely, so these + tests would measure the real clock and still pass. Its DURATION side still + reads `time.monotonic()` for `turn_start_time` and the deadline, so that + global is patched off the same counter; leaving it real would straddle a + scripted clock and a live one. """ BASE = datetime(2026, 9, 11, 9, 0, 0) - class _Stepped(datetime): + class _Stepped: + """A `TurnClock` stand-in the test moves by hand, in ms from `BASE`.""" + at_ms = 0.0 - @staticmethod - def now(tz=None): # type: ignore[override] - return TestClaudeFirstWindowReseed.BASE + timedelta(milliseconds=TestClaudeFirstWindowReseed._Stepped.at_ms) + def now(self): + return TestClaudeFirstWindowReseed.BASE + timedelta(milliseconds=self.at_ms) def _state(self, monkeypatch): from coder_eval.agents import claude_code_agent as claude_module @@ -1382,9 +1386,7 @@ def _state(self, monkeypatch): from coder_eval.streaming.callbacks import CompositeStreamCallback from coder_eval.streaming.collector import EventCollector - stepped = self._Stepped - stepped.at_ms = 0.0 - monkeypatch.setattr(claude_module, "datetime", stepped) + stepped = self._Stepped() monkeypatch.setattr(claude_module, "time", SimpleNamespace(monotonic=lambda: stepped.at_ms / 1000.0)) agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits")) @@ -1400,6 +1402,7 @@ def _state(self, monkeypatch): log=agent._log, turn_start_time=0.0, deadline=None, + clock=stepped, ) @staticmethod diff --git a/tests/test_command_telemetry_result_data.py b/tests/test_command_telemetry_result_data.py index 06b72be9e..7fa214a9a 100644 --- a/tests/test_command_telemetry_result_data.py +++ b/tests/test_command_telemetry_result_data.py @@ -94,6 +94,7 @@ def test_resolve_pending_command_populates_result_data_for_json_object() -> None content, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] @@ -112,6 +113,7 @@ def test_resolve_pending_command_does_not_truncate_long_result_summary() -> None content, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] @@ -130,6 +132,7 @@ def test_resolve_pending_command_populates_result_data_for_json_array() -> None: content, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] @@ -147,6 +150,7 @@ def test_resolve_pending_command_leaves_result_data_none_for_plain_text() -> Non content, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] @@ -168,6 +172,7 @@ def test_resolve_pending_command_populates_result_data_for_flow_debug_fixture() content, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] @@ -191,6 +196,7 @@ def test_resolve_pending_command_handles_sdk_list_content_shape() -> None: content, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] @@ -211,6 +217,7 @@ def test_resolve_pending_command_concatenates_multiple_text_blocks() -> None: content, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] @@ -229,6 +236,7 @@ def test_resolve_pending_command_list_without_text_blocks_yields_none() -> None: content, pending, set(), + now=datetime.now(), ) assert pending[tool_id]["telemetry"].result_data is None @@ -244,6 +252,7 @@ def test_resolve_pending_command_none_content_yields_none() -> None: None, pending, set(), + now=datetime.now(), ) cmd = pending[tool_id]["telemetry"] diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index f9e6f5dd6..0ccc8b8dc 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -1410,6 +1410,85 @@ def test_a_duplicate_turn_end_does_not_republish_the_previous_window(self): assert messages[1].started_at == messages[0].completed_at assert sum(m.generation_duration_ms or 0.0 for m in messages) == pytest.approx(2000.0) + def test_a_duplicate_turn_end_does_not_republish_the_previous_content(self): + """The CONTENT half of the same reset, and the same argument. + + `turn_text_parts` / `turn_tool_ids` were cleared in `on_turn_start` + only, so the replayed line re-emitted the first turn's text as its own + assistant message and re-listed the same `tool_use_ids` — one tool call + appearing to belong to two generations, and the text counted twice by + anything that reads the transcript. The sibling above pinned the timing + half while this one silently stayed broken, which is why it is asserted + separately rather than folded in. + """ + clock = _SteppedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) + state.on_turn_start() + state.on_message_update( + {"assistantMessageEvent": {"type": "text_delta", "delta": "First."}}, + ) + state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) + clock.at_ms = 1000 + state.on_turn_end(_turn_end_payload()) + clock.at_ms = 2000 + state.on_turn_end(_turn_end_payload()) # no intervening `turn_start` + + messages = [m for m in state.messages if m.role == "assistant"] + assert len(messages) == 2 + assert [b.text for b in messages[0].content_blocks if b.block_type == "text"] == ["First."] + assert messages[0].tool_use_ids == ["c1"] + assert messages[1].content_blocks == [] + assert messages[1].tool_use_ids == [] + + def test_an_unresolved_orphan_is_not_given_a_completion_or_a_duration(self): + """Force-closing is not observing a completion. + + The orphan sweep runs at finalization; stamping its instant as + `execution_completed_at` manufactures a bound, and the `duration_ms` + derived from it is the distance to whenever the sweep happened to run. + The pair then reads as a measured span that + `EventCollector.subtract_tool_time` takes back out of a generation + window the tool never occupied. `execution_started_at` IS kept: the CLI + really did emit that start, and one bound alone forms no span. Same + rule as claude-code's `_finalize_commands` — unknown status and unknown + duration are one fact (CE058). + """ + clock = _SteppedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) + state.on_turn_start() + clock.at_ms = 500 + state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) + clock.at_ms = 4000 + closed: list[CommandTelemetry] = [] + state.bind(lambda e: closed.append(e.tool) if isinstance(e, ToolEndEvent) else None) + state.close_open_tools() + + assert len(closed) == 1 + assert closed[0].result_status == "unknown" + assert closed[0].execution_started_at == _SPAN_BASE + timedelta(milliseconds=500) + assert closed[0].execution_completed_at is None + assert closed[0].duration_ms is None + + def test_a_resolved_tool_still_gets_both_bounds_and_a_duration(self): + """The guard narrows the UNRESOLVED case only. + + Without this, deleting the whole stamping block would leave the sibling + above green while every real tool call lost its timing. + """ + clock = _SteppedClock() + state = _PiTurnState(task_id="t", iteration=1, user_input="go", model="m", clock=clock) + state.on_turn_start() + clock.at_ms = 500 + state.on_tool_execution_start({"toolCallId": "c1", "toolName": "bash", "args": {}}) + clock.at_ms = 1200 + closed: list[CommandTelemetry] = [] + state.bind(lambda e: closed.append(e.tool) if isinstance(e, ToolEndEvent) else None) + state.on_tool_execution_end({"toolCallId": "c1", "result": "ok"}) + + assert len(closed) == 1 + assert closed[0].execution_completed_at == _SPAN_BASE + timedelta(milliseconds=1200) + assert closed[0].duration_ms == pytest.approx(700.0) + def test_a_turn_that_never_finishes_does_not_advance_the_mark(self): """The half of this that is still the reducer's job. diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index 027a44cf8..e523fdf3e 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -28,15 +28,15 @@ per-harness suites (this module reuses their idiom rather than inventing a fourth): -* an injected ``TurnClock`` — pi and antigravity take ``clock=`` / build one - through a patched ``TurnClock`` factory; +* an injected ``TurnClock`` — pi, antigravity and claude-code take ``clock=`` + / build one through a patched ``TurnClock`` factory; * a ``datetime`` SUBCLASS monkeypatched onto the module — opencode, which also calls ``datetime.fromtimestamp`` through the same global (see ``tests/test_opencode_agent.py``'s ``_SteppedClock`` for why a stub breaks); -* ``time.monotonic`` AND ``datetime`` both patched — claude-code. Its window is - wall-derived now, but ``turn_start_time`` and the turn deadline still read - ``time.monotonic()``, so patching only one leaves the reducer straddling a - real clock and a scripted one. +* ``time.monotonic`` patched ON TOP of an injected clock — claude-code, whose + ``turn_start_time``, turn deadline and measured tool durations still read + ``time.monotonic()``, so scripting only the clock leaves the reducer + straddling a real clock and a scripted one. Codex is the fifth and takes its stamps from SDK epoch milliseconds rather than from any host clock, so its case scripts those stamps directly. @@ -414,17 +414,22 @@ def _codex_turn() -> Turn: # -------------------------------------------------------------------------- -# claude-code — BOTH the monotonic and the wall clock patched +# claude-code — an injected TurnClock, plus the monotonic global # -------------------------------------------------------------------------- def _claude_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: """A tool call between two emissions, with a real head and a real tail. - Both module globals are patched off one counter. The window itself is - wall-derived, but ``turn_start_time`` and the deadline still read - ``time.monotonic()``, so patching only one leaves the reducer straddling a - real clock and a scripted one. + The clock is INJECTED, like pi's and antigravity's: every wall stamp this + reducer records now derives from the turn's ``TurnClock``, and a derived + stamp escapes a monkeypatched module ``datetime`` entirely — the case would + quietly measure the real clock and pass by accident. ``time.monotonic`` is + still patched off the same counter, because ``turn_start_time``, the + deadline and the tool call's own measured duration read it; leaving it real + leaves the reducer straddling a scripted clock and a live one, and the tool + span (a monotonic duration subtracted back off a clock reading) would be + nonsense. The first `message_start` re-seeds the window, so the CLI spawn and the query build before it are head rather than msg0's generation. That a LATER @@ -444,24 +449,18 @@ def _claude_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage as SdkAssistantMessage from tests._fixtures.golden_streams.claude_fixtures import ToolUseBlock, UserMessage, message_start - class _Stepped(datetime): - at_ms = 0.0 - - @staticmethod - def now(tz: Any = None) -> datetime: # type: ignore[override] - return at(_Stepped.at_ms) + clock = _InjectedClock() def _monotonic() -> float: - return _Stepped.at_ms / 1000.0 + return clock.at_ms / 1000.0 - monkeypatch.setattr(claude_module, "datetime", _Stepped) monkeypatch.setattr(claude_module, "time", SimpleNamespace(monotonic=_monotonic)) agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits")) collector = EventCollector() commands: list[CommandTelemetry] = [] - _Stepped.at_ms = 500 # the turn state is built here; the head runs past it + clock.at_ms = 500 # the turn state is built here; the head runs past it state = _ClaudeTurnState( agent, emit=CompositeStreamCallback( @@ -478,15 +477,16 @@ def _monotonic() -> float: log=agent._log, turn_start_time=_monotonic(), deadline=None, + clock=clock, ) # The stream really does put `message_start` before the emission it # announces — the recorded corpus shows it and the SDK guarantees it — and # the FIRST one is what re-seeds the window, so an ordering this case got # wrong would silently stop exercising the re-seed at all. - _Stepped.at_ms = 800 + clock.at_ms = 800 state.on_stream_event(message_start("m1")) - _Stepped.at_ms = 1000 + clock.at_ms = 1000 state.on_assistant_message( SdkAssistantMessage( [ToolUseBlock("c1", "Bash", {"command": "ls"})], @@ -494,11 +494,11 @@ def _monotonic() -> float: message_id="m1", ) ) - _Stepped.at_ms = 1800 # the tool ran for the whole gap + clock.at_ms = 1800 # the tool ran for the whole gap state.on_user_message(UserMessage("c1", False, "ok")) - _Stepped.at_ms = 2000 + clock.at_ms = 2000 state.on_stream_event(message_start("m2")) # does NOT re-seed: once per turn - _Stepped.at_ms = 2500 + clock.at_ms = 2500 state.on_assistant_message(SdkAssistantMessage([], usage={"input_tokens": 10, "output_tokens": 5}, message_id="m2")) state.finalize(_AgentEndStatus.COMPLETED) From 74b157a36457e1a8a7f20b080f97301e0164e586 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 09:21:53 -0700 Subject: [PATCH 37/54] fix(timing): stamp the turn bracket off the turn clock (CE064) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `decompose_turn` computes the head and the tail by subtracting a generation window bound from an AgentStart/AgentEnd timestamp, so the two have to share a basis. The three harnesses that own a `TurnClock` derived their window bounds from it and let the bracket fall back to `StreamEvent.timestamp`'s `default_factory=datetime.now` — a monotonic-derived stamp and a raw wall stamp inside one subtraction, which is the exact split `TurnClock` exists to remove, reintroduced at the one seam the clock did not own. Measured, not hypothetical. Instrumenting `decompose_turn` on a live antigravity turn printed: PROBE tail: elapsed=-0.017000ms busy=0.000000ms raw=-0.017000ms last_completed = 09:05:22.033099 agent_end = 09:05:22.033082 an AgentEndEvent stamped 17 us BEFORE its own last message finished, which cannot happen: the event is constructed strictly after the final flush. `decompose_turn` clamped the negative and published `0.0` — "measured, and instant", the CE058 confusion reached from the other direction — for a harness whose real tail is ~0.1 ms. After the fix the same task records 0.035 ms, a real measurement rather than a clamp. It only showed on one harness because the drift between the two clocks is tens of microseconds, so it can flip a sign only where the true interval is itself that small. Antigravity is the only harness that spawns its process once in `start()` and holds it across turns, so nothing happens between its last flush and its AgentEndEvent; every other harness books a head of 0.2-6 s and a tail of 7-543 ms, where the drift is invisible. Invisible is not absent, so the fix is applied at every clocked site: that is what makes the subtraction single-basis rather than usually-close, which is not a property a millisecond field can rest on. Note this was widened by the previous commit. claude-code's bounds used to be raw `datetime.now()` — the same basis as the events — so its subtraction was single-basis until the TurnClock conversion. CE064 keeps it fixed: in `agents/`, a module that imports `TurnClock` must pass an explicit `timestamp=` to AgentStartEvent/AgentEndEvent. Scope is DERIVED from that import, never a harness list — codex and opencode take their spans from the CLI's own epoch stamps and deliberately have no clock, so a raw `datetime.now()` bracket is consistent with their bounds and the rule must not fire on them; the day either adopts a clock the rule starts applying with no edit here. The rule checks presence, not spelling, because the three harnesses reach their clock three different ways and pinning a spelling would make it a syntax check on their internals; what it removes is the silent case, a default nobody chose, which is the one that shipped. Mutation-checked against the real tree. `_model_ctor.reaches_models_module` is generalized to `reaches_module` so CE064 reuses the binding resolver rather than copying it (the argument that file already makes for CE060/CE061 sharing it). Its relative-import matcher compared a single `rpartition` tail, which was right only while every target was one segment deep and silently missed `coder_eval.streaming.events` outright — a rule blind for a whole file rather than a near miss. It now matches any segment-wise suffix. CE060/CE061 behaviour is unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FpDo37ypvLjLiWXFsEkg6k --- pyproject.toml | 1 + src/coder_eval/agents/antigravity_agent.py | 22 +++- src/coder_eval/agents/claude_code_agent.py | 17 +++ src/coder_eval/agents/pi_agent.py | 9 ++ tests/lint/rules/_model_ctor.py | 45 +++++-- .../rules/ce064_turn_bracket_on_the_clock.py | 116 ++++++++++++++++++ tests/lint/runner.py | 4 +- tests/test_custom_lint.py | 98 +++++++++++++++ 8 files changed, 301 insertions(+), 11 deletions(-) create mode 100644 tests/lint/rules/ce064_turn_bracket_on_the_clock.py diff --git a/pyproject.toml b/pyproject.toml index 57aef68b2..6f6c66a58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -298,6 +298,7 @@ external = [ "CE060", "CE061", "CE063", + "CE064", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index 5d016f4a4..ec7b884a1 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -584,7 +584,24 @@ async def communicate( ) try: - emit.on_event(AgentStartEvent(task_id=task_id, prompt=user_input, iteration=self._iteration, model=model)) + # `timestamp` from the TURN CLOCK, not the event model's raw + # `datetime.now()` default: this bound is subtracted against window + # bounds the same clock produced (`decompose_turn`), and two bases + # in one subtraction is what `TurnClock` exists to remove. Measured + # HERE: this harness's tail came out at -0.017 ms — an end stamped + # 17 us before its own last message finished — which clamped to the + # `0.0` that means "measured, and instant" (CE058). It holds its + # process across turns, so its true tail is ~0.1 ms, which is the + # only scale at which the drift between two clocks can flip a sign. + emit.on_event( + AgentStartEvent( + task_id=task_id, + prompt=user_input, + iteration=self._iteration, + model=model, + timestamp=clock.now(), + ) + ) def _on_turn_timeout() -> None: state.timeout_hit = True @@ -1215,6 +1232,9 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso crash_reason=crash_reason, max_turns_exhausted=status is AgentEndStatus.MAX_TURNS_EXHAUSTED, duration_seconds=time.monotonic() - self.turn_start_time, + # One basis with the window bounds — see the AgentStartEvent + # site in `communicate`. + timestamp=self.clock.now(), ) ) diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index cd78f649c..5715dfc28 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -723,6 +723,9 @@ def finalize(self, status: AgentEndStatus, *, crashed: bool = False, crash_reaso crashed=crashed, crash_reason=crash_reason, duration_seconds=time.monotonic() - self.turn_start_time, + # One basis with the window bounds — see the AgentStartEvent + # site in `communicate`. + timestamp=self.clock.now(), ) ) @@ -1084,6 +1087,20 @@ def capture_stderr(line: str) -> None: prompt=user_input, iteration=self._iteration, model=effective_model, + # Stamped from the TURN CLOCK, not the event model's raw + # `datetime.now()` default. This bound is subtracted against + # window bounds the same clock produced (`decompose_turn`), and + # two bases inside one subtraction is what `TurnClock` exists to + # remove. Measured: antigravity's tail came out at -0.017 ms — + # an `AgentEndEvent` stamped 17 us BEFORE its own last message + # finished, which cannot happen — and `decompose_turn` clamped + # it to the `0.0` that means "measured, and instant" (CE058). + # It only bites where the true interval is smaller than the + # drift between the two clocks, which is the one harness that + # holds its process across turns; the fix belongs at every + # clocked site regardless, since that is what makes the + # subtraction single-basis rather than usually-close. + timestamp=state.clock.now(), ) ) diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index da02b0476..4e6ec2bba 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -768,6 +768,9 @@ def finalize( crashed=crashed, crash_reason=crash_reason, duration_seconds=time.monotonic() - self.started_at, + # One basis with the window bounds — see the AgentStartEvent + # site in `communicate`. + timestamp=self.clock.now(), ) ) @@ -1037,6 +1040,12 @@ def emit(event: StreamEvent) -> None: prompt=user_input, iteration=self._iteration, model=self.config.model, + # One basis with the window bounds this is subtracted + # against — see `timing.TurnClock`. The event model's raw + # `datetime.now()` default put two clocks inside one + # `decompose_turn` subtraction, which clamped a -0.017 ms tail + # to the `0.0` that means "measured, and instant" (CE058). + timestamp=state.clock.now(), ) ) diff --git a/tests/lint/rules/_model_ctor.py b/tests/lint/rules/_model_ctor.py index b5074cd63..da355c9d0 100644 --- a/tests/lint/rules/_model_ctor.py +++ b/tests/lint/rules/_model_ctor.py @@ -28,14 +28,12 @@ AGENTS_ROOT = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]agents[/\\]") _MODELS_MODULE = "coder_eval.models" -_MODELS_TAIL = _MODELS_MODULE.rpartition(".")[2] - # Taken from the model, never spelled here: a rename then moves the rules too. ASSISTANT_MESSAGE = AssistantMessage.__name__ -def reaches_models_module(node: ast.ImportFrom) -> bool: - """True if this `from ... import` reaches `coder_eval.models`. +def reaches_module(node: ast.ImportFrom, module_path: str) -> bool: + """True if this `from ... import` reaches `module_path`. A relative import inside `agents/` (`from ..models import ...`) carries only the tail in `node.module`, so testing the absolute path alone would leave a @@ -43,24 +41,53 @@ def reaches_models_module(node: ast.ImportFrom) -> bool: imports. """ module = node.module or "" - if module.startswith(_MODELS_MODULE): + if module.startswith(module_path): return True - return bool(node.level) and (module == _MODELS_TAIL or module.startswith(f"{_MODELS_TAIL}.")) + if not node.level: + return False + # A relative spelling carries only a TRAILING SLICE of the absolute path, + # and how much of it depends on the dot count: `from ..timing import` gives + # "timing", `from ..streaming.events import` gives "streaming.events". So + # match any suffix of the target, segment-wise, allowing the import to + # continue on into a submodule below it (`..models.criteria`). Comparing a + # single `rpartition` tail was right only while every target was + # one segment deep; it silently missed `coder_eval.streaming.events` + # entirely, which is a rule blind for a whole file rather than a near miss. + segments = module_path.split(".") + spelled = module.split(".") + return any(spelled[: len(segments) - i] == segments[i:] for i in range(1, len(segments))) -def local_bindings(tree: ast.AST, class_name: str) -> set[str]: - """Every local name this module binds `coder_eval.models.` to. +def reaches_models_module(node: ast.ImportFrom) -> bool: + """`reaches_module` pinned to `coder_eval.models` — CE060/CE061's question.""" + return reaches_module(node, _MODELS_MODULE) + + +def bindings_from(tree: ast.AST, class_name: str, module_path: str) -> set[str]: + """Every local name this module binds `.` to. Built per file: caching it across files would leak one module's alias into another's matching. + + Parameterized on the module because CE064 asks the identical question about + `coder_eval.streaming.events` and `coder_eval.timing` rather than about + `coder_eval.models`. Copying the resolver into it would mean a new import + spelling needs three fixes in three rules, and the third is the one that + gets missed — which is the argument this file already makes for CE060 and + CE061 sharing it. """ names: set[str] = set() for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom) and reaches_models_module(node): + if isinstance(node, ast.ImportFrom) and reaches_module(node, module_path): names.update(a.asname or a.name for a in node.names if a.name == class_name) return names +def local_bindings(tree: ast.AST, class_name: str) -> set[str]: + """`bindings_from` pinned to `coder_eval.models` — CE060/CE061's question.""" + return bindings_from(tree, class_name, _MODELS_MODULE) + + def constructor_name(func: ast.expr, names: set[str], class_name: str) -> str | None: """The spelling this call used to name the model, or None if it did not. diff --git a/tests/lint/rules/ce064_turn_bracket_on_the_clock.py b/tests/lint/rules/ce064_turn_bracket_on_the_clock.py new file mode 100644 index 000000000..713f823f9 --- /dev/null +++ b/tests/lint/rules/ce064_turn_bracket_on_the_clock.py @@ -0,0 +1,116 @@ +"""CE064: a clocked harness must stamp its turn BRACKET off that same clock. + +``decompose_turn`` computes the head and the tail by subtracting a generation +window bound from an ``AgentStartEvent`` / ``AgentEndEvent`` timestamp. Those +two stamps therefore have to share a basis, and a reducer that derives its +window bounds from a ``TurnClock`` while letting the bracket fall back to +``StreamEvent.timestamp``'s ``default_factory=datetime.now`` puts a +monotonic-derived stamp and a raw wall stamp inside one subtraction — the exact +split ``timing.TurnClock`` exists to remove, reintroduced at the one seam the +clock does not own. + +MEASURED, not hypothetical. Instrumenting ``decompose_turn`` on a live +antigravity turn printed:: + + PROBE tail: elapsed=-0.017000ms busy=0.000000ms raw=-0.017000ms + last_completed = 09:05:22.033099 + agent_end = 09:05:22.033082 + +an ``AgentEndEvent`` stamped 17 us BEFORE its own last message finished, which +cannot happen: the event is constructed strictly after the final flush. +``decompose_turn`` then clamps the negative to ``0.0`` and publishes it, which +is "measured, and instant" — the CE058 confusion, arrived at from the other +direction. The published ``harness_teardown_ms`` was ``0.0`` for a harness +whose real tail is ~0.1 ms. + +WHY IT ONLY SHOWED ON ONE HARNESS, and why the rule is not scoped to that one: +the drift between the two clocks is tens of microseconds, so it can only flip a +sign where the true interval is itself that small. Antigravity is the only +harness that spawns its process ONCE in ``start()`` and holds it across turns, +so nothing happens between its last flush and its ``AgentEndEvent``; every +other harness books a head of 0.2-6 s and a tail of 7-543 ms, where the drift +is invisible. Invisible is not absent. The fix belongs at every clocked site +because that is what makes the subtraction single-basis rather than +usually-close, and "usually-close" is not a property a millisecond field can +rest on. + +SCOPE IS DERIVED, never listed. The rule applies to a module under +``agents/`` that imports ``TurnClock`` — antigravity, pi and claude-code today. +Codex and OpenCode take their spans from the CLI's own epoch stamps and +deliberately have no ``TurnClock`` (see that class's docstring), so a raw +``datetime.now()`` bracket is CONSISTENT with their bounds and the rule must +not fire on them; the noop agent has no windows at all. The day one of them +adopts a clock, this rule starts applying to it with no edit here — which is +the half a hardcoded harness list would get wrong. + +Separate id from CE058/CE059/CE060/CE061 for the reason CE060 states: one +invariant per id, so a ``# noqa`` means one thing. CE058 is about publishing a +literal for an unknown duration, CE059 about a window built from a single clock +read, CE060 about identity, CE061 about where a window's arithmetic comes from. +This one is about the turn's OUTER bounds, which no other rule looks at — they +all scope to ``AssistantMessage``, and the bracket is not one. + +BLIND SPOT: presence, not correctness. The rule requires ``timestamp=`` to be +passed; it cannot tell ``self.clock.now()`` from ``datetime.now()`` written out +at the call site, because an agent may legitimately reach its clock through any +expression (a local ``clock`` in ``communicate``, ``state.clock`` from the +caller, ``self.clock`` inside the state). Demanding a specific spelling would +make the rule a syntax check on three harnesses' internal structure. What it +removes is the SILENT case — a default nobody chose — which is the one that +shipped. +""" + +import ast + +from coder_eval.streaming.events import AgentEndEvent, AgentStartEvent +from coder_eval.timing import TurnClock +from tests.lint.rules._model_ctor import AGENTS_ROOT, bindings_from, constructor_name, keywords_of +from tests.lint.rules.base import BaseRule +from tests.lint.violation import Violation + + +_EVENTS_MODULE = "coder_eval.streaming.events" +_TIMING_MODULE = "coder_eval.timing" + +# Taken from the classes themselves, never spelled here: a rename moves the +# rule with them, the way CE056 imports IN_CONTAINER_ENV. +_BRACKETS = (AgentStartEvent.__name__, AgentEndEvent.__name__) +_CLOCK = TurnClock.__name__ + + +class TurnBracketOnTheClock(BaseRule): + id = "CE064" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(AGENTS_ROOT.search(filepath)) + self._clocked = False + self._names: dict[str, set[str]] = {} + + def check(self, tree: ast.AST) -> list[Violation]: + if not self._in_scope: + return [] + self._clocked = bool(bindings_from(tree, _CLOCK, _TIMING_MODULE)) + if not self._clocked: + return [] + self._names = {name: bindings_from(tree, name, _EVENTS_MODULE) for name in _BRACKETS} + return super().check(tree) + + def visit_Call(self, node: ast.Call) -> None: + for bracket in _BRACKETS: + name = constructor_name(node.func, self._names[bracket], bracket) + if name is None: + continue + if "timestamp" not in keywords_of(node): + self.violation( + node, + f"{name}(...) leaves 'timestamp' to StreamEvent's default_factory " + "(a raw datetime.now()), but this harness derives its generation-window " + f"bounds from a {_CLOCK}. `timing.decompose_turn` subtracts one from the " + "other to get harness_startup_ms / harness_teardown_ms, so the two bases " + "meet inside one subtraction — measured at -0.017 ms on antigravity, an " + "agent end stamped BEFORE its own last message finished, which " + "decompose_turn then clamped to the 0.0 that means 'measured, and " + "instant' (CE058). Pass timestamp=.now().", + ) + self.generic_visit(node) diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 2014c7bd5..1ea349a1c 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -40,6 +40,7 @@ from tests.lint.rules.ce060_message_id_declared import MessageIdDeclared from tests.lint.rules.ce061_window_via_close_window import WindowViaCloseWindow from tests.lint.rules.ce063_no_busy_ms_in_agents import NoBusyMsInAgents +from tests.lint.rules.ce064_turn_bracket_on_the_clock import TurnBracketOnTheClock from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -60,7 +61,7 @@ # to 063. It was claimed during the turn-timing work and then folded into CE063 # rather than shipped. An id is a permanent documentation anchor: a suppression # comment carrying 062 in an older branch, review or commit message must never -# start meaning something new. Claim 064 next. +# start meaning something new. Claim 065 next. type RuleClass = type[BaseRule] ALL_RULES: list[RuleClass] = [ @@ -108,6 +109,7 @@ MessageIdDeclared, WindowViaCloseWindow, NoBusyMsInAgents, + TurnBracketOnTheClock, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 2183098c6..9192baf54 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4735,3 +4735,101 @@ def test_is_suppressible(self, tmp_path): target = agents / "pi_agent.py" target.write_text("from coder_eval.timing import busy_ms # noqa: CE063\n", encoding="utf-8") assert not check_file(target, [NoBusyMsInAgents]) + + +class TestCE064TurnBracketOnTheClock: + """CE064 flags a clocked reducer that lets its turn BRACKET default. + + `decompose_turn` subtracts a generation-window bound from an + AgentStart/AgentEnd timestamp. A harness that derives the first from a + `TurnClock` and lets the second fall back to `StreamEvent.timestamp`'s + `default_factory=datetime.now` puts two bases inside one subtraction. + Measured on antigravity: a tail of -0.017 ms, an agent end stamped 17 us + before its own last message finished, clamped to the `0.0` that means + "measured, and instant". + """ + + CLOCKED = "from coder_eval.timing import TurnClock\n" + START = "from coder_eval.streaming.events import AgentStartEvent\n" + END = "from coder_eval.streaming.events import AgentEndEvent\n" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/agents/pi_agent.py"): + import ast + + from tests.lint.rules.ce064_turn_bracket_on_the_clock import TurnBracketOnTheClock + + return TurnBracketOnTheClock(filepath).check(ast.parse(src)) + + def test_flags_a_defaulted_agent_start(self): + assert len(self._run(self.CLOCKED + self.START + "e = AgentStartEvent(task_id='t', prompt='p')")) == 1 + + def test_flags_a_defaulted_agent_end(self): + assert len(self._run(self.CLOCKED + self.END + "e = AgentEndEvent(task_id='t', status=s)")) == 1 + + def test_flags_both_brackets_in_one_module(self): + src = ( + self.CLOCKED + self.START + self.END + "a = AgentStartEvent(task_id='t')\nb = AgentEndEvent(task_id='t')\n" + ) + assert len(self._run(src)) == 2 + + def test_accepts_an_explicit_timestamp(self): + src = self.CLOCKED + self.START + "e = AgentStartEvent(task_id='t', timestamp=state.clock.now())" + assert not self._run(src) + + def test_accepts_it_through_any_clock_expression(self): + """Presence, not spelling — see the rule's BLIND SPOT note. + + Three harnesses reach their clock three ways (a `communicate` local, + `state.clock`, `self.clock`); pinning a spelling would make the rule a + syntax check on their internal structure. + """ + for expr in ("clock.now()", "self.clock.now()", "state.clock.now()"): + src = self.CLOCKED + self.END + f"e = AgentEndEvent(task_id='t', timestamp={expr})" + assert not self._run(src), expr + + def test_does_not_fire_on_an_unclocked_harness(self): + """Codex and OpenCode take their spans from the CLI's epoch stamps. + + They deliberately have no `TurnClock`, so a raw `datetime.now()` + bracket is CONSISTENT with their bounds. Firing here would push them + toward the mixed basis the rule exists to prevent. + """ + assert not self._run(self.START + "e = AgentStartEvent(task_id='t', prompt='p')") + + def test_starts_applying_the_day_an_unclocked_harness_adopts_one(self): + # Scope is derived from the import, never a hardcoded harness list. + src = self.START + "e = AgentStartEvent(task_id='t')" + assert not self._run(src, filepath="src/coder_eval/agents/codex_agent.py") + assert len(self._run(self.CLOCKED + src, filepath="src/coder_eval/agents/codex_agent.py")) == 1 + + def test_resolves_an_aliased_import(self): + src = self.CLOCKED + "from coder_eval.streaming.events import AgentEndEvent as Done\n" + "e = Done(task_id='t')" + assert len(self._run(src)) == 1 + + def test_resolves_a_relative_import(self): + src = ( + "from ..timing import TurnClock\nfrom ..streaming.events import AgentStartEvent\ne = AgentStartEvent(t='t')" + ) + assert len(self._run(src)) == 1 + + def test_does_not_fire_outside_agents(self): + src = self.CLOCKED + self.START + "e = AgentStartEvent(task_id='t')" + assert not self._run(src, filepath="src/coder_eval/streaming/collector.py") + + def test_does_not_fire_on_an_unrelated_event(self): + src = self.CLOCKED + "from coder_eval.streaming.events import ToolEndEvent\ne = ToolEndEvent(task_id='t')" + assert not self._run(src) + + def test_is_suppressible(self, tmp_path): + from tests.lint.rules.ce064_turn_bracket_on_the_clock import TurnBracketOnTheClock + from tests.lint.runner import check_file + + agents = tmp_path / "src" / "coder_eval" / "agents" + agents.mkdir(parents=True) + target = agents / "pi_agent.py" + target.write_text( + self.CLOCKED + self.START + "e = AgentStartEvent(task_id='t') # noqa: CE064\n", + encoding="utf-8", + ) + assert not check_file(target, [TurnBracketOnTheClock]) From a5bfeea14cb5c15577b5a41f5e48b56f33a31f63 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 09:33:21 -0700 Subject: [PATCH 38/54] feat(timing): name the setup and grading phases; union a row's tool time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes to make the published numbers mean what they say. 1. A row's EXEC cell is the UNION of its tool calls, not their sum, and goes through the same `toolExecutionMs` the header strip uses so the two cannot answer one question two ways. Summing double-books concurrent calls: one measured antigravity turn issued two `sleep 2` Bash calls overlapping almost entirely and the cell read 4.1s for 2.1s of wall clock — more tool time in one message than the whole task's Tool exec cell, which is impossible on its face. The comment on that line claimed parity with the strip; that stopped being true when `toolExecutionMs` was changed to union and this line was not. Expanding a row still shows each call's own wall clock, so sequential calls add up to the row total and concurrent ones deliberately do not — which is where the concurrency becomes visible. 2. `EvaluationResult.setup_ms` and `grading_ms`, so the evalboard's Unaccounted cell is a residual rather than a name for the setup phase. It held ~1.9s of known, constant orchestrator cost on every row — 10% of a 19s task, and it would read 60% of a 3s one. Measured after the split: 1.9% (claude-code) and 7.3% (pi), and what remains is post-AgentEnd subprocess reaping, post_run, cleanup and persistence, which is why the remainder is larger for the harnesses that drain a CLI. They are TASK-scoped and deliberately NOT a fifth and sixth member of the turn's four buckets. Folding setup into the first turn's `harness_startup_ms` is wrong three times over: it breaks the turn identity (head + generation + tool + tail == the turn's span) by construction; a dialog-mode task runs N turns against ONE setup, so turn 1 would stop being comparable with turns 2..N; and it is not harness time at all — measured at ~1.86s for claude-code and pi alike on the same machine, which is the tell that it is the orchestrator's own. `grading_ms` accumulates on the SuccessChecker rather than at the four orchestrator call sites, so a fifth cannot be added without it, and in a `finally` so a grade that raises still books the time it spent. Both are `None` rather than 0.0 when never measured — an ungraded `execute` row grades nothing (CE058). `setup_ms` is CARRIED on a detached re-grade (a fact about the run, like `duration_seconds`; a re-grade ADOPTS a workspace instead of provisioning one) and `grading_ms` RECOMPUTED (the verdict came from this pass). The fail-closed field partition in tests/test_seed_from_prior_result.py caught both as unclassified, which is that sensor working. The evalboard's own CE058 twin caught the two new `?? 0` subtractions and its staleness check caught the allowlist entry for the line this change removed. Both new row-union tests were mutation-checked: restoring the sum makes them read 4.0s against an expected 3.0s. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FpDo37ypvLjLiWXFsEkg6k --- docs/agents/HARNESS_PARITY.md | 23 ++++ .../__tests__/message-timeline.test.tsx | 111 +++++++++++++++++- .../app/runs/[id]/[...task]/_sections.tsx | 56 ++++++++- evalboard/app/runs/[id]/[...task]/page.tsx | 2 + .../lib/__tests__/no-zero-coalesce.test.ts | 14 ++- evalboard/lib/runs.ts | 19 +++ src/coder_eval/evaluation/checker.py | 30 ++++- src/coder_eval/models/results.py | 28 +++++ src/coder_eval/orchestrator.py | 23 ++++ tests/test_seed_from_prior_result.py | 15 +++ 10 files changed, 305 insertions(+), 16 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 4ca8e5f56..7d971d906 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -214,6 +214,29 @@ figures in the table above are means of six live `tasks/hello_date` turns per harness and move with CLI cache warmth, so read their ORDER OF MAGNITUDE, not the digits. +**Four turn buckets, two task buckets — and they are different scopes.** The +four above tile ONE TURN and their identity (`head + Σgeneration + UNION(tool) ++ tail == the turn's span`) is asserted to the millisecond by +`tests/test_timing_identity_contract.py`. A task's wall clock is longer than +its turns, and the difference is the orchestrator's own work: criterion +discovery, sandbox provisioning, `agent.start()` and `pre_run` before the first +turn; criteria checking, `post_run` and cleanup after the last. Those are +booked as `EvaluationResult.setup_ms` and `EvaluationResult.grading_ms` — +TASK-scoped, deliberately NOT a fifth and sixth member of the turn's four. + +Folding setup into the first turn's `harness_startup_ms` was considered and is +wrong three times over: it would break the turn identity by construction; a +dialog-mode task runs N turns against ONE setup, so turn 1 would stop being +comparable with turns 2..N; and it is not harness time at all — measured at +~1.86 s for claude-code and pi alike on the same machine, which is the tell. + +Naming them is what makes the evalboard's **Unaccounted** cell a residual +rather than a label. It used to hold that ~1.9 s constant on every row, which +reads as 10% of a 19 s task and would read 60% of a 3 s one. Measured after the +split: 1.9% (claude-code) and 7.3% (pi) of task wall clock, and what remains is +post-`AgentEnd` subprocess reaping, `post_run`, cleanup and record persistence — +which is why the remainder is larger for the harnesses that drain a CLI. + **The head means one thing on all five.** It is the wall clock from the turn starting until the harness first observed **model output**, and that instant is also where the harness opens its first generation window — which is what keeps diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx index 55e6bc335..7ca0ef0ae 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx @@ -674,7 +674,116 @@ describe("MessageTimelineSection — Unaccounted cell", () => { renderStrip(10); expect( screen.getByText("Unaccounted").parentElement, - ).toHaveAttribute("title", expect.stringContaining("sandbox setup")); + ).toHaveAttribute("title", expect.stringContaining("post_run")); + }); + + test("it no longer claims to hold the setup phase, which has its own cell", () => { + // The residual used to name sandbox setup as one of its contents, and + // that was ~1.9s of known, constant orchestrator cost on every row — + // a named phase hiding inside a bucket called "unaccounted". + renderStrip(10); + const title = screen + .getByText("Unaccounted") + .parentElement!.getAttribute("title")!; + expect(title).not.toContain("sandbox setup"); + }); + + test("setup and grading are subtracted out of the residual", () => { + render( + , + ); + expect(cell("Setup").textContent).toBe("2.0s"); + expect(cell("Grading").textContent).toBe("500ms"); + // 10s − 1s generation − 2s setup − 0.5s grading = 6.5s. + expect(cell("Unaccounted").textContent).toBe("6.5s (65%)"); + }); + + test("a run predating the fields leaves their time IN the residual", () => { + // The whole point of subtracting only what was measured: an absent + // field must not be silently taken off as a zero, and must not turn + // the residual into a different number than the run used to publish. + render( + , + ); + expect(cell("Setup").textContent).toBe("—"); + expect(cell("Grading").textContent).toBe("—"); + expect(cell("Unaccounted").textContent).toBe("9.0s (90%)"); + }); +}); + +describe("MessageTimelineSection — a row's EXEC cell", () => { + function span(start: number, end: number) { + return { + toolName: "Bash", + toolUseId: `tu_${start}`, + summary: "sleep", + argText: "sleep", + description: null, + genMs: null, + durationMs: end - start, + isError: false, + resultPreview: null, + outputTokens: null, + resultTokens: null, + execStartMs: start, + execEndMs: end, + }; + } + + // The message row lays out GEN then EXEC as the first two numeric spans of + // its own grid; `:scope >` keeps expanded tool sub-rows out of the match. + function execOf(container: HTMLElement): string { + // The message row is `ol > li > details > summary`, laying out + // #, GEN, EXEC as its first three numeric spans. `:scope >` keeps the + // expanded tool sub-rows inside the
body out of the match. + const row = container.querySelector("ol > li > details > summary") as HTMLElement; + const nums = row.querySelectorAll(":scope > span.tabular-nums"); + return nums[2]?.textContent ?? ""; + } + + function execCell(toolUses: ReturnType[]): string { + const { container } = render( + , + ); + return execOf(container); + } + + test("concurrent calls count their overlap ONCE", () => { + // The bug this replaced: two `sleep 2` Bash calls overlapping almost + // entirely rendered 4.1s for 2.1s of wall clock — more tool time in + // one message than the whole task's Tool exec cell, which is + // impossible on its face. + // union 0->3000 = 3.0s; the sum of the two durations would be 4.0s. + expect(execCell([span(0, 2_000), span(1_000, 3_000)])).toBe("3.0s"); + }); + + test("sequential calls still add up, so the row reconciles with its parts", () => { + // Expanding the row shows each call's own wall clock. When they did + // not overlap, those add to this number; when they did, they do not, + // and that difference is the concurrency. + expect(execCell([span(0, 1_000), span(2_000, 3_000)])).toBe("2.0s"); + }); + + test("it agrees with the header for a single message", () => { + const toolUses = [span(0, 2_000), span(1_000, 3_000)]; + const { container } = render( + , + ); + const header = screen.getByText("Tool exec").parentElement!.querySelectorAll("div")[1]; + expect(execOf(container)).toBe(header.textContent); }); }); diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index 671ca4344..6582a3cca 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -320,6 +320,8 @@ export function MessageTimelineSection({ taskDurationSeconds, harnessStartupMs, harnessTeardownMs, + setupMs, + gradingMs, }: { messages: MessageEvent[]; // Per-Agent-call sub-agent token breakdown (input/output/cache-create/ @@ -341,6 +343,15 @@ export function MessageTimelineSection({ // the cells then read "—" while Unaccounted keeps exactly its old meaning. harnessStartupMs?: number | null; harnessTeardownMs?: number | null; + // TASK-scoped phases either side of the turns: provisioning before the + // first turn, criteria checking after the last. Named so Unaccounted is a + // residual instead of a label for the setup phase — it was ~1.9s of known, + // constant orchestrator cost on every row, which reads as 10% of a 19s + // task and would read 60% of a 3s one. Null/absent on a run predating the + // capture, and the cells then read "—" while Unaccounted keeps exactly its + // old meaning. + setupMs?: number | null; + gradingMs?: number | null; }) { // Token columns can be shown as counts or as their estimated USD value. const [unit, setUnit] = useState("tokens"); @@ -422,7 +433,9 @@ export function MessageTimelineSection({ totalGenMs - toolExecMs - (harnessStartupMs ?? 0) - - (harnessTeardownMs ?? 0) + (harnessTeardownMs ?? 0) - + (setupMs ?? 0) - + (gradingMs ?? 0) : null; const unaccountedShare = taskMs != null && taskMs > 0 && unaccountedMs != null @@ -447,13 +460,21 @@ export function MessageTimelineSection({ both sums on one line, where nothing said which total each part belonged to. The time cells are ordered as the turn runs. */}
-
+
Messages
{messageCount}
+
+
+ Setup +
+
+ {fmtMs(setupMs ?? null)} +
+
Startup @@ -486,7 +507,15 @@ export function MessageTimelineSection({ {fmtMs(harnessTeardownMs ?? null)}
-
+
+
+ Grading +
+
+ {fmtMs(gradingMs ?? null)} +
+
+
Unaccounted
@@ -811,6 +840,8 @@ export function CostExplorerSection({ taskDurationSeconds, harnessStartupMs, harnessTeardownMs, + setupMs, + gradingMs, }: { messages: MessageEvent[]; subAgentUsageByToolId?: Record; @@ -821,6 +852,10 @@ export function CostExplorerSection({ // Forwarded verbatim to the timeline's Startup/Teardown cells. harnessStartupMs?: number | null; harnessTeardownMs?: number | null; + // Forwarded straight through to MessageTimelineSection — this component + // renders it and owns no timing of its own. + setupMs?: number | null; + gradingMs?: number | null; }) { const [scale, setScale] = useState(1); const [toolScale, setToolScale] = useState(1); @@ -861,6 +896,8 @@ export function CostExplorerSection({ taskDurationSeconds={taskDurationSeconds} harnessStartupMs={harnessStartupMs} harnessTeardownMs={harnessTeardownMs} + setupMs={setupMs} + gradingMs={gradingMs} /> {model && tokens.total > 0 && (
@@ -1490,8 +1527,17 @@ function MessageRow({ const slowTool = m.toolUses.some((t) => (t.durationMs ?? 0) >= SLOW_TOOL_MS); const hasErrorTool = m.toolUses.some((t) => t.isError); const preview = summaryPreview(m); - // Sum tool exec time for this message — matches the rollup strip. - const execMs = m.toolUses.reduce((a, t) => a + (t.durationMs ?? 0), 0); + // UNION, not sum — and through the same helper the rollup strip uses, so + // the row and the header cannot answer one question two ways. Summing + // double-books concurrent calls: one measured antigravity turn issued two + // `sleep 2` Bash calls overlapping almost entirely, and this cell read + // 4.1s for 2.1s of wall clock — more tool time in one message than the + // whole task's Tool exec cell, which is impossible on its face. The old + // comment here claimed parity with the strip; that stopped being true when + // `toolExecutionMs` was changed to union and this line was not. Expand the + // row to see each call's own wall clock: sequential calls still add up to + // this number, concurrent ones deliberately do not. + const execMs = toolExecutionMs([m]); const hasExec = m.toolUses.some((t) => t.durationMs != null); // Render full body only when something more than the summary exists. const hasBody = diff --git a/evalboard/app/runs/[id]/[...task]/page.tsx b/evalboard/app/runs/[id]/[...task]/page.tsx index 84efa4610..077a00d70 100644 --- a/evalboard/app/runs/[id]/[...task]/page.tsx +++ b/evalboard/app/runs/[id]/[...task]/page.tsx @@ -366,6 +366,8 @@ export default async function TaskPage({ recordedCostUsd={task.totalCostUsd} taskDurationSeconds={task.durationSeconds} harnessStartupMs={task.harnessStartupMs} + setupMs={task.setupMs} + gradingMs={task.gradingMs} harnessTeardownMs={task.harnessTeardownMs} /> )} diff --git a/evalboard/lib/__tests__/no-zero-coalesce.test.ts b/evalboard/lib/__tests__/no-zero-coalesce.test.ts index 444ef86f9..fbdf5cdc4 100644 --- a/evalboard/lib/__tests__/no-zero-coalesce.test.ts +++ b/evalboard/lib/__tests__/no-zero-coalesce.test.ts @@ -89,9 +89,17 @@ const ALLOWED = new Map([ "The residual. Subtracting only what was measured is the whole point; an unmeasured head leaves its time IN the residual rather than silently claiming it.", ], [ - "(harnessTeardownMs ?? 0)", + "(harnessTeardownMs ?? 0) -", "The residual's tail half: subtracting only what was measured leaves unmeasured time IN the residual.", ], + [ + "(setupMs ?? 0) -", + "Same residual rule: a run predating the field leaves its setup time IN the residual rather than having it silently subtracted as zero.", + ], + [ + "(gradingMs ?? 0)", + "Same residual rule, and it is also the ungraded case — `coder-eval execute` grades nothing, so there is no grading time to subtract.", + ], [ "const slowExec = (execMs ?? 0) >= SLOW_TOOL_MS;", "A threshold comparison: an untimed execution is not a slow one, so 0 answers the question asked.", @@ -104,10 +112,6 @@ const ALLOWED = new Map([ "const slowTool = m.toolUses.some((t) => (t.durationMs ?? 0) >= SLOW_TOOL_MS);", "A threshold comparison: an untimed call is not a slow call.", ], - [ - "const execMs = m.toolUses.reduce((a, t) => a + (t.durationMs ?? 0), 0);", - "Summing the measured calls of one message; an untimed call adds nothing to that sum.", - ], ]); // `x ?? 0` / `x || 0` where the coalesced identifier names a measured interval. diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 824316598..04288b119 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -370,6 +370,14 @@ export interface TaskDetail extends TaskResultSummary { // that residual is what is left after every named bucket. harnessStartupMs: number | null; harnessTeardownMs: number | null; + // TASK-scoped phases either side of the agent's turns, so the timeline's + // Unaccounted cell is a residual rather than a name for the setup phase. + // `setupMs` is sandbox provisioning + agent start() + pre_run; `gradingMs` + // is every success-criteria check the row made. Both `null` when the run + // predates the fields or (for grading) when nothing was graded — never 0, + // which would claim the phase ran instantly. + setupMs: number | null; + gradingMs: number | null; // Per-call ACTUAL cost + cache audit rows, grouped by turn iteration. Only // turns whose `provider_call_costs` list is non-empty appear (LiteLLM/ // open-weight backend; empty on Claude/Bedrock). Rendered as a standalone @@ -2522,6 +2530,10 @@ export async function readTaskDetail( success_criteria_results?: RawCriterionResult[]; post_failure_criteria_results?: RawCriterionResult[]; iterations?: TurnEntry[]; + // Task-scoped phases either side of the turns. Absent on runs that + // predate them, which `sumMeasured` maps to null rather than 0. + setup_ms?: number | null; + grading_ms?: number | null; environment_info?: RawRunJson["environment_info"]; }>(path.join(contentDir, "task.json")); @@ -2565,6 +2577,11 @@ export async function readTaskDetail( const subAgentUsageByToolId = aggregateSubAgentUsage(messages); const { startupMs: harnessStartupMs, teardownMs: harnessTeardownMs } = sumHarnessOverhead(task?.iterations ?? []); + // Through `sumMeasured` for the single-value case too, so the None-vs-0 + // and non-finite rules have ONE implementation: a `?? null` here would + // pass a NaN straight into the Unaccounted subtraction. + const setupMs = sumMeasured([task?.setup_ms]); + const gradingMs = sumMeasured([task?.grading_ms]); const taskDescription = task?.task_config?.resolved?.initial_prompt ?? @@ -2607,6 +2624,8 @@ export async function readTaskDetail( subAgentUsageByToolId, harnessStartupMs, harnessTeardownMs, + setupMs, + gradingMs, providerCalls, }; } diff --git a/src/coder_eval/evaluation/checker.py b/src/coder_eval/evaluation/checker.py index 0e7c2c4d2..8ec436a88 100644 --- a/src/coder_eval/evaluation/checker.py +++ b/src/coder_eval/evaluation/checker.py @@ -7,6 +7,7 @@ import asyncio import logging +import time from pathlib import Path from typing import TYPE_CHECKING, Any @@ -93,6 +94,15 @@ def __init__( # Cached turn records - set by check()/check_all() when provided self._turn_records: TurnRecords | None = None self.route = route + # Cumulative wall ms spent grading, across every `check_all_async` call + # this checker serves. Accumulated HERE rather than at the four + # orchestrator call sites (single-shot, evaluate-only, the per-dialog-turn + # check, the post-failure diagnostics) so a fifth call site cannot be + # added without it — the same reason the tool subtraction lives at the + # one collector seam. `None` until something is actually checked, so an + # ungraded row reports "never measured" rather than an instant 0.0 + # (CE058). + self.grading_ms: float | None = None # V3: Lazy initialization - registry loaded here, not at import if init_registry: @@ -205,12 +215,22 @@ async def check_all_async( """ records, ref_dir = self._resolve_refs(turn_records, reference_dir) + # Monotonic, like every other duration in this codebase: a wall-clock + # delta would move if the clock stepped mid-grade, and an `agent_judge` + # criterion can run for minutes. + started = time.monotonic() results: list[CriterionResult] = [] - for criterion in criteria: - if self._is_native_async(criterion.type): - results.append(await self._check_single_async(criterion, records, ref_dir)) - else: - results.append(await asyncio.to_thread(self._check_single, criterion, records, ref_dir)) + try: + for criterion in criteria: + if self._is_native_async(criterion.type): + results.append(await self._check_single_async(criterion, records, ref_dir)) + else: + results.append(await asyncio.to_thread(self._check_single, criterion, records, ref_dir)) + finally: + # In `finally` so a grade that raises still books the time it spent. + # Its cost is what the caller is trying to account for, and a crash + # does not un-spend it. + self.grading_ms = (self.grading_ms or 0.0) + (time.monotonic() - started) * 1000.0 return results def _is_native_async(self, criterion_type: str) -> bool: diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 9b8f152ec..c1efcd718 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -548,6 +548,34 @@ class EvaluationResult(BaseModel): started_at: datetime = Field(description="When evaluation started") completed_at: datetime | None = Field(default=None, description="When evaluation completed") duration_seconds: float = Field(default=0.0, description="Total evaluation duration") + setup_ms: float | None = Field( + default=None, + description=( + "Wall milliseconds from the task starting until the agent phase begins — sandbox " + "setup, agent construction and start(), and any pre_run commands. TASK-scoped, " + "which is why it is here and not a fifth member of TurnRecord's four buckets: " + "those tile ONE TURN and their identity (head + generation + tool + tail == the " + "turn's span) is asserted to the millisecond, while setup happens once for a task " + "that may run N turns. Folding it into the first turn's harness_startup_ms would " + "break that identity by construction AND make turn 1 incomparable with turns " + "2..N. It is also not harness time: it is the orchestrator's own, measured at " + "~1.86s for claude-code and pi alike. Named rather than left in the report's " + "residual because it is a known, measurable phase, and a residual holding a " + "nameable constant is how a number stops meaning what it says." + ), + ) + grading_ms: float | None = Field( + default=None, + description=( + "Wall milliseconds spent checking success criteria, summed across every " + "``SuccessChecker.check_all_async`` call this evaluation made — the single-shot " + "check, the per-dialog-turn checks, and the post-failure diagnostic pass. " + "Accumulated on the checker rather than at the four call sites so a fifth one " + "cannot be added without it. ``None`` on an ungraded row (``coder-eval " + "execute``), where nothing was checked — never 0.0, which would claim a " + "measurement was taken and came back instant (CE058)." + ), + ) # Results final_status: FinalStatus = Field(description="Final status of the evaluation") diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index cf02242ef..5c11d37bd 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -650,6 +650,7 @@ async def run(self) -> EvaluationResult: with task_log_handler(task_log_file, task_id=self._log_task_id) as log_tail: try: # Setup components + setup_started = time.monotonic() await self._setup() # Run pre-run commands inside the sandbox before the agent starts. @@ -658,6 +659,14 @@ async def run(self) -> EvaluationResult: # the run as FinalStatus.ERROR; _run_post_run_commands and # _cleanup still execute via the finally block. await self._run_pre_run_commands() + # Everything before the agent phase, booked as ONE task-level + # bucket. Measured at ~1.86s on this machine for claude-code and + # pi alike, which is the tell that it is the orchestrator's own + # cost rather than any harness's: criterion discovery, sandbox + # setup, agent start(), pre_run. It used to land in the report's + # residual, where a known constant reads as unexplained time — + # 10% of a 19s task, and it would read 60% of a 3s one. + self.result.setup_ms = (time.monotonic() - setup_started) * 1000.0 # Enforce task-level timeout via an OS-thread watchdog that # SIGKILLs the in-flight CLI subprocess AND cancels this @@ -856,6 +865,14 @@ def _seed_from_prior_result(self) -> None: self.result.agent_config = prior.agent_config self.result.expected_commands = prior.expected_commands self.result.simulation = prior.simulation + # The run's own setup cost, for the reason `duration_seconds` is + # restored: it is a fact about the run, not about this pass. A detached + # grade ADOPTS the workspace rather than building one + # (`Sandbox.adopt`), so its own setup is a different activity — writing + # it here would report the re-grade's cheap adoption as the run's + # provisioning. `grading_ms` goes the other way and is deliberately NOT + # carried: the verdict this row now holds came from THIS pass's grading. + self.result.setup_ms = prior.setup_ms # pre_run belongs to the execute phase and is NOT re-run against an # adopted workspace (see _skip_pre_run_for_adopted), so its recorded @@ -1147,6 +1164,12 @@ def _finalize_result(self, start_time: float) -> None: self.result.completed_at = datetime.now() self.result.duration_seconds = time.time() - start_time + # Read off the checker, which accumulated it across every call site it + # served. Stays None when nothing was graded (`coder-eval execute`), + # which is the distinction CE058 is about: no criteria ran, so no + # measurement exists — as opposed to one that came back instant. + if self.success_checker is not None: + self.result.grading_ms = self.success_checker.grading_ms # Re-grade: the row keeps the agent run's duration (see # _seed_from_prior_result). The grading pass's own cost is preserved diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py index 9cfe80e31..8edceecb6 100644 --- a/tests/test_seed_from_prior_result.py +++ b/tests/test_seed_from_prior_result.py @@ -57,6 +57,11 @@ "post_run_results", "sandbox_path", "environment_info", + # A fact about the run being graded, not about the grading pass — the same + # rule `duration_seconds` follows. A detached grade adopts an existing + # workspace instead of building one, so its own "setup" is a different + # activity entirely; overwriting would report it as the run's. + "setup_ms", } # Recomputed by this pass — carrying them would defeat the point. @@ -85,6 +90,12 @@ # when the row reached its final state, which the grade genuinely changes. "duration_seconds", "completed_at", + # The cost of THIS pass's grading, which is the only grading that produced + # the verdict the row now carries. Its sibling `duration_seconds` is + # restored from the prior instead, for the opposite reason: that one is a + # fact about the agent run. The pair is the same split + # `environment_info["grading_duration_seconds"]` already makes. + "grading_ms", } @@ -116,6 +127,10 @@ def _prior() -> EvaluationResult: pre_run_results=[PostRunResult(command="prior-pre", exit_code=0)], post_run_results=[PostRunResult(command="prior-post", exit_code=0)], sandbox_path="/prior/workspace", + # Distinctive, not the model default: the re-grade adopts a workspace + # rather than provisioning one, so the run's own setup cost has to + # survive or the row reports the cheap adoption as the run's. + setup_ms=1234.5, environment_info={"installed_tools": "prior", "coder_eval": "1.0.0-run"}, early_stop=EarlyStopInfo( reason=EarlyStopReason.CRITERION_FAILED, From 6d0122dbdf93fcd82300a691f9b1abde95c22754 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:01:26 -0700 Subject: [PATCH 39/54] fix(timing): setup_ms marks from the task's start, not from _setup() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The field's own description said "from the task starting until the agent phase begins" and the mark sat at `_setup()`, which is not the same instant. Instrumenting the seams on a live claude-code task showed what fell in the gap: get_version_info() 733.0 ms post_run 32.1 ms _cleanup() 1.4 ms _finalize_result (writes task.json) 1.9 ms eval-loop preamble + post-AgentEnd 4.7 ms ------------------------------------------------ residual 758.4 ms (4.3% of wall clock) 97% of the residual was ONE call. `utils.get_version_info()` shells out for the git commit and every CLI's `--version` — timed directly at 733 ms — and it runs while `EvaluationResult` is being constructed, which is before `_setup()` is reached. So the largest item in a phase named "setup" was outside it, and landed in the report's residual where it read as a real unknown. Moving the mark beside `start_time` makes the field mean what it says. Measured after: 42.3 ms, 0.26% of a 16.4 s task, down from 758 ms (4.3%) and from ~1.9 s (10%) before any of this work. What is left is the tail of that table — post_run, sandbox preservation, persistence, post-AgentEnd reaping — with no single nameable phase in it, which is what "unaccounted" should mean. Worth stating plainly rather than leaving implied: `setup_ms` now carries a ~733 ms constant that is instrumentation overhead, not work any task needed. Naming a cost is not the same as removing it; caching `get_version_info()` across a batch run would take ~0.7 s off every task in a suite and is recorded in the doc as the follow-up. Verified on claude-code only. Pi could not be re-measured: its CLI began hanging with zero stream events partway through this session, reproduced standalone outside the harness on the same prompt, and its last successful run (09:30:46) postdates the last commit touching `pi_agent.py` (09:21:53) by nine minutes — so the hang is external and the pi figure is simply not claimed here. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FpDo37ypvLjLiWXFsEkg6k --- docs/agents/HARNESS_PARITY.md | 26 +++++++++++++++++++++----- src/coder_eval/models/results.py | 6 ++++-- src/coder_eval/orchestrator.py | 24 +++++++++++++++++------- 3 files changed, 42 insertions(+), 14 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 7d971d906..752f7aa82 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -231,11 +231,27 @@ comparable with turns 2..N; and it is not harness time at all — measured at ~1.86 s for claude-code and pi alike on the same machine, which is the tell. Naming them is what makes the evalboard's **Unaccounted** cell a residual -rather than a label. It used to hold that ~1.9 s constant on every row, which -reads as 10% of a 19 s task and would read 60% of a 3 s one. Measured after the -split: 1.9% (claude-code) and 7.3% (pi) of task wall clock, and what remains is -post-`AgentEnd` subprocess reaping, `post_run`, cleanup and record persistence — -which is why the remainder is larger for the harnesses that drain a CLI. +rather than a label. It used to hold a ~1.9 s constant on every row, which +reads as 10% of a 19 s task and would read 60% of a 3 s one. + +`setup_ms` is marked from the top of `run()` and not from `_setup()`, which +matters more than it sounds: instrumenting the seams showed **733 of the +remaining 758 ms was one call**, `utils.get_version_info()`, which shells out +for the git commit and every CLI's `--version` while `EvaluationResult` is +being constructed — before `_setup()` is reached. Marking from `_setup()` left +it outside every named bucket. The rest of that 758 ms was `post_run` (32 ms), +`_cleanup()` (1.4 ms), `task.json` persistence (1.9 ms) and ~5 ms of loop +preamble. + +Measured after the move: **42 ms, 0.26%** of task wall clock on a 16 s +claude-code task. What remains is that tail — `post_run`, sandbox preservation, +persistence and post-`AgentEnd` reaping — with no single nameable phase left in +it, which is what "unaccounted" should mean. + +NOTE `setup_ms` therefore carries a ~733 ms constant that is instrumentation +overhead rather than work the task needed. Naming it is not the same as making +it cheap; caching `get_version_info()` across a batch run is the obvious +follow-up and would take ~0.7 s off every task in a suite. **The head means one thing on all five.** It is the wall clock from the turn starting until the harness first observed **model output**, and that instant is diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index c1efcd718..3ebe47047 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -551,8 +551,10 @@ class EvaluationResult(BaseModel): setup_ms: float | None = Field( default=None, description=( - "Wall milliseconds from the task starting until the agent phase begins — sandbox " - "setup, agent construction and start(), and any pre_run commands. TASK-scoped, " + "Wall milliseconds from the task starting until the agent phase begins — the environment " + "capture (`get_version_info`, which shells out for the git commit and every CLI " + "version: 733 ms measured, the single largest item), criterion discovery, sandbox " + "provisioning, agent construction and start(), and any pre_run commands. TASK-scoped, " "which is why it is here and not a fifth member of TurnRecord's four buckets: " "those tile ONE TURN and their identity (head + generation + tool + tail == the " "turn's span) is asserted to the millisecond, while setup happens once for a task " diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index 5c11d37bd..a7b04e471 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -624,6 +624,15 @@ async def run(self) -> EvaluationResult: agent_type = self.task.agent.type start_time = time.time() + # The monotonic twin of `start_time`, and the mark `setup_ms` measures + # from. It sits HERE rather than at `_setup()` because the phase is + # defined as everything before the agent runs, and the single largest + # item is already behind us by then: `get_version_info()` shells out for + # the git commit and every CLI's `--version` and costs 733 ms measured. + # Starting the mark at `_setup()` put that outside every named bucket, + # so it landed in the report's residual — 733 of the 758 ms that made + # "Unaccounted" look like a real unknown when it was one nameable call. + setup_started = time.monotonic() started_at = datetime.now() # Initialize result @@ -650,7 +659,6 @@ async def run(self) -> EvaluationResult: with task_log_handler(task_log_file, task_id=self._log_task_id) as log_tail: try: # Setup components - setup_started = time.monotonic() await self._setup() # Run pre-run commands inside the sandbox before the agent starts. @@ -660,12 +668,14 @@ async def run(self) -> EvaluationResult: # _cleanup still execute via the finally block. await self._run_pre_run_commands() # Everything before the agent phase, booked as ONE task-level - # bucket. Measured at ~1.86s on this machine for claude-code and - # pi alike, which is the tell that it is the orchestrator's own - # cost rather than any harness's: criterion discovery, sandbox - # setup, agent start(), pre_run. It used to land in the report's - # residual, where a known constant reads as unexplained time — - # 10% of a 19s task, and it would read 60% of a 3s one. + # bucket: the environment capture, criterion discovery, sandbox + # provisioning, agent start() and pre_run. Roughly harness- + # independent — measured within ~10 ms of each other for + # claude-code and pi on the same machine — which is the tell + # that it is the orchestrator's own cost rather than any + # harness's. It used to land in the report's residual, where a + # known constant reads as unexplained time: 10% of a 19s task, + # and it would read 60% of a 3s one. self.result.setup_ms = (time.monotonic() - setup_started) * 1000.0 # Enforce task-level timeout via an OS-thread watchdog that From c34e2cb4dbebb93f0e2720f3ebfe52ca9956724f Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:16:13 -0700 Subject: [PATCH 40/54] =?UTF-8?q?test(timing):=201/8=20=E2=80=94=20the=20b?= =?UTF-8?q?racket's=20SOURCE,=20not=20just=20its=20presence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CE064 declares its own blind spot: it can only see that `timestamp=` is passed, never that the value came from the turn's clock, because the three clocked harnesses legitimately reach theirs three ways (a `communicate` local, `state.clock`, `self.clock`). Pinning a spelling would make the rule a syntax check on their internal structure. So the rule removes the SILENT case — a default nobody chose — and a behavioural test has to cover the rest. That test cannot be written against real time. A bracket left on `StreamEvent.timestamp`'s `default_factory=datetime.now` lands within microseconds of a clock-derived one; an assertion comparing the two would pass either way, which is the same "green sensor that measured nothing" this branch keeps registering. `tests/_bracket_clock.py` injects a `TurnClock` stand-in anchored at 2027-01-15 — ~1.1e10 ms out, the device `test_timing_identity_contract` already uses — so a reverted argument fails by a year. Verified by mutation, one site at a time: deleting any one of the six fails at least one test. The same fixture carries the second half. It advances on the real monotonic clock, so with the bracket and the window bounds finally sharing a basis the head and tail come out as small positive measurements. Antigravity asserts `harness_teardown_ms > 0.0` directly, which is the published defect: its tail was the `0.0` `decompose_turn` produced by clamping a negative that two clocks disagreeing had created. Also records CE064 on the two doc surfaces, and adds the lint arm the pre-committed test class was missing — it aliased the EVENT class but never `TurnClock`, so nothing asserted that an aliased clock import still puts a module in scope. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- CLAUDE.md | 2 +- docs/agents/HARNESS_PARITY.md | 30 +++++++++++++ tests/_bracket_clock.py | 76 +++++++++++++++++++++++++++++++++ tests/test_agent_telemetry.py | 55 ++++++++++++++++++++++++ tests/test_antigravity_agent.py | 48 +++++++++++++++++++++ tests/test_custom_lint.py | 10 +++++ tests/test_pi_agent.py | 39 +++++++++++++++++ 7 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 tests/_bracket_clock.py diff --git a/CLAUDE.md b/CLAUDE.md index cabde3f6b..8552e1261 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,7 +236,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE063** (no module in `src/coder_eval/agents/` may import `busy_ms` — tool execution comes out of a generation window in exactly ONE place, `streaming/collector.py::subtract_tool_time`. Five reducers used to do it themselves while the head and tail were already computed centrally at the same seam, and that asymmetry is where every timing defect on this branch lived — none of them in the arithmetic, all of them in the bookkeeping AROUND it: when to reset a per-step span list (clearing it at `step_start` wiped a span before the flush could subtract it, a 100% overstatement of that window), when to clear a spent start stamp (a second flush with no intervening start republished the previous span — 3000 ms of generation for a 2000 ms turn), when to advance the mark. A sixth harness reaching for `busy_ms` rebuilds that, and its tool time is then subtracted TWICE — by the reducer and again by the collector — under-reporting generation on one harness only, which takes a corpus comparison to notice. A separate id from CE061 rather than a rebody: CE061 asks where a window's ARITHMETIC came from and four reducers still call `close_window`, so its property is live and unsuperseded; this asks whether a reducer subtracts at all. It deliberately does NOT reuse CE061's `_imports_the_helper`, whose bare-module-import branch exists so `timing.close_window(...)` counts as reaching the helper — inverted into a ban that branch flags four of the five reducers. CE061 is now **exemption-free**: claude-code was its one permanent `# noqa` and, with the subtraction moved, calls the shrunken `close_window` like the other four), **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. The damage was not confined to the timeline, which is why "only granularity is lost" was the wrong way to describe it: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose prompt-cache cascade is quadratic in that count, so a single-shot Antigravity run had every cascade coefficient pinned at zero; the `Messages` count and the 10 s slow-generation bar were per-turn too. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). +Recent additions, each traceable to a shipped defect: **CE064** (in `src/coder_eval/agents/`, a module that imports `TurnClock` must pass an explicit `timestamp=` to `AgentStartEvent` and `AgentEndEvent` — the turn's OUTER bounds, which no other rule looks at, since CE058-CE061 all scope to `AssistantMessage` and the bracket is not one. `timing.decompose_turn` produces `harness_startup_ms` / `harness_teardown_ms` by subtracting a generation-window bound from a bracket timestamp, so the two must share a basis; all three clocked harnesses derived their bounds from the `TurnClock` and let the bracket fall back to `StreamEvent.timestamp`'s `default_factory=datetime.now`, putting a monotonic-derived stamp and a raw wall stamp inside one subtraction — the exact split `TurnClock` exists to remove, reintroduced at the one seam the clock did not own. Measured on a live antigravity turn: an `AgentEndEvent` stamped **17 us BEFORE its own last message finished**, which cannot happen (the event is constructed strictly after the final flush), and `decompose_turn` clamped that negative and published `0.0` — "measured, and instant", the CE058 confusion arrived at from the other direction — for a harness whose real tail is ~0.1 ms; it now records 0.035 ms. It surfaced on one harness only because the drift is tens of microseconds and antigravity is the only one that holds its process across turns, so nothing happens between its last flush and its end event; every other harness books a tail of 7-543 ms, where the drift is invisible rather than absent — which is why the fix is at every clocked site rather than at that one. SCOPE IS DERIVED, never a harness list: codex and opencode take their spans from the CLI's own epoch stamps, deliberately have no `TurnClock`, and are correctly invisible to the rule — a raw bracket is CONSISTENT with their bounds — and the day either adopts a clock the rule starts applying with no edit. BLIND SPOT, in the rule's docstring: presence, not correctness. It cannot tell `self.clock.now()` from a `datetime.now()` spelled out at the call site, because the three harnesses legitimately reach their clock three ways; the guard for the SOURCE is behavioural (`tests/_bracket_clock.py` injects a stand-in anchored a year from real time, so a reverted argument fails by a year rather than by the microseconds that separate the two clocks), which is the division of labour CE060 states — a rule removes the SILENT case, a default nobody chose), **CE063** (no module in `src/coder_eval/agents/` may import `busy_ms` — tool execution comes out of a generation window in exactly ONE place, `streaming/collector.py::subtract_tool_time`. Five reducers used to do it themselves while the head and tail were already computed centrally at the same seam, and that asymmetry is where every timing defect on this branch lived — none of them in the arithmetic, all of them in the bookkeeping AROUND it: when to reset a per-step span list (clearing it at `step_start` wiped a span before the flush could subtract it, a 100% overstatement of that window), when to clear a spent start stamp (a second flush with no intervening start republished the previous span — 3000 ms of generation for a 2000 ms turn), when to advance the mark. A sixth harness reaching for `busy_ms` rebuilds that, and its tool time is then subtracted TWICE — by the reducer and again by the collector — under-reporting generation on one harness only, which takes a corpus comparison to notice. A separate id from CE061 rather than a rebody: CE061 asks where a window's ARITHMETIC came from and four reducers still call `close_window`, so its property is live and unsuperseded; this asks whether a reducer subtracts at all. It deliberately does NOT reuse CE061's `_imports_the_helper`, whose bare-module-import branch exists so `timing.close_window(...)` counts as reaching the helper — inverted into a ban that branch flags four of the five reducers. CE061 is now **exemption-free**: claude-code was its one permanent `# noqa` and, with the subtraction moved, calls the shrunken `close_window` like the other four), **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. The damage was not confined to the timeline, which is why "only granularity is lost" was the wrong way to describe it: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose prompt-cache cascade is quadratic in that count, so a single-shot Antigravity run had every cascade coefficient pinned at zero; the `Messages` count and the 10 s slow-generation bar were per-turn too. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 752f7aa82..1b8c31e21 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -35,6 +35,7 @@ wall clock its numbers account for. | `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID`; `None` when absent | CLI `responseId`; `None` when absent | | `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | | clock basis for recorded stamps | one `TurnClock` per turn | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | +| turn bracket (`AgentStartEvent` / `AgentEndEvent`) stamp | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms bounds | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms tool spans | the same `TurnClock` (**CE064**) | | window built by `timing.py::close_window` | yes | yes | yes | yes | yes | [^identity]: "yes" is load-bearing, and THREE sensors check it, each seeing @@ -136,6 +137,35 @@ claude-code needed it for the other reason — their stamps were naive-local, an nightly runs start at 04:18 and last hours, so an hour-long jump landing in a millisecond field is reachable rather than theoretical. +**The turn BRACKET is on that clock too, and was the last seam that was not.** +`timing.decompose_turn` produces `harness_startup_ms` / `harness_teardown_ms` by +subtracting a generation-window bound from an `AgentStartEvent` / +`AgentEndEvent` timestamp, so those two stamps have to share a basis. All three +clocked harnesses derived their window bounds from the `TurnClock` and let the +bracket fall back to `StreamEvent.timestamp`'s `default_factory=datetime.now` — +a monotonic-derived stamp and a raw wall stamp meeting inside one subtraction. +Measured on a live antigravity turn: + +``` +PROBE tail: elapsed=-0.017000ms busy=0.000000ms raw=-0.017000ms + last_completed = 09:05:22.033099 + agent_end = 09:05:22.033082 +``` + +an `AgentEndEvent` stamped 17 us BEFORE its own last message finished, which +cannot happen — the event is constructed strictly after the final flush. +`decompose_turn` clamped the negative and published `0.0`, "measured, and +instant", for a harness whose real tail is ~0.1 ms; the same task now records +0.035 ms. It surfaced only here because the drift between the two clocks is +tens of microseconds and antigravity holds its process across turns, so nothing +happens between its last flush and its end event; every other harness books a +tail of 7-543 ms, where the drift is invisible rather than absent. **CE064** +keeps a sixth harness from reintroducing it: a module under `agents/` that +imports `TurnClock` must pass an explicit `timestamp=` on both brackets. Codex +and OpenCode have no `TurnClock`, so the rule does not see them and their raw +`datetime.now()` bracket stays — which is *consistent* with their own CLI-epoch +bounds rather than a gap. + claude-code has exactly one raw `datetime.now()` left, on the synthesized sub-agent terminal message. Those bounds are an admitted placeholder for a generation that arrives as a tool result and is never streamed diff --git a/tests/_bracket_clock.py b/tests/_bracket_clock.py new file mode 100644 index 000000000..d5c52ee21 --- /dev/null +++ b/tests/_bracket_clock.py @@ -0,0 +1,76 @@ +"""A ``TurnClock`` stand-in anchored far from real time, for the CE064 tests. + +CE064 checks only that ``timestamp=`` is PRESENT on an ``AgentStartEvent`` / +``AgentEndEvent`` emit — its own declared blind spot is that it cannot tell +``self.clock.now()`` from a ``datetime.now()`` written out at the call site. +This is the guard for the SOURCE of that stamp, on the three harnesses that own +a clock. + +ANCHORED FAR FROM NOW, and that is the whole trick. A bracket left on +``StreamEvent.timestamp``'s ``default_factory=datetime.now`` lands within +microseconds of a clock-derived one, so an assertion written against real time +would pass either way. Anchoring the stand-in a year out (the same device as +``tests/test_timing_identity_contract.py``'s ``EPOCH_MS``) makes a reverted +``timestamp=`` fail by a year rather than by a microsecond. + +It advances on the REAL monotonic clock instead of stepping by hand, which is +what lets the same fixture assert the second half: with the bracket and the +window bounds finally on one basis, ``decompose_turn``'s head and tail come out +as small positive measurements rather than as the clamped ``0.0`` a cross-basis +subtraction produced (see ``ce064_turn_bracket_on_the_clock``'s measured probe). +""" + +import time +from datetime import datetime, timedelta + + +#: Far enough from ``datetime.now()`` that a defaulted bracket cannot be mistaken +#: for a clock-derived one. +ANCHOR = datetime(2027, 1, 15, 0, 0, 0) + + +class AnchoredClock: + """``TurnClock``'s shape, re-anchored: ``ANCHOR`` plus real monotonic elapsed.""" + + def __init__(self) -> None: + self._mono0 = time.monotonic() + + def now(self) -> datetime: + return ANCHOR + timedelta(seconds=time.monotonic() - self._mono0) + + +def assert_bracket_on_the_clock(events: list) -> None: + """Both turn brackets were stamped from the injected clock, not from ``now()``. + + Also asserts the pair is ordered, since a start and an end drawn from two + different bases is exactly what produced the inverted antigravity tail. + """ + from coder_eval.streaming.events import AgentEndEvent, AgentStartEvent + + starts = [e for e in events if isinstance(e, AgentStartEvent)] + ends = [e for e in events if isinstance(e, AgentEndEvent)] + assert len(starts) == 1, f"expected one AgentStartEvent, saw {len(starts)}" + assert len(ends) == 1, f"expected one AgentEndEvent, saw {len(ends)}" + for event in (*starts, *ends): + assert event.timestamp >= ANCHOR, ( + f"{type(event).__name__}.timestamp is {event.timestamp}, which is not from the turn's " + f"clock (anchored at {ANCHOR}). It fell back to StreamEvent's default_factory=datetime.now, " + "so the turn bracket and the generation-window bounds sit on two bases inside one " + "`decompose_turn` subtraction — see CE064." + ) + assert ends[0].timestamp >= starts[0].timestamp + + +def assert_overhead_is_measured(record) -> None: + """The turn's head and tail are real sub-second measurements on one basis. + + A cross-basis subtraction shows up here rather than in the stamps: the + clamp in ``decompose_turn`` turns the negative into a ``0.0`` that reads as + "measured, and instant". Bounding them well below the anchor offset is what + proves both ends came from the same clock — a mixed pair would be off by + about a year, not by a millisecond. + """ + for name in ("harness_startup_ms", "harness_teardown_ms"): + value = getattr(record, name) + assert value is not None, f"{name} was never measured" + assert 0.0 <= value < 60_000.0, f"{name} is {value} ms — the two ends are not on one clock" diff --git a/tests/test_agent_telemetry.py b/tests/test_agent_telemetry.py index 50582537c..adcf59108 100644 --- a/tests/test_agent_telemetry.py +++ b/tests/test_agent_telemetry.py @@ -3,6 +3,7 @@ import time from datetime import datetime, timedelta from types import SimpleNamespace +from typing import Any import pytest @@ -1521,3 +1522,57 @@ def test_the_four_buckets_account_for_a_tool_free_turn(self, monkeypatch): generation = sum(m.generation_duration_ms or 0.0 for m in record.messages if m.role == "assistant") assert generation == pytest.approx(200.0) assert record.harness_startup_ms + generation + record.harness_teardown_ms == pytest.approx(1500.0) + + +class TestTheTurnBracketComesFromTheTurnClock: + """CE064's behavioural half for claude-code: the SOURCE of the two stamps. + + The rule can only see that `timestamp=` is present — it cannot tell + `state.clock.now()` from a `datetime.now()` spelled out at the call site. + Anchoring the stand-in a year from real time is what makes a reverted + argument fail by a year instead of by the microseconds that separate the + two clocks in practice. + """ + + @pytest.mark.asyncio + async def test_both_brackets_are_stamped_from_the_injected_clock(self, tmp_path, monkeypatch): + import coder_eval.agents.claude_code_agent as agent_module + from tests._bracket_clock import AnchoredClock, assert_bracket_on_the_clock + + _, assistant_message_cls, _, text_block_cls, _, result_message_cls = create_mock_sdk_messages() + assistant_msg = assistant_message_cls([text_block_cls("done")], message_id="m1") + + async def mock_query(prompt, options): + yield assistant_msg + yield result_message_cls() + + monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) + monkeypatch.setattr(agent_module, "query", mock_query) + + agent = agent_module.ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) + await agent.start(str(tmp_path)) + seen: list[Any] = [] + await agent.communicate("go", stream_callback=SimpleNamespace(on_event=seen.append)) + + assert_bracket_on_the_clock(seen) + + @pytest.mark.asyncio + async def test_the_head_and_tail_are_measured_within_one_basis(self, tmp_path, monkeypatch): + import coder_eval.agents.claude_code_agent as agent_module + from tests._bracket_clock import AnchoredClock, assert_overhead_is_measured + + _, assistant_message_cls, _, text_block_cls, _, result_message_cls = create_mock_sdk_messages() + assistant_msg = assistant_message_cls([text_block_cls("done")], message_id="m1") + + async def mock_query(prompt, options): + yield assistant_msg + yield result_message_cls() + + monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) + monkeypatch.setattr(agent_module, "query", mock_query) + + agent = agent_module.ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE)) + await agent.start(str(tmp_path)) + record = await agent.communicate("go") + + assert_overhead_is_measured(record) diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 54763f2ee..6ad2dc26b 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -28,6 +28,7 @@ from coder_eval.models import AgentKind, AntigravityAgentConfig, AssistantMessage, parse_agent_config from coder_eval.plugins import ensure_plugins_loaded from coder_eval.pricing import calculate_cost +from tests._bracket_clock import AnchoredClock, assert_bracket_on_the_clock, assert_overhead_is_measured from tests._fixtures.golden_streams._scrub import assert_reconciliation from tests._fixtures.golden_streams.antigravity_fixtures import ( _agent_with_steps, @@ -2189,3 +2190,50 @@ def test_a_turn_that_streams_no_step_keeps_the_turn_entry_mark(self): state = self._state(clock) assert state._first_output_seen is False assert state._gen_mark_wall == self.BASE + + +class TestTheTurnBracketComesFromTheTurnClock: + """CE064's behavioural half: the SOURCE of the two bracket stamps. + + This is the harness the defect was measured on. It holds its process across + turns, so nothing happens between its last flush and its `AgentEndEvent` + and its true tail is ~0.1 ms — the only scale at which the drift between a + raw `datetime.now()` and a monotonic-derived stamp can flip a sign. It did: + a tail of -0.017 ms, clamped and published as the `0.0` that means + "measured, and instant". + """ + + @staticmethod + def _steps(): + return [ + _step("THINKING", "DONE", thinking="plan", usage=_usage(100, 0, 5, 5)), + _step( + "TEXT_RESPONSE", + "DONE", + content="done", + content_delta="done", + complete=True, + usage=_usage(200, 0, 10, 0), + ), + ] + + async def test_both_brackets_are_stamped_from_the_injected_clock(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) + seen: list[Any] = [] + await _agent_with_steps(self._steps()).communicate("go", stream_callback=SimpleNamespace(on_event=seen.append)) + + assert_bracket_on_the_clock(seen) + + async def test_the_tail_is_a_measurement_rather_than_a_clamped_zero(self, monkeypatch: pytest.MonkeyPatch): + """The published defect, asserted directly. + + `harness_teardown_ms` was `0.0` here because `decompose_turn` clamped a + negative produced by two clock bases. With one basis the interval is + tiny but real, so a strict `> 0` is the assertion that fails on a + revert. + """ + monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) + record = await _agent_with_steps(self._steps()).communicate("go") + + assert_overhead_is_measured(record) + assert record.harness_teardown_ms > 0.0 diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 9192baf54..64b1c6b18 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4807,6 +4807,16 @@ def test_resolves_an_aliased_import(self): src = self.CLOCKED + "from coder_eval.streaming.events import AgentEndEvent as Done\n" + "e = Done(task_id='t')" assert len(self._run(src)) == 1 + def test_resolves_an_aliased_clock_import(self): + """The scope side of the same question: `TurnClock as Clock` still clocks the module. + + A harness reaching its clock through an alias is still a clocked + harness; missing the binding would put it silently out of scope, which + is the half a hardcoded harness list would also get wrong. + """ + src = "from coder_eval.timing import TurnClock as Clock\n" + self.START + "e = AgentStartEvent(task_id='t')" + assert len(self._run(src)) == 1 + def test_resolves_a_relative_import(self): src = ( "from ..timing import TurnClock\nfrom ..streaming.events import AgentStartEvent\ne = AgentStartEvent(t='t')" diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index 0ccc8b8dc..a68d903d1 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -40,6 +40,7 @@ TurnStartEvent, ) from coder_eval.timing import TurnClock +from tests._bracket_clock import AnchoredClock, assert_bracket_on_the_clock, assert_overhead_is_measured from tests._fixtures.golden_streams.pi_fixtures import ( EXPECTED_CACHE_READ, EXPECTED_COST, @@ -1559,3 +1560,41 @@ async def test_the_agent_retains_no_clock_between_turns(self, patch_exec, tmp_pa leaked = [name for name, value in vars(agent).items() if isinstance(value, _PiTurnState | TurnClock)] assert not leaked, f"a turn's clock outlived its turn via {leaked}" + + +class TestTheTurnBracketComesFromTheTurnClock: + """CE064's behavioural half: the SOURCE of the two bracket stamps. + + The rule can only see that `timestamp=` is present. Reverting it to + `StreamEvent.timestamp`'s `default_factory=datetime.now` would leave the + stamp within microseconds of the clock-derived one, which is precisely why + the stand-in is anchored a year out — the revert then fails by a year. + """ + + async def test_both_brackets_are_stamped_from_the_injected_clock( + self, patch_exec, tmp_path, monkeypatch: pytest.MonkeyPatch + ): + from coder_eval.agents import pi_agent as agent_module + + monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) + patch_exec(_FakeProcess(HAPPY_STREAM)) + recorder = _EventRecorder() + await _run(_agent(), tmp_path, stream_callback=recorder) + + assert_bracket_on_the_clock(recorder.events) + + async def test_the_head_and_tail_are_measured_within_one_basis( + self, patch_exec, tmp_path, monkeypatch: pytest.MonkeyPatch + ): + """Both ends of `decompose_turn`'s subtraction come from one clock. + + A mixed pair is off by the anchor offset, not by a millisecond, so the + bound here is what the assertion rests on rather than the sign. + """ + from coder_eval.agents import pi_agent as agent_module + + monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) + patch_exec(_FakeProcess(HAPPY_STREAM)) + record = await _run(_agent(), tmp_path) + + assert_overhead_is_measured(record) From cf20c0739a15952693d84b90686b5e700158bd4e Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:16:53 -0700 Subject: [PATCH 41/54] =?UTF-8?q?fix(timing):=202/8=20=E2=80=94=20a=20publ?= =?UTF-8?q?ished=20window=20must=20match=20its=20own=20bounds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `generation_duration_ms` is a field each reducer PUBLISHES, and nothing checked it against the bounds published beside it. Deriving it at the collector instead was the alternative, and was cut: it costs five reducers, a regeneration of every golden, and a rewrite of CE059 — whose exemption keys on the kwarg being present at the call site, so removing it makes three legitimate placeholder sites start claiming a window. This assertion is what makes that deferral safe, and the docstring says so, because the next reader will otherwise re-derive the decision. It overlaps CE061 on purpose. All five reducers build the window with `close_window(mark=…, now=…)` and write `completed_at=now`, and CE061 forces that shape statically, so the equality is largely true by construction. What this adds is the runtime half: a reducer bypassing the helper in a way an import-level check cannot see, and a third-party agent registered through the `coder_eval.plugins` SPI, which lives outside `agents/` where no rule scoped to that directory reaches it — the same exposure `_require_same_awareness` at this seam is for, and the same trade: it raises rather than degrading, because the condition is unreachable without a reducer bug. Measured before writing it: zero violations from any reducer across the full suite. The 17 that did fail were all hand-built fixtures in one file, modelling a shape no reducer produces — a `generation_duration_ms=1.0` beside bounds seconds apart. They are corrected rather than exempted, at three seams rather than seventeen call sites, and two `TestReconciliation` cases needed distinct windows: messages sharing one instant are grouped as a Codex-style split and then sum to twice the window they claim. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- src/coder_eval/streaming/collector.py | 43 +++++++++ tests/test_event_collector.py | 127 +++++++++++++++++++++++--- 2 files changed, 159 insertions(+), 11 deletions(-) diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index 9fa5bfeb0..bae81206e 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -22,6 +22,7 @@ from __future__ import annotations +import math from collections.abc import Iterable from datetime import datetime @@ -139,6 +140,35 @@ def subtract_tool_time( A window entirely covered by tool execution reaches ``0.0``, and that is a measurement rather than an absence. + + THE GROUP'S RAW TOTAL MUST EQUAL THE SPAN ITS BOUNDS DESCRIBE, and this + function raises if it does not. That equality is the contract that lets + ``generation_duration_ms`` stay a PUBLISHED field rather than one the + collector derives from the bounds: a reducer publishes the raw window it + measured, so the duration is ``completed_at - started_at`` (or, for a group + Codex split across two sub-messages, sums to it). Deriving it here instead + was considered and cut — it would cost five reducers, a regeneration of + every golden and a rewrite of CE059, whose exemption keys on the kwarg being + present at the call site — and this assertion is the sensor that makes + deferring that safe. A mismatch means a reducer narrowed or widened a window + without moving its bounds, which is the drift + ``tests/_fixtures/golden_streams/_scrub.py::assert_timing_captured``'s + "bounds that span it" check catches one replay at a time. + + It OVERLAPS with CE061 and is deliberately kept anyway. All five reducers + build the window with ``timing.close_window(mark=…, now=…)`` and write + ``started_at=started, completed_at=now``, and CE061 — now exemption-free — + forces that shape statically, so the equality is largely true by + construction. What this adds is the runtime half: a reducer that bypasses + ``close_window`` in a way an import-level check cannot see, and a + third-party agent registered through the ``coder_eval.plugins`` SPI, which + lives outside ``src/coder_eval/agents/`` where no lint rule reaches it. It + is not load-bearing on its own. + + RAISING KILLS THE TURN, and that is accepted — the same trade + ``timing._require_same_awareness`` makes at this seam. The condition is + unreachable without a reducer bug; all five are exercised by the golden + corpus and by the ms-exact identity contract. """ # (index, raw window ms) per group. The raw value is captured HERE, where # the message is already narrowed to AssistantMessage, so the apportioning @@ -159,6 +189,19 @@ def subtract_tool_time( # group already at zero stays at zero. if raw_total <= 0: continue + bounds_ms = (completed - started).total_seconds() * 1000.0 + if not math.isclose(raw_total, bounds_ms, rel_tol=1e-9, abs_tol=1e-6): + raise ValueError( + f"generation_duration_ms: a group of {len(members)} message(s) bounded " + + f"{started} -> {completed} ({bounds_ms:.6f} ms) publishes {raw_total:.6f} ms of " + + "generation. A reducer publishes the RAW window it measured, so its duration is " + + "`completed_at - started_at` (or, across the sub-messages Codex splits one window " + + "into, sums to it) — tool execution comes back out HERE, once, for every harness. " + + "A disagreement means the reducer narrowed or widened a window without moving its " + + "bounds, which makes the duration and the bounds two answers to one question and " + + "breaks the four-bucket identity. Build the window with `timing.close_window` and " + + "write `completed_at=now` (CE061), rather than adjusting the duration in place." + ) net = max(raw_total - busy_ms(spans, started, completed), 0.0) assigned = 0.0 for n, (index, raw) in enumerate(members): diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 6072c970b..765e53578 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -220,9 +220,10 @@ class TestFullFieldParity: def _full_agent_end(self) -> AgentEndEvent: """An AgentEndEvent with every verbatim field set to a non-default sentinel.""" + started = datetime(2026, 9, 11, 9, 0, 0) msg = AssistantMessage( - started_at=datetime.now(), - completed_at=datetime.now(), + started_at=started, + completed_at=started + timedelta(milliseconds=12.0), generation_duration_ms=12.0, output_tokens=7, ) @@ -282,18 +283,33 @@ def test_every_verbatim_field_round_trips(self): assert record_value == event_value, f"{name}: record={record_value!r} event={event_value!r}" +_GEN_BASE = datetime(2026, 9, 11, 9, 0, 0) +_GEN_WINDOW_MS = 1.0 + + def _assistant( *, + window: int = 0, input_tokens: int = 0, output_tokens: int = 0, cache_creation_tokens: int = 0, cache_read_tokens: int = 0, parent_tool_use_id: str | None = None, ) -> AssistantMessage: + """One generation, shaped the way a reducer emits one. + + The bounds SPAN the published duration, and `window` tiles successive + messages rather than leaving them on one instant. Both matter to + `subtract_tool_time`, which asserts that a group's published total equals + the span its bounds describe and which GROUPS on those bounds: two + messages sharing an instant would be read as one Codex-style split window + and then violate the equality by summing to twice it. + """ + started = _GEN_BASE + timedelta(milliseconds=_GEN_WINDOW_MS * window) return AssistantMessage( - started_at=datetime.now(), - completed_at=datetime.now(), - generation_duration_ms=1.0, + started_at=started, + completed_at=started + timedelta(milliseconds=_GEN_WINDOW_MS), + generation_duration_ms=_GEN_WINDOW_MS, input_tokens=input_tokens, output_tokens=output_tokens, cache_creation_tokens=cache_creation_tokens, @@ -346,8 +362,8 @@ def test_claude_shaped_gap_is_booked_so_transcript_reconciles(self): # Claude: model_usage total exceeds the per-message sum (a fixed ~512 input # slice + sub-agent input ride on no streamed message). messages = [ - _assistant(input_tokens=100, output_tokens=40, cache_read_tokens=2000), - _assistant(input_tokens=50, output_tokens=20, cache_read_tokens=3000), + _assistant(window=0, input_tokens=100, output_tokens=40, cache_read_tokens=2000), + _assistant(window=1, input_tokens=50, output_tokens=20, cache_read_tokens=3000), ] usage = TokenUsage( uncached_input_tokens=662, # 150 + 512 unattributed @@ -375,8 +391,8 @@ def test_codex_shaped_with_subagent_messages_reconciles(self): # Codex: parent + recovered sub-agent (parent_tool_use_id) generations, with # the folded total slightly above the streamed sum. messages = [ - _assistant(input_tokens=200, output_tokens=80, cache_read_tokens=1000), - _assistant(input_tokens=300, output_tokens=20, parent_tool_use_id="call_sub"), + _assistant(window=0, input_tokens=200, output_tokens=80, cache_read_tokens=1000), + _assistant(window=1, input_tokens=300, output_tokens=20, parent_tool_use_id="call_sub"), ] usage = TokenUsage( uncached_input_tokens=520, # 500 + 20 residual @@ -474,6 +490,11 @@ def test_minimal_record_without_agent_end(self): assert [c.tool_id for c in record.commands] == ["a"] +def _span_ms(started: datetime, completed: datetime) -> float: + """The window its own bounds describe — what every reducer publishes.""" + return (completed - started).total_seconds() * 1000.0 + + class TestHarnessOverheadBuckets: """The turn's two unexplained ends: before the first generation, after the last. @@ -497,7 +518,9 @@ def _msg(started: datetime, completed: datetime, *, measurable: bool = True) -> return AssistantMessage( started_at=started, completed_at=completed, - generation_duration_ms=1.0 if measurable else None, + # Derived, not a literal: `subtract_tool_time` asserts a published + # window equals the span its own bounds describe. + generation_duration_ms=_span_ms(started, completed) if measurable else None, ) @staticmethod @@ -507,7 +530,7 @@ def _subagent_msg(started: datetime, completed: datetime) -> AssistantMessage: return AssistantMessage( started_at=started, completed_at=completed, - generation_duration_ms=1.0, + generation_duration_ms=_span_ms(started, completed), parent_tool_use_id="toolu_agent", ) @@ -851,6 +874,88 @@ def test_non_assistant_entries_pass_through_by_identity(self): assert out[1] is reconciliation +class TestAPublishedWindowMustMatchItsOwnBounds: + """The seam assertion: a group's raw total is the span its bounds describe. + + That equality is what lets `generation_duration_ms` stay a PUBLISHED field + instead of one the collector derives from the bounds — the migration that + was considered and cut, on the grounds that this check makes deferring it + safe. It is largely true by construction (CE061 forces every reducer + through `timing.close_window`); what it catches is a reducer that bypasses + the helper, and a third-party agent registered through the + `coder_eval.plugins` SPI, which no lint rule scoped to `agents/` can see. + """ + + BASE: ClassVar[datetime] = datetime(2026, 9, 11, 9, 0, 0) + + @classmethod + def _at(cls, ms: float) -> datetime: + return cls.BASE + timedelta(milliseconds=ms) + + @classmethod + def _msg(cls, lo: float, hi: float, gen: float | None, **kwargs) -> AssistantMessage: + return AssistantMessage(started_at=cls._at(lo), completed_at=cls._at(hi), generation_duration_ms=gen, **kwargs) + + def test_a_narrowed_window_raises_and_names_both_numbers(self): + with pytest.raises(ValueError) as excinfo: + subtract_tool_time([self._msg(0, 1000, 400.0)], []) + message = str(excinfo.value) + assert "400.000000" in message and "1000.000000" in message + assert "generation_duration_ms" in message + + def test_a_widened_window_raises_too(self): + with pytest.raises(ValueError): + subtract_tool_time([self._msg(0, 1000, 1600.0)], []) + + def test_a_window_that_matches_its_bounds_passes(self): + out = subtract_tool_time([self._msg(0, 1000, 1000.0)], []) + assert out[0].generation_duration_ms == pytest.approx(1000.0) + + def test_codexs_split_passes_when_the_parts_sum_to_the_window(self): + """Built with `_flush_message`'s own idiom, not a hand-picked pair. + + Codex divides one window across two sub-messages by output-token share, + rounding every share but the last to 6 places and giving the last the + remainder — so the tolerance is exercised against the real rounding + rather than against exact halves. + """ + window_ms = 1000.0 + first = round(window_ms * (1.0 / 3.0), 6) + parts = [first, window_ms - first] + out = subtract_tool_time([self._msg(0, 1000, part, message_id="m") for part in parts], []) + assert sum(m.generation_duration_ms or 0.0 for m in out) == pytest.approx(window_ms) + + def test_a_split_whose_parts_sum_to_the_wrong_total_raises(self): + with pytest.raises(ValueError): + subtract_tool_time([self._msg(0, 1000, 400.0, message_id="m"), self._msg(0, 1000, 400.0)], []) + + def test_a_zero_group_is_skipped_before_the_check_runs(self): + """The `raw_total <= 0` skip runs FIRST, and must keep running first. + + A window measured at zero between IDENTICAL bounds would satisfy the + equality anyway; the case that needs the order is a `0.0` published + beside bounds that are not identical, which is a shape the tree + tolerates today. Raising on it would turn a tolerated record into a + killed turn, so the bounds here are deliberately 500 ms apart. + """ + out = subtract_tool_time([self._msg(0, 500, 0.0)], []) + assert out[0].generation_duration_ms == 0.0 + + def test_an_unmeasured_window_never_reaches_the_check(self): + out = subtract_tool_time([self._msg(0, 5000, None)], []) + assert out[0].generation_duration_ms is None + + def test_a_sub_agent_message_is_excluded_even_with_placeholder_bounds(self): + """Its bounds are an admitted placeholder and cannot support its duration. + + Codex's recovered child messages carry the CHILD's clock, and Claude's + synthesized sub-agent terminal stamps one instant on both bounds. They + are skipped before the group is built, so the check never sees them. + """ + out = subtract_tool_time([self._msg(0, 0, 900.0, parent_tool_use_id="toolu_agent")], []) + assert out[0].generation_duration_ms == pytest.approx(900.0) + + class TestBuildTurnRecordIsIdempotent: """Building the record twice must give the same numbers. From 3243ea406cdc338073f8ec90715b1e1fb3e8da27 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:18:51 -0700 Subject: [PATCH 42/54] fix(timing): the bracket fixture's anchor cannot expire, and the tail is checked MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from the phase review, both of which left the guard weaker than its own docstring claimed. The anchor was a written date four months out, so `timestamp >= ANCHOR` would have been satisfied by exactly the raw `datetime.now()` stamp it exists to reject the moment wall time passed 2027-01-15 — a green sensor measuring nothing, on a date nobody would have connected to this test. It is now computed relative to import. (`test_timing_identity_contract`'s fixed EPOCH_MS is not the same hazard: that timeline is fully synthetic and never compared against real time.) And `assert_overhead_is_measured` only caught one of the two reverts. A defaulted AgentStartEvent blows the upper bound by the whole anchor offset, but a defaulted AgentEndEvent fails the other way — it lands BEFORE its own last message, `decompose_turn` clamps the negative, and the published `0.0` sails through an upper bound. Measured: deleting the end-event argument on pi and claude-code left that test green, and only the sibling assertion caught it. A strict `> 0.0` on the tail is the fix, hoisted into the shared helper rather than left as antigravity's local extra — it holds on all three, and antigravity is merely where the margin is thinnest (0.007-0.03 ms, since it alone holds its process across turns). Re-verified by mutation with only that test selected. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- tests/_bracket_clock.py | 53 +++++++++++++++++++++++---------- tests/test_antigravity_agent.py | 8 ++--- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/tests/_bracket_clock.py b/tests/_bracket_clock.py index d5c52ee21..46faa1ea2 100644 --- a/tests/_bracket_clock.py +++ b/tests/_bracket_clock.py @@ -23,10 +23,16 @@ import time from datetime import datetime, timedelta +from coder_eval.models import TurnRecord +from coder_eval.streaming.events import AgentEndEvent, AgentStartEvent, StreamEvent + #: Far enough from ``datetime.now()`` that a defaulted bracket cannot be mistaken -#: for a clock-derived one. -ANCHOR = datetime(2027, 1, 15, 0, 0, 0) +#: for a clock-derived one. RELATIVE, never a written date: a fixed anchor stops +#: discriminating the moment wall time passes it, and the assertion below would +#: then be satisfied by exactly the `datetime.now()` stamp it exists to reject — +#: a green sensor measuring nothing, on a date nobody would connect to the test. +ANCHOR = datetime.now() + timedelta(days=365) class AnchoredClock: @@ -39,14 +45,12 @@ def now(self) -> datetime: return ANCHOR + timedelta(seconds=time.monotonic() - self._mono0) -def assert_bracket_on_the_clock(events: list) -> None: +def assert_bracket_on_the_clock(events: list[StreamEvent]) -> None: """Both turn brackets were stamped from the injected clock, not from ``now()``. Also asserts the pair is ordered, since a start and an end drawn from two different bases is exactly what produced the inverted antigravity tail. """ - from coder_eval.streaming.events import AgentEndEvent, AgentStartEvent - starts = [e for e in events if isinstance(e, AgentStartEvent)] ends = [e for e in events if isinstance(e, AgentEndEvent)] assert len(starts) == 1, f"expected one AgentStartEvent, saw {len(starts)}" @@ -61,16 +65,33 @@ def assert_bracket_on_the_clock(events: list) -> None: assert ends[0].timestamp >= starts[0].timestamp -def assert_overhead_is_measured(record) -> None: - """The turn's head and tail are real sub-second measurements on one basis. +def assert_overhead_is_measured(record: TurnRecord) -> None: + """The turn's head and tail are real measurements taken on one basis. + + The two ends fail differently, and each needs its own assertion. + + A defaulted ``AgentStartEvent`` lands ~365 days before the clock-derived + first window, so the HEAD blows any sane bound by that whole offset — the + upper bound is what catches it. - A cross-basis subtraction shows up here rather than in the stamps: the - clamp in ``decompose_turn`` turns the negative into a ``0.0`` that reads as - "measured, and instant". Bounding them well below the anchor offset is what - proves both ends came from the same clock — a mixed pair would be off by - about a year, not by a millisecond. + A defaulted ``AgentEndEvent`` fails the other way: it lands ~365 days + BEFORE its own last message, so ``decompose_turn`` clamps the negative and + publishes ``0.0`` — "measured, and instant", which sails through an upper + bound. Only a strict ``> 0.0`` catches it, and it holds on all three + harnesses because a turn's last flush and its end event are separated by + real work. The margin is small where it is smallest: antigravity holds its + process across turns and measures 0.007-0.03 ms here, which is 7-30 ticks + of the 1 us resolution both `datetime` and `time.monotonic()` have on + Linux, macOS and Windows. That is the magnitude the clamped defect hid, so + do not relax this to ``>= 0.0`` — a zero is the defect. """ - for name in ("harness_startup_ms", "harness_teardown_ms"): - value = getattr(record, name) - assert value is not None, f"{name} was never measured" - assert 0.0 <= value < 60_000.0, f"{name} is {value} ms — the two ends are not on one clock" + assert record.harness_startup_ms is not None, "harness_startup_ms was never measured" + assert record.harness_teardown_ms is not None, "harness_teardown_ms was never measured" + assert 0.0 <= record.harness_startup_ms < 60_000.0, ( + f"harness_startup_ms is {record.harness_startup_ms} ms — the bracket and the first " + "window bound are not on one clock" + ) + assert 0.0 < record.harness_teardown_ms < 60_000.0, ( + f"harness_teardown_ms is {record.harness_teardown_ms} ms — a 0.0 here is the clamped " + "inversion CE064 exists to remove, not an instant teardown" + ) diff --git a/tests/test_antigravity_agent.py b/tests/test_antigravity_agent.py index 6ad2dc26b..f1b32272d 100644 --- a/tests/test_antigravity_agent.py +++ b/tests/test_antigravity_agent.py @@ -2228,12 +2228,12 @@ async def test_the_tail_is_a_measurement_rather_than_a_clamped_zero(self, monkey """The published defect, asserted directly. `harness_teardown_ms` was `0.0` here because `decompose_turn` clamped a - negative produced by two clock bases. With one basis the interval is - tiny but real, so a strict `> 0` is the assertion that fails on a - revert. + negative produced by two clock bases. The strict `> 0.0` that catches a + revert lives in `assert_overhead_is_measured`, which every harness + shares — this harness is simply where the margin is thinnest, since it + holds its process across turns and so has the shortest real tail. """ monkeypatch.setattr(agent_module, "TurnClock", AnchoredClock) record = await _agent_with_steps(self._steps()).communicate("go") assert_overhead_is_measured(record) - assert record.harness_teardown_ms > 0.0 From 9c72c39618bf95c3d1b7692704b4566cee69c63d Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:22:15 -0700 Subject: [PATCH 43/54] =?UTF-8?q?fix(evalboard):=203/8=20=E2=80=94=20an=20?= =?UTF-8?q?unbounded=20tool=20call=20contributes=20to=20no=20bucket?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two languages disagreed about one policy. `toolExecutionMs` folded a call with a `durationMs` and no execution bounds into its union; Python has always dropped it, because `main_thread_tool_spans` filters on `is not None`. So the same task.json produced two different tool totals depending on which surface read it, and the four-bucket identity held on one side only. The union is the side that has to win. A duration with no start and end cannot be placed on the timeline, so it cannot be unioned with anything — adding it to a union double-books whatever it overlapped and can drive the residual negative, which destroys the disjointness the identity rests on. That time is not lost: it reads as Unaccounted, which is exactly what that cell means, a duration the harness measured but cannot place. MEASURED BLAST RADIUS, because someone comparing an August dashboard before and after will otherwise read this as data loss. 9336 of 12170 commands in the run history on disk are unbounded — every one a codex `Bash`, ~28.8 million ms, ~8 h in aggregate. On historical codex runs that much moves out of Tool exec and into Unaccounted. The population is CLOSED: this branch's own `_item_timing` work took codex from 0% bounded before 2026-09-10 to 100% after, so no future run joins it, and a fifth bucket to serve a shrinking historical population would be YAGNI. Going forward the only producer is the out-of-tree `delegate-sdk`, which both divergence records now say so. Neither implementation owns the policy: `unbounded_cases` in the shared corpus does, and both suites replay it — Python through the production SELECTOR rather than through `union_ms`, since the `is not None` filter one layer up is the thing being pinned. The timeline tests moved with it. Their strip fixtures declared `durationMs` with `execStartMs: null`, a shape no in-tree harness has produced since 2026-09-10; they are bounded now, every asserted number unchanged, and the fallback test is inverted rather than deleted. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- .claude/harness-candidates.md | 6 ++ docs/agents/HARNESS_PARITY.md | 12 +++- .../__tests__/message-timeline.test.tsx | 52 ++++++++++++---- .../lib/__tests__/timing-union-parity.test.ts | 62 ++++++++++++++++--- evalboard/lib/timing.ts | 39 ++++++++---- tests/_fixtures/timing_union_cases.json | 45 +++++++++++++- tests/test_timing_union_parity.py | 46 ++++++++++++++ 7 files changed, 229 insertions(+), 33 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 94744d3f9..1dfb913a3 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -538,6 +538,12 @@ divergences, so the deferred-work record is one place. Measurements in Coverage is ~88%, so it is not urgent. The agent lives in the separate `coder_eval_uipath` repo; mirror the Codex change there (`_item_timing` + threading the SDK stamps through the telemetry builders). + **Consequence, as of the turn-timing consolidation (2026-09-12):** such a + call now contributes to no bucket on EITHER surface — the evalboard's + `toolExecutionMs` lost its `durationMs` fallback, matching the `is not None` + filter Python has always had — so the time reads as Unaccounted rather than + as tool execution. Pinned by `unbounded_cases` in + `tests/_fixtures/timing_union_cases.json`, which both suites replay. - [ ] **Pre-existing, surfaced by this work's final review: `TokenUsage._adopt_legacy_input_tokens` double-counts the cache buckets.** The validator copies a legacy record's diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 1b8c31e21..4eb7fb1e0 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -460,7 +460,17 @@ needed to drive it. - **Delegate (`delegate-sdk`, out of tree)** records `duration_ms` but no execution bounds, so its tool calls cannot be placed on a timeline. Its coverage is ~88%. Mirror the Codex change in `coder_eval_uipath` - (audit P3-1). + (audit P3-1). **The consequence is now the same on both surfaces:** such a + call contributes to NO bucket. Python has always dropped it + (`timing.main_thread_tool_spans` filters on `is not None`), and + `evalboard/lib/timing.ts::toolExecutionMs` no longer folds the bare duration + into its union — a duration with no bounds cannot be placed on the timeline, + so unioning it double-books whatever it overlapped and can drive the + four-bucket residual negative. Its time reads as **Unaccounted**, which is + what that cell means: measured, but not placeable. Codex was in the same + state until `_item_timing` landed on 2026-09-10 (0% bounded before, 100% + after), so on historical codex runs ~8 h in aggregate moves out of Tool exec + and into Unaccounted; that population is closed and no new record joins it. - **Antigravity books orphan-poll waiting as agent duration.** A task can spend `0.8 × turn_timeout` waiting on a tool call that never reaches DONE — 14 tasks and 9.6h of one 83h run. Only CLOSED tool intervals are subtracted, so that diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx index 7ca0ef0ae..8e22019f8 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx @@ -521,8 +521,11 @@ describe("MessageTimelineSection — Unaccounted cell", () => { resultPreview: null, outputTokens: null, resultTokens: null, - execStartMs: null, - execEndMs: null, + // BOUNDED. `toolExecutionMs` unions bounded intervals and + // drops a bare duration, matching the Python selector, so + // a durationMs-only tool would contribute 0 here. + execStartMs: 0, + execEndMs: 1000, }, ], ...overrides, @@ -602,8 +605,8 @@ describe("MessageTimelineSection — Unaccounted cell", () => { resultPreview: null, outputTokens: null, resultTokens: null, - execStartMs: null, - execEndMs: null, + execStartMs: 0, + execEndMs: 6000, }; const main = makeMessage({ index: 1, @@ -663,11 +666,35 @@ describe("MessageTimelineSection — Unaccounted cell", () => { expect(cell("Unaccounted").textContent).toBe("2.5s (26%)"); }); - test("a tool with no recorded bounds still contributes its duration", () => { - // Runs predating the execution bounds, and harnesses that report only - // a duration, must not silently drop out of the tool total. - renderStrip(10); - expect(cell("Tool exec").textContent).toBe("1.0s"); + test("a tool with no recorded bounds contributes nothing", () => { + // The policy both languages now share: a duration with no start and + // end cannot be placed on the timeline, so it cannot be unioned with + // anything — folding it in double-books whatever it overlapped. Python + // has always dropped it (`main_thread_tool_spans` filters on + // `is not None`); this cell used to add it. Its time is not lost, it + // moves into Unaccounted, which is what that cell means. + renderStrip(10, { + toolUses: [ + { + toolName: "Bash", + toolUseId: "tu_unbounded", + summary: "ls", + argText: "ls", + description: null, + genMs: null, + durationMs: 1000, + isError: false, + resultPreview: null, + outputTokens: null, + resultTokens: null, + execStartMs: null, + execEndMs: null, + }, + ], + }); + expect(cell("Tool exec").textContent).toBe("0ms"); + // 10s − 4s generation − 0s tool exec: the second is the point. + expect(cell("Unaccounted").textContent).toBe("6.0s (60%)"); }); test("the cell explains that the residual is not only agent time", () => { @@ -816,8 +843,11 @@ describe("MessageTimelineSection — Startup and Teardown cells", () => { resultPreview: null, outputTokens: null, resultTokens: null, - execStartMs: null, - execEndMs: null, + // BOUNDED. `toolExecutionMs` unions bounded intervals and + // drops a bare duration, matching the Python selector, so + // a durationMs-only tool would contribute 0 here. + execStartMs: 0, + execEndMs: 1000, }, ], }); diff --git a/evalboard/lib/__tests__/timing-union-parity.test.ts b/evalboard/lib/__tests__/timing-union-parity.test.ts index 58bcfd391..ee3a8397a 100644 --- a/evalboard/lib/__tests__/timing-union-parity.test.ts +++ b/evalboard/lib/__tests__/timing-union-parity.test.ts @@ -35,9 +35,23 @@ interface ExtentCase { expected_ms: number; } -const corpus: { cases: UnionCase[]; union_cases: ExtentCase[] } = JSON.parse( - readFileSync(fixture, "utf8"), -); +// `unbounded_cases` pins the POLICY rather than the arithmetic: a call the +// harness TIMED but did not BOUND contributes nothing to the union. Python +// replays these through the production selector +// `coder_eval.timing.main_thread_tool_spans`, whose `is not None` filter IS +// that policy; `toolExecutionMs` makes the same decision inline. +interface UnboundedCase { + name: string; + spans: [number, number][]; + unbounded_ms: number[]; + expected_ms: number; +} + +const corpus: { + cases: UnionCase[]; + union_cases: ExtentCase[]; + unbounded_cases: UnboundedCase[]; +} = JSON.parse(readFileSync(fixture, "utf8")); describe("busyMs matches the shared union corpus", () => { test("the corpus is non-empty (a silently emptied file must not pass)", () => { @@ -133,17 +147,20 @@ describe("toolExecutionMs", () => { expect(ms).toBe(6000); }); - test("a timed call with no bounds falls back to its own duration", () => { - // A run predating the execution_started_at/completed_at fields, or a - // harness that reports a duration without them: its duration is the - // best statement available, so it is added rather than dropped. + test("a timed call with no bounds contributes nothing", () => { + // It used to add its own durationMs, which was the one policy on which + // the two languages disagreed — every Python surface has always dropped + // it (main_thread_tool_spans filters on `is not None`). A duration with + // no bounds cannot be placed on the timeline, so it cannot be unioned; + // folding it in double-books whatever it overlapped. Its time reads as + // Unaccounted instead. const ms = toolExecutionMs([ message([ toolUse({ execStartMs: 1000, execEndMs: 2000 }), toolUse({ durationMs: 250 }), ]), ]); - expect(ms).toBe(1250); + expect(ms).toBe(1000); }); test("a call the harness never timed contributes nothing", () => { @@ -177,3 +194,32 @@ describe("toolExecutionMs matches the shared extent corpus", () => { }); } }); + +describe("toolExecutionMs matches the shared unbounded-policy corpus", () => { + test("the policy corpus is non-empty (a silently emptied file must not pass)", () => { + expect(corpus.unbounded_cases.length).toBeGreaterThan(2); + }); + + // Bounded spans become execStartMs/execEndMs calls; unbounded ones become + // durationMs-only calls. `expected_ms` is the union of the bounded spans + // alone, so a side that folds the unbounded durations back in fails here. + for (const c of corpus.unbounded_cases) { + test(c.name, () => { + const ms = toolExecutionMs([ + message([ + ...c.spans.map(([s, e], i) => + toolUse({ + toolUseId: `b${i}`, + execStartMs: s, + execEndMs: e, + }), + ), + ...c.unbounded_ms.map((durationMs, i) => + toolUse({ toolUseId: `u${i}`, durationMs }), + ), + ]), + ]); + expect(ms).toBeCloseTo(c.expected_ms, 6); + }); + } +}); diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts index 37174600b..f26fba3d4 100644 --- a/evalboard/lib/timing.ts +++ b/evalboard/lib/timing.ts @@ -165,28 +165,43 @@ export function busyMs( return total + (openEnd - openStart); } -// Wall-clock milliseconds these messages' tool calls occupied. +// Wall-clock milliseconds these messages' tool calls occupied: the UNION of +// their BOUNDED execution intervals. // -// Bounded calls are UNIONED — concurrent tools occupy the wall clock once, and -// summing them drove the task page's Unaccounted cell to -615ms on a task with -// two concurrent sleeps, where the honest answer was +2.5s of sandbox setup and -// grading. A call the harness timed but did not bound contributes its own -// `durationMs`, which is the best available statement about it and reproduces -// the previous behaviour for that call alone. +// Concurrent tools occupy the wall clock once, and summing them drove the task +// page's Unaccounted cell to -615ms on a task with two concurrent sleeps, where +// the honest answer was +2.5s of sandbox setup and grading. +// +// A call the harness TIMED but did not BOUND contributes nothing — no +// `durationMs` fallback. That is a policy, and it is the same one +// `coder_eval.timing.main_thread_tool_spans` has always had on the Python side: +// its `is not None` filter drops a command with no `execution_started_at` / +// `execution_completed_at`, so every Python surface already ignored these while +// this function folded them in. A duration with no bounds cannot be placed on +// the timeline, so it cannot be unioned with anything; adding it to a union +// double-books whatever it overlapped and can drive the residual negative, +// which destroys the disjointness the four-bucket identity rests on. Such time +// lands in Unaccounted instead, which is precisely what that cell is for — the +// harness measured a duration it cannot place. +// +// Measured blast radius: 9336 of 12170 commands in the run history on disk are +// unbounded, every one a codex `Bash`, ~8 h in aggregate. That population is +// CLOSED — codex went from 0% bounded before 2026-09-10 to 100% after — so no +// future run changes, but on historical codex runs this moves up to ~8 h out of +// Tool exec and into Unaccounted. Going forward the only harness reporting a +// bare duration is the out-of-tree `delegate-sdk`; see +// docs/agents/HARNESS_PARITY.md. export function toolExecutionMs(messages: MessageEvent[]): number { const spans: [number, number][] = []; - let unbounded = 0; for (const m of messages) { for (const t of m.toolUses) { if (t.execStartMs != null && t.execEndMs != null) { spans.push([t.execStartMs, t.execEndMs]); - } else if (t.durationMs != null) { - unbounded += t.durationMs; } } } - if (spans.length === 0) return unbounded; + if (spans.length === 0) return 0; const lo = Math.min(...spans.map(([s]) => s)); const hi = Math.max(...spans.map(([, e]) => e)); - return busyMs(spans, lo, hi) + unbounded; + return busyMs(spans, lo, hi); } diff --git a/tests/_fixtures/timing_union_cases.json b/tests/_fixtures/timing_union_cases.json index e4ac69c80..1ba8722e9 100644 --- a/tests/_fixtures/timing_union_cases.json +++ b/tests/_fixtures/timing_union_cases.json @@ -15,7 +15,18 @@ "spans' own min/max. Python replays it through coder_eval.timing.union_ms;", "TypeScript through evalboard/lib/timing.ts::toolExecutionMs, which derives", "that extent itself rather than being handed one — which is exactly why it", - "needs its own cases instead of being assumed to agree." + "needs its own cases instead of being assumed to agree.", + "", + "`unbounded_cases` pins the POLICY rather than the arithmetic: a tool call", + "the harness TIMED but did not BOUND contributes NOTHING to the union, on", + "both sides. `unbounded_ms` lists such calls; `expected_ms` is the union of", + "`spans` alone, so a side that folds them back in fails. A duration with no", + "start and end cannot be placed on the timeline, so it cannot be unioned", + "with anything — adding it double-books whatever it overlapped and can drive", + "the four-bucket residual negative. Its time reads as Unaccounted, which is", + "what that cell means. Python replays these through the production selector", + "coder_eval.timing.main_thread_tool_spans (the `is not None` filter IS the", + "policy), not through union_ms alone; TypeScript through toolExecutionMs." ], "cases": [ { @@ -155,5 +166,37 @@ "spans": [[10000, 10250]], "expected_ms": 250 } + ], + "unbounded_cases": [ + { + "name": "a timed call with no bounds contributes nothing", + "spans": [], + "unbounded_ms": [250], + "expected_ms": 0 + }, + { + "name": "several unbounded calls still contribute nothing", + "spans": [], + "unbounded_ms": [250, 1000, 30], + "expected_ms": 0 + }, + { + "name": "one bounded call beside an unbounded one is just the bounded one", + "spans": [[1000, 2000]], + "unbounded_ms": [250], + "expected_ms": 1000 + }, + { + "name": "overlapping bounded calls union, and the unbounded one is still dropped", + "spans": [[1000, 6000], [2000, 7000]], + "unbounded_ms": [5000], + "expected_ms": 6000 + }, + { + "name": "an inverted pair is dropped by both sides, like an unbounded one", + "spans": [[900, 400], [1000, 2000]], + "unbounded_ms": [], + "expected_ms": 1000 + } ] } diff --git a/tests/test_timing_union_parity.py b/tests/test_timing_union_parity.py index 7579a39b1..b63b9de75 100644 --- a/tests/test_timing_union_parity.py +++ b/tests/test_timing_union_parity.py @@ -20,6 +20,8 @@ import pytest +from coder_eval.models import CommandTelemetry +from coder_eval.streaming.collector import main_thread_tool_spans from coder_eval.timing import busy_ms, union_ms @@ -30,6 +32,7 @@ _CORPUS = json.loads(_FIXTURE.read_text()) _CASES = _CORPUS["cases"] _UNION_CASES = _CORPUS["union_cases"] +_UNBOUNDED_CASES = _CORPUS["unbounded_cases"] def _at(offset_ms: float) -> datetime: @@ -57,6 +60,45 @@ def test_union_ms_matches_the_shared_corpus(case: dict) -> None: assert union_ms(spans) == pytest.approx(case["expected_ms"]) +@pytest.mark.parametrize("case", _UNBOUNDED_CASES, ids=[c["name"] for c in _UNBOUNDED_CASES]) +def test_an_unbounded_call_contributes_nothing_to_the_union(case: dict) -> None: + """The POLICY half, replayed through the PRODUCTION selector. + + ``union_ms`` alone cannot pin this: by the time a span list reaches it the + unbounded calls are already gone. The decision lives one layer up, in + ``main_thread_tool_spans``'s ``is not None`` filter — so that is what this + replays, over hand-built ``CommandTelemetry`` rows shaped the way a harness + records them. The TypeScript twin (``toolExecutionMs``) makes the same + decision inline, which is why the corpus and not either implementation owns + the answer. + """ + commands = [ + CommandTelemetry( + tool_id=f"bounded-{i}", + tool_name="Bash", + timestamp=_at(start), + execution_started_at=_at(start), + execution_completed_at=_at(end), + result_status="success", + ) + for i, (start, end) in enumerate(case["spans"]) + ] + commands += [ + # Timed, never bounded: exactly the codex `Bash` shape, and the shape + # the out-of-tree delegate-sdk still reports. + CommandTelemetry( + tool_id=f"unbounded-{i}", + tool_name="Bash", + timestamp=_BASE, + duration_ms=duration, + result_status="success", + ) + for i, duration in enumerate(case["unbounded_ms"]) + ] + spans = main_thread_tool_spans([], commands) + assert union_ms(spans) == pytest.approx(case["expected_ms"]) + + def test_the_typescript_half_replays_the_same_file() -> None: """A parity corpus only one side reads is not a parity corpus. @@ -73,3 +115,7 @@ def test_the_typescript_half_replays_the_same_file() -> None: "the TS test must also iterate `union_cases`, the half that pins toolExecutionMs's " "own min/max extent against union_ms's" ) + assert re.search(r"\.unbounded_cases\b", source), ( + "the TS test must also iterate `unbounded_cases`, the half that pins the POLICY: a " + "call the harness timed but did not bound contributes nothing on either side" + ) From e0633a1a26bf348ccf83502b916dcf9e32b37327 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:24:35 -0700 Subject: [PATCH 44/54] =?UTF-8?q?docs(harness):=204/8=20=E2=80=94=20say=20?= =?UTF-8?q?what=20each=20harness's=20clock=20basis=20actually=20is?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parity table's `clock basis` row called OpenCode "CLI epoch ms, `datetime.now()` only as a fallback". That contradicted the table's own RAW window row ten lines above it — "harness clock: previous `step_finish` to this one" — and the code agrees with the second, not the first. OpenCode's window bounds are host `datetime.now()` (`opencode_agent.py:362`, `:696`) while its TOOL SPANS are CLI epoch ms (`:406`, `:462`). It is MIXED, and the row now says so. That misstatement was load-bearing, which is why it is worth more than a cell edit. The paragraph explaining why Codex and OpenCode are not converted gave them ONE reason — "converting only the window bounds would put two bases inside one `busy_ms` subtraction" — and that describes a state OpenCode is already in. Codex is the one it is true of: both halves come from `_ms_to_dt`, so converting either alone creates the mix. OpenCode's real argument is different and smaller: a monotonic-derived anchor would trade a narrow NTP exposure on the bounds for intra-turn drift against the CLI's own tool stamps. The two now have separate, true reasons. `TurnClock`'s docstring was asserting a current property of two other modules, which is the shape that drifted here in the first place. It now names which harnesses use it and points at the table, which is the designated SSOT for per-harness composition. And the tree stated two positions on one clamp. `decompose_turn` defends it — a measured inversion IS a real zero, both ends were observed — while `_seed_first_generation_window` called the identical clamp "the exact confusion CE058 exists to prevent". The second is reworded to say what it actually meant: the head was measured against the WRONG INSTANT, which is a different fault from the clamp. The clamp itself is untouched. Also registers the two findings this plan cut on evidence — A2-full (it retires neither lint rule it claimed to, and the new seam assertion guards the property at runtime) and C2's counter (CE064 removed the reachable cause) — each with the trigger that would reopen it, so the reasoning is not re-derived later. No executable line changed. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- .claude/harness-candidates.md | 36 ++++++++++++++++++++++ docs/agents/HARNESS_PARITY.md | 26 +++++++++++----- src/coder_eval/agents/claude_code_agent.py | 18 ++++++----- src/coder_eval/timing.py | 23 +++++++------- 4 files changed, 78 insertions(+), 25 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 1dfb913a3..6a41b7c16 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -791,3 +791,39 @@ divergences, so the deferred-work record is one place. Measurements in and nothing else. The timing half had a unit test that stayed green while the content half was broken, so the two are now asserted separately (`test_a_duplicate_turn_end_does_not_republish_the_previous_content`). + +## From the turn-timing consolidation (2026-09-12) + +Two findings from `c/turn-audit.md` were CUT during planning, on evidence. They +are registered here with their corrected cost/benefit and the trigger that would +reopen them — not because they are cheap guards waiting to be written, but +because the reason they were cut is the part a later reader will otherwise +re-derive from scratch. + +- [ ] **A2-full — reducers publish BOUNDS only; the collector derives the raw + window.** The audit justified this partly as retiring two lint rules. Neither + holds. `CE058` form 1 is a generic keyword rule over five constructors + (`ce058_no_timing_literal.py:71-76`) and stays live whatever a reducer + publishes. `CE059` keys its exemption on `generation_duration_ms=None` being + PRESENT at the call site (`ce059_generation_window_is_two_reads.py:68`), so + removing the kwarg makes `claims_a_window` true at the three legitimate + placeholder sites and forces a rule REWRITE rather than a retirement. Net + cost: five reducers, a regeneration of every golden, and a CE059 rework; net + benefit: SSOT alone. **Deferring it is safe because the seam assertion in + `timing.subtract_tool_time` now checks the property at runtime** — a group's + raw total must equal the span its own bounds describe — which also covers a + third-party agent registered through the `coder_eval.plugins` SPI, where no + lint rule scoped to `agents/` reaches. REVISIT IF: that assertion ever has to + be relaxed for a legitimate reducer, which would mean the equality is no + longer the contract and storage has stopped paying for itself. + +- [ ] **C2 — a persisted `clock_inversions` counter.** The audit wanted the + number of clamped negatives recorded on the turn. CE064 removed the reachable + cause (the cross-basis head/tail comparison), and a field nothing may ever + read is YAGNI. The DOC half was done instead: the two contradictory clamp + docstrings now state one position — a measured inversion IS a real zero, + because both ends were observed (`timing.decompose_turn`), and claude-code's + case was never about the clamp but about the head being measured against the + wrong instant. REVISIT IF: an inversion is observed on a live run after + CE064, which would mean a basis is still mixed somewhere the rule cannot see + (the plugin SPI, or a harness whose spans come from a CLI). diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 4eb7fb1e0..d6fb384e8 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -34,7 +34,7 @@ wall clock its numbers account for. | `generation_completed_at` | set | `None` — see below | `None` | `None` | `None` | | `message_id` source | SDK `message_id`; `None` when the stream carries none; `subagent-` for a synthesized sub-agent terminal | synthetic `turn_id-msg-N`, shared across the sub-messages of one generation; `turn_id-subagent-N` for recovered sub-agent generations | synthetic `turn_id-msg-N`, one per generation | CLI `messageID`; `None` when absent | CLI `responseId`; `None` when absent | | `Σ generation + ∪ tool + head + tail ≈ turn duration` | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | yes [^identity] | -| clock basis for recorded stamps | one `TurnClock` per turn | SDK epoch ms — the subprocess's own clock, unreachable from the host | one `TurnClock` per turn | CLI epoch ms (`_epoch_ms_to_dt`), `datetime.now()` only as a fallback | one `TurnClock` per turn | +| clock basis for recorded stamps | one `TurnClock` per turn | SDK epoch ms (`_ms_to_dt`) — the subprocess's own clock, unreachable from the host, for BOTH window bounds and tool spans | one `TurnClock` per turn | **MIXED**: window bounds on the host `datetime.now()` (`:362`, `:696`); tool spans on CLI epoch ms (`_epoch_ms_to_dt`, `:406`/`:462`) | one `TurnClock` per turn | | turn bracket (`AgentStartEvent` / `AgentEndEvent`) stamp | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms bounds | the same `TurnClock` (**CE064**) | raw `datetime.now()` — consistent with its epoch-ms tool spans | the same `TurnClock` (**CE064**) | | window built by `timing.py::close_window` | yes | yes | yes | yes | yes | @@ -173,12 +173,24 @@ generation that arrives as a tool result and is never streamed excludes the message from `subtract_tool_time` and from the head/tail bracket. A stamp no bucket reads has no basis to share. -Codex and OpenCode are **not** converted and the hazard is narrowed rather than -removed. Their tool spans are the CLI's own epoch-millisecond stamps -(`codex_agent.py::_ms_to_dt`, `opencode_agent.py::_epoch_ms_to_dt`), which -cannot be re-derived host-side; converting only the window bounds would put two -bases inside one `busy_ms` subtraction, relocating the defect instead of -removing it. Both therefore keep the naive-local exposure. +Codex and OpenCode are **not** converted, and their reasons are DIFFERENT — they +were stated as one, and that reading described a state OpenCode is already in. + +**Codex** is genuinely single-basis: both its window bounds and its tool spans +come from `_ms_to_dt` over the CLI's own epoch milliseconds, which cannot be +re-derived host-side. Converting only the window bounds would put two bases +inside one `busy_ms` subtraction — relocating the defect instead of removing it — +so it stays whole, and keeps the naive-local exposure. + +**OpenCode is already mixed, today.** Its window bounds are host +`datetime.now()` (`opencode_agent.py:362` at `step_start`, `:696` at +`step_finish`) while its tool spans are CLI epoch ms (`:406`, assigned to +`execution_started_at` at `:420`, and `:462`), so the two bases already meet +inside one subtraction. The argument for leaving it is therefore not the Codex +one: it is that a monotonic-derived anchor would trade a narrow NTP exposure on +the window bounds for intra-turn drift against the CLI's own tool stamps, which +is the larger of the two. The mixed basis is recorded here rather than defended +as uniform. Deadlines on every harness stay on raw `time.monotonic()` and must — a deadline may not move when the wall clock steps. diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 5715dfc28..4e792b0a8 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -523,13 +523,17 @@ def _seed_first_generation_window(self) -> None: closes. Without this the mark is stamped in ``__init__``, BEFORE - ``AgentStartEvent`` is emitted, so the head is a small negative that - ``decompose_turn`` clamps to ``0.0`` — a clamped inversion published as - "measured, and instant", which is the exact confusion CE058 exists to - prevent everywhere else. Everything the CLI spent booting, resolving a - provider and reaching its first token was booked as msg0's generation - instead: ~3.6 s per turn on this harness, inflating every generation - figure, the Generation split and the 10 s slow-generation bar. + ``AgentStartEvent`` is emitted, so the head comes out negative and + ``decompose_turn`` clamps it to ``0.0``. The fault is NOT the clamp — + that function's own docstring is right that a measured inversion is a + real zero, because both ends were observed. The fault is that the head + was measured against the WRONG INSTANT: the mark sat before the turn + bracket rather than at the first observed model output, so the interval + being measured was not the one the field is defined as. Everything the + CLI spent booting, resolving a provider and reaching its first token was + booked as msg0's generation instead: ~3.6 s per turn on this harness, + inflating every generation figure, the Generation split and the 10 s + slow-generation bar. The old rejection rested on this harness running the model in-process. It does not: ``claude-agent-sdk`` spawns the ``claude`` CLI over diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index e5a902134..dce2290a7 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -59,21 +59,22 @@ class TurnClock: ONE PER TURN, never module-level and never reused across turns: a long run would accumulate drift between the pair and real wall time. The turn-state constructors take it as an argument so the lifetime is visible in the - signature, and so tests can inject a fake instead of monkeypatching a - module global out from under the reducer. + signature, and so a unit test can pass a fake straight in. An end-to-end + test driving ``communicate()`` cannot: the state is built inside it, out of + the caller's reach, so those replace this class through the agent module + instead (``tests/_bracket_clock.py``). Both reach the same object. NOT for deadlines. Those stay on ``time.monotonic()`` directly: a deadline must not move when the wall clock steps. - Codex and OpenCode deliberately do NOT use it. Their tool spans are the - CLI's own epoch-millisecond stamps, unreachable from the host, so - converting only the window bounds would put two bases inside one - ``busy_ms`` subtraction — relocating the defect instead of removing it. - - claude-code DOES use it, and is the third of the three that can. Its one - remaining raw ``datetime.now()`` is the synthesized sub-agent terminal - message, whose bounds are an admitted placeholder that no bucket reads — - see the comment at that call site. + Antigravity, Pi and claude-code use it — for their window bounds and, since + CE064, for their turn bracket. Codex and OpenCode do not. This docstring + deliberately says no more than that: asserting a current property of two + other modules from here is the drift that put a wrong OpenCode row in the + parity table for months, and that table is the designated SSOT for + per-harness composition. See the `clock basis for recorded stamps` row in + docs/agents/HARNESS_PARITY.md, and the paragraph below it for why each + unconverted harness stays that way. """ def __init__(self) -> None: From 74701da65d8869e3bdae9cc6cf96c207d0bdabc2 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:53:23 -0700 Subject: [PATCH 45/54] =?UTF-8?q?refactor(timing):=205/8=20=E2=80=94=20one?= =?UTF-8?q?=20home=20for=20the=20rules,=20and=20a=20stored=20tool=20union?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes that all serve the same end: one producer per published figure. `main_thread_tool_spans` and `subtract_tool_time` move out of `streaming/collector.py` into `timing.py`, the declared cycle-free home for timing rules. `reports_stats` was reaching across a layer into `streaming/` to get the span SELECTOR, which is the rule the collector measures all four buckets against; a report module importing from the capture layer to borrow it is the shape that ends with a second typed copy. `timing.py` gains a `coder_eval.models` import, verified cycle-free at runtime: importing models loads neither `timing` nor `streaming`. `TurnRecord` gains ONE field, `tool_union_ms`, written by the collector from the same span set the head and tail are measured against. It is the one bucket a dict consumer cannot cheaply reproduce — union arithmetic plus a sub-agent filter. `generation_total_ms` was considered and cut: the reconciliation entry exists so a consumer SUMS the message stream rather than reading a separate aggregate, and storing a generation total would create a value that can silently disagree with the stream `subtract_tool_time` has just rewritten. CE058 had to widen for it, and that is a deliverable rather than a detail: its `_TIMING_NAME` matched `duration_ms`, `[a-z_]*_duration_ms` and the `harness_*` pair, and `tool_union_ms` matched none of them — so `TurnRecord(tool_union_ms=0.0)` would have shipped outside the None-vs-0 guard every sibling bucket has, even though `TurnRecord` was already in the rule's constructor set. Naming the field `tool_union_duration_ms` to inherit the generic arm for free was rejected: the two fields beside it needed their own arm for the same reason, and one spelling across the buckets is worth two lines of regex. `_overhead_ms`'s `tool_spans` parameter is now required. Its fallback would have built a SECOND span set, which the comment at its only call site already said must never happen — the subtraction and the head/tail have to agree about which calls exist, or the buckets stop being disjoint. `reports_stats` prefers the stored value and falls back to deriving it, with `is not None` rather than truthiness: a stored `0.0` is a measurement (spans were recorded and occupied no measurable time) and must not be silently replaced by a re-derivation. All 34 goldens regenerated; every diff is exactly one added key, `null` where the fixture recorded no bounded span and scrubbed where it did. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- CLAUDE.md | 2 +- src/coder_eval/agents/antigravity_agent.py | 2 +- src/coder_eval/agents/codex_agent.py | 2 +- src/coder_eval/agents/pi_agent.py | 4 +- src/coder_eval/models/results.py | 23 ++ src/coder_eval/models/telemetry.py | 2 +- src/coder_eval/reports_stats.py | 25 ++- src/coder_eval/streaming/collector.py | 197 ++---------------- src/coder_eval/timing.py | 179 +++++++++++++++- tests/_fixtures/golden_streams/_scrub.py | 5 +- .../antigravity_a_single_text_turn.json | 1 + .../antigravity_b_tool_call_resolved.json | 1 + ...y_c_thinking_and_tool_same_generation.json | 1 + .../expected/antigravity_d_orphaned_tool.json | 1 + .../antigravity_e_multi_generation.json | 1 + .../expected/claude_a_single_text_turn.json | 1 + .../expected/claude_b_tool_use_result.json | 1 + .../claude_c_multi_emission_delta.json | 1 + .../expected/claude_d_subagent_terminal.json | 1 + .../claude_e_model_usage_and_backfill.json | 1 + .../expected/claude_f_orphaned_tool.json | 1 + .../claude_g_crash_format_placeholder.json | 1 + .../claude_h1_timeout_process_error.json | 1 + .../claude_h2_process_error_crash.json | 1 + .../claude_i_in_loop_deadline_break.json | 1 + .../expected/codex_a_agent_message_only.json | 1 + .../expected/codex_b_command_execution.json | 1 + .../codex_c_reasoning_placeholder.json | 1 + .../codex_d_cross_flush_is_error.json | 1 + .../expected/codex_e_orphan_tool.json | 1 + .../expected/codex_f_collab_fallback.json | 1 + .../expected/codex_g_items_rebuild.json | 1 + .../codex_h_no_turn_completed_crash.json | 1 + .../expected/opencode_a_single_text_turn.json | 1 + .../opencode_b_tool_call_resolved.json | 1 + .../opencode_c_multi_step_tiling.json | 1 + .../expected/opencode_d_orphaned_tool.json | 1 + .../opencode_e_error_after_generation.json | 1 + .../expected/pi_a_single_text_turn.json | 1 + .../expected/pi_b_tool_call_resolved.json | 1 + .../expected/pi_c_multi_turn_tiling.json | 1 + .../expected/pi_d_orphaned_tool.json | 1 + .../expected/pi_e_error_after_generation.json | 1 + .../expected/pi_f_duplicate_turn_end.json | 1 + tests/lint/rules/ce058_no_timing_literal.py | 11 +- .../rules/ce061_window_via_close_window.py | 4 +- .../lint/rules/ce063_no_busy_ms_in_agents.py | 4 +- tests/test_codex_agent.py | 4 +- tests/test_custom_lint.py | 26 ++- tests/test_event_collector.py | 157 +++++++++++++- tests/test_models.py | 43 ++++ tests/test_pi_agent.py | 6 +- tests/test_reports_html.py | 63 ++++++ tests/test_timing_identity_contract.py | 16 +- tests/test_timing_union_parity.py | 3 +- 55 files changed, 598 insertions(+), 214 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8552e1261..857fb4470 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -236,7 +236,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE064** (in `src/coder_eval/agents/`, a module that imports `TurnClock` must pass an explicit `timestamp=` to `AgentStartEvent` and `AgentEndEvent` — the turn's OUTER bounds, which no other rule looks at, since CE058-CE061 all scope to `AssistantMessage` and the bracket is not one. `timing.decompose_turn` produces `harness_startup_ms` / `harness_teardown_ms` by subtracting a generation-window bound from a bracket timestamp, so the two must share a basis; all three clocked harnesses derived their bounds from the `TurnClock` and let the bracket fall back to `StreamEvent.timestamp`'s `default_factory=datetime.now`, putting a monotonic-derived stamp and a raw wall stamp inside one subtraction — the exact split `TurnClock` exists to remove, reintroduced at the one seam the clock did not own. Measured on a live antigravity turn: an `AgentEndEvent` stamped **17 us BEFORE its own last message finished**, which cannot happen (the event is constructed strictly after the final flush), and `decompose_turn` clamped that negative and published `0.0` — "measured, and instant", the CE058 confusion arrived at from the other direction — for a harness whose real tail is ~0.1 ms; it now records 0.035 ms. It surfaced on one harness only because the drift is tens of microseconds and antigravity is the only one that holds its process across turns, so nothing happens between its last flush and its end event; every other harness books a tail of 7-543 ms, where the drift is invisible rather than absent — which is why the fix is at every clocked site rather than at that one. SCOPE IS DERIVED, never a harness list: codex and opencode take their spans from the CLI's own epoch stamps, deliberately have no `TurnClock`, and are correctly invisible to the rule — a raw bracket is CONSISTENT with their bounds — and the day either adopts a clock the rule starts applying with no edit. BLIND SPOT, in the rule's docstring: presence, not correctness. It cannot tell `self.clock.now()` from a `datetime.now()` spelled out at the call site, because the three harnesses legitimately reach their clock three ways; the guard for the SOURCE is behavioural (`tests/_bracket_clock.py` injects a stand-in anchored a year from real time, so a reverted argument fails by a year rather than by the microseconds that separate the two clocks), which is the division of labour CE060 states — a rule removes the SILENT case, a default nobody chose), **CE063** (no module in `src/coder_eval/agents/` may import `busy_ms` — tool execution comes out of a generation window in exactly ONE place, `streaming/collector.py::subtract_tool_time`. Five reducers used to do it themselves while the head and tail were already computed centrally at the same seam, and that asymmetry is where every timing defect on this branch lived — none of them in the arithmetic, all of them in the bookkeeping AROUND it: when to reset a per-step span list (clearing it at `step_start` wiped a span before the flush could subtract it, a 100% overstatement of that window), when to clear a spent start stamp (a second flush with no intervening start republished the previous span — 3000 ms of generation for a 2000 ms turn), when to advance the mark. A sixth harness reaching for `busy_ms` rebuilds that, and its tool time is then subtracted TWICE — by the reducer and again by the collector — under-reporting generation on one harness only, which takes a corpus comparison to notice. A separate id from CE061 rather than a rebody: CE061 asks where a window's ARITHMETIC came from and four reducers still call `close_window`, so its property is live and unsuperseded; this asks whether a reducer subtracts at all. It deliberately does NOT reuse CE061's `_imports_the_helper`, whose bare-module-import branch exists so `timing.close_window(...)` counts as reaching the helper — inverted into a ban that branch flags four of the five reducers. CE061 is now **exemption-free**: claude-code was its one permanent `# noqa` and, with the subtraction moved, calls the shrunken `close_window` like the other four), **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. The damage was not confined to the timeline, which is why "only granularity is lost" was the wrong way to describe it: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose prompt-cache cascade is quadratic in that count, so a single-shot Antigravity run had every cascade coefficient pinned at zero; the `Messages` count and the 10 s slow-generation bar were per-turn too. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). +Recent additions, each traceable to a shipped defect: **CE064** (in `src/coder_eval/agents/`, a module that imports `TurnClock` must pass an explicit `timestamp=` to `AgentStartEvent` and `AgentEndEvent` — the turn's OUTER bounds, which no other rule looks at, since CE058-CE061 all scope to `AssistantMessage` and the bracket is not one. `timing.decompose_turn` produces `harness_startup_ms` / `harness_teardown_ms` by subtracting a generation-window bound from a bracket timestamp, so the two must share a basis; all three clocked harnesses derived their bounds from the `TurnClock` and let the bracket fall back to `StreamEvent.timestamp`'s `default_factory=datetime.now`, putting a monotonic-derived stamp and a raw wall stamp inside one subtraction — the exact split `TurnClock` exists to remove, reintroduced at the one seam the clock did not own. Measured on a live antigravity turn: an `AgentEndEvent` stamped **17 us BEFORE its own last message finished**, which cannot happen (the event is constructed strictly after the final flush), and `decompose_turn` clamped that negative and published `0.0` — "measured, and instant", the CE058 confusion arrived at from the other direction — for a harness whose real tail is ~0.1 ms; it now records 0.035 ms. It surfaced on one harness only because the drift is tens of microseconds and antigravity is the only one that holds its process across turns, so nothing happens between its last flush and its end event; every other harness books a tail of 7-543 ms, where the drift is invisible rather than absent — which is why the fix is at every clocked site rather than at that one. SCOPE IS DERIVED, never a harness list: codex and opencode take their spans from the CLI's own epoch stamps, deliberately have no `TurnClock`, and are correctly invisible to the rule — a raw bracket is CONSISTENT with their bounds — and the day either adopts a clock the rule starts applying with no edit. BLIND SPOT, in the rule's docstring: presence, not correctness. It cannot tell `self.clock.now()` from a `datetime.now()` spelled out at the call site, because the three harnesses legitimately reach their clock three ways; the guard for the SOURCE is behavioural (`tests/_bracket_clock.py` injects a stand-in anchored a year from real time, so a reverted argument fails by a year rather than by the microseconds that separate the two clocks), which is the division of labour CE060 states — a rule removes the SILENT case, a default nobody chose), **CE063** (no module in `src/coder_eval/agents/` may import `busy_ms` — tool execution comes out of a generation window in exactly ONE place, `timing.py::subtract_tool_time`. Five reducers used to do it themselves while the head and tail were already computed centrally at the same seam, and that asymmetry is where every timing defect on this branch lived — none of them in the arithmetic, all of them in the bookkeeping AROUND it: when to reset a per-step span list (clearing it at `step_start` wiped a span before the flush could subtract it, a 100% overstatement of that window), when to clear a spent start stamp (a second flush with no intervening start republished the previous span — 3000 ms of generation for a 2000 ms turn), when to advance the mark. A sixth harness reaching for `busy_ms` rebuilds that, and its tool time is then subtracted TWICE — by the reducer and again by the collector — under-reporting generation on one harness only, which takes a corpus comparison to notice. A separate id from CE061 rather than a rebody: CE061 asks where a window's ARITHMETIC came from and four reducers still call `close_window`, so its property is live and unsuperseded; this asks whether a reducer subtracts at all. It deliberately does NOT reuse CE061's `_imports_the_helper`, whose bare-module-import branch exists so `timing.close_window(...)` counts as reaching the helper — inverted into a ban that branch flags four of the five reducers. CE061 is now **exemption-free**: claude-code was its one permanent `# noqa` and, with the subtraction moved, calls the shrunken `close_window` like the other four), **CE060** (in `src/coder_eval/agents/`, every `AssistantMessage(...)` must pass `message_id` explicitly — an identity invariant, which is why it is its own id rather than a second arm of CE058/CE059, both of which are about timing. Antigravity omitted the kwarg, so the field defaulted to `None` on every message it ever recorded, and the evalboard — which groups assistant emissions by `message_id` and falls back to a `SAME_EMISSION_GAP_MS` wall-clock gap when either side lacks one — collapsed a whole turn's generations into ONE timeline row as soon as the harness's generation windows became contiguous (the gap is then exactly 0 ms, always). Nothing failed: the consumer SUMS the group, so the totals and the reconciliation invariant stayed right, and the golden snapshots had ratified the `null` on the day they were written — a snapshot is regenerated from whatever the code currently does, so it catches a later change and never an initial omission. The damage was not confined to the timeline, which is why "only granularity is lost" was the wrong way to describe it: a grouped emission is one API call to the evalboard's thinking-cost simulator, whose prompt-cache cascade is quadratic in that count, so a single-shot Antigravity run had every cascade coefficient pinned at zero; the `Messages` count and the 10 s slow-generation bar were per-turn too. Unlike its two siblings it **derives its constructor set from each module's own `coder_eval.models` imports** instead of hardcoding the spelling, which closes exactly the blind spot CE058's clause below concedes: `claude_code_agent.py` binds only `AssistantMessage as AssistantMessageTelemetry`, so a name list guards that file's two construction sites purely by coincidence, and an arbitrary `as Msg` is missed outright. Widening CE058/CE059 the same way is recorded in `.claude/harness-candidates.md`. BLIND SPOT, in the rule's docstring: the runtime `None` — the kwarg must be PRESENT, not statically non-`None`, because OpenCode's `messageID` and Pi's `responseId` legitimately evaluate to `None` when the CLI omits them, and passing a fallback expression *is* deciding), **CE058** (in `src/coder_eval/`, an unknown timing value may not become a numeric literal — `duration_ms is None` means *never timed* and `0.0` means *timed and instant*, so writing the literal publishes the second while meaning the first. One invariant, one id, five syntactic forms — a zero constructor keyword, `x or 0`, `x if x is not None else 0.0`, `if x.duration_ms is None: x.duration_ms = 0.0`, and a `model_copy(update={...})` dict (the shape the Antigravity DONE path writes through, which a keyword-only rule cannot see). Antigravity constructed EVERY message with `generation_duration_ms=0.0`, so the task page's Generation cell read `0ms` and its breakdown rendered `0%` for months with nothing failing; Codex published the SDK's `0.0` as a measured command duration, so `avg_command_time_ms` divided real milliseconds by a command count of which 70 of 211 in one nightly had never been timed. The fourth form is the one no existing rule shape covered and is where a live instance was hiding — `claude_code_agent._finalize_commands` set `0.0` on every command force-closed without a tool result, in the one harness a timing audit had called healthy. BLIND SPOT, stated in the rule's docstring: form 1 keys on the callee's spelling, so renaming the `AssistantMessageTelemetry` import alias silently disarms it there), **CE059** (in `src/coder_eval/agents/`, an `AssistantMessage` may not receive the same `ast.Name` for both `started_at` and `completed_at` — the Antigravity reducer read `datetime.now()` once and passed it as both bounds, so `started_at == completed_at` on 368 of 368 sampled messages. A separate id from CE058 because it is a separate invariant, a zero-length window whatever the duration field says, and one invariant per id is what makes a `# noqa` mean one thing. It does NOT fire when the same call passes `generation_duration_ms=None`: a call that says, in the field built to say it, that no window was measurable is not claiming one — that exemption is what keeps the rule pointed at the misleading case instead of accumulating four permanent suppressions on the rollout-rebuild and sub-agent-synthesis sites), **CE056** (no bare `CODER_EVAL_IN_CONTAINER` literal outside `models/container_paths.py` — the CE053 shape again: a rename-safety constant that shipped beside the literal it replaced, and the straggler was the single WRITER, so a rename would have disarmed four security/correctness gates at once with nothing failing; CE052 cannot catch it because that rule inspects `if` guards and the writer is not one), **CE055** (a criterion `path:` in `tasks/` must be sandbox-relative — an absolute path is joined onto the sandbox root, which DISCARDS the root, so containment refuses it and the criterion can never match whatever the agent does; two in-tree tasks were broken this way and the pair is the argument for a static rule on top of the runtime `CheckerMisuseError`: `byod_smoke_test` IS in a CI bucket and produced only `Results: 7/8 succeeded` plus a gating 0.0 reading "file does not exist" for a file that existed, while `dockerfile_build_example` is in NO bucket, so nothing ran it and no runtime guard was ever reached — the fix is never to relax containment but to say what the criterion means, `run_command: test -f /opt/marker`, a claim about the container IMAGE rather than about the agent's workspace), **CE054** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record or run-LOG filename literal outside `path_utils` — widened to `docker.log` / `grade.docker.log` / `task.log` / `grade.log` after the same shape recurred: `docker.log` was produced in `isolation/` and consumed in `orchestration/` as three unrelated literals, and because the consumer guards its copy with `is_file()`, a rename would have silently discarded the only record of why a grading container failed — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's), **CE047** (every onboarding/marketing surface — README, `docs/index.md`, `docs/comparison.md`, `docs/llms.txt`, `mkdocs.yml`'s `site_description`, the Pages stub, and pyproject's `description`/`keywords` — must name every built-in `AgentKind`; OpenCode shipped while four of those seven still listed three harnesses, and nothing failed), **CE057** (a module copied into the recorder directory beside a generated sandbox shim — `models.sandbox.SIDECAR_MODULES`, currently `argv_match.py` — may import stdlib only. The failure is silent: the sidecar runs where `coder_eval` and its dependencies are not installed, so one package import makes every shadowed CLI die with an ImportError the agent reads as "the tool is broken", costing a whole run to diagnose. The rule derives its target set from that exported tuple and a test asserts it matches a file that exists — a lint rule guarding zero files must fail, not pass). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/src/coder_eval/agents/antigravity_agent.py b/src/coder_eval/agents/antigravity_agent.py index ec7b884a1..209a7e6de 100644 --- a/src/coder_eval/agents/antigravity_agent.py +++ b/src/coder_eval/agents/antigravity_agent.py @@ -1108,7 +1108,7 @@ def _flush_generation(self, gen: TokenUsage, reasoning_tokens: int) -> None: # # This harness interleaves a tool INTO a window rather than tiling # around it, so the window legitimately contains time that is not model - # time. `EventCollector.subtract_tool_time` clips the union to these + # time. `timing.subtract_tool_time` clips the union to these # bounds and takes it out. Measured here before any of that existed: a # Bash opening 1.7 ms before the flush drove Sum(generation) + # Sum(command) 0.26 ms PAST the turn wall, on a turn whose whole diff --git a/src/coder_eval/agents/codex_agent.py b/src/coder_eval/agents/codex_agent.py index cb438f68d..649818b1f 100644 --- a/src/coder_eval/agents/codex_agent.py +++ b/src/coder_eval/agents/codex_agent.py @@ -480,7 +480,7 @@ def _flush_message(self, last: Any) -> None: # The RAW window. It is extended to the LAST item's completion, so a # generation containing a tool call already CONTAINS that tool's # execution — but taking it back out is no longer this reducer's job. - # `EventCollector.subtract_tool_time` does it for all five, which is + # `timing.subtract_tool_time` does it for all five, which is # also what makes the sub-message split below safe: the two specs share # these bounds, so the collector groups them and subtracts the overlap # ONCE rather than once per part. diff --git a/src/coder_eval/agents/pi_agent.py b/src/coder_eval/agents/pi_agent.py index 4e6ec2bba..604eb3590 100644 --- a/src/coder_eval/agents/pi_agent.py +++ b/src/coder_eval/agents/pi_agent.py @@ -462,7 +462,7 @@ def _close_tool( # sweep runs is not a completion — stamping it manufactures both an # `execution_completed_at` and the `duration_ms` derived from it, and # the pair then reads as a measured span that - # `EventCollector.subtract_tool_time` takes back out of a generation + # `timing.subtract_tool_time` takes back out of a generation # window it never actually occupied. `execution_started_at` IS kept: # the CLI really did emit that start, and one bound alone forms no # span (`main_thread_tool_spans` requires both). This is the guard the @@ -613,7 +613,7 @@ def on_turn_end(self, obj: dict[str, Any]) -> None: blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) # Tile from the previous turn's end. The RAW window only — - # `EventCollector.subtract_tool_time` takes the tool union back out of + # `timing.subtract_tool_time` takes the tool union back out of # it, once, for every harness. turn_start = self.turn_started_at if self.turn_started_at is not None else completed started, generation_ms = close_window( diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 3ebe47047..6eddf3541 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -353,6 +353,29 @@ class TurnRecord(BaseModel): "caveat as harness_startup_ms. None when the turn produced no assistant message." ), ) + tool_union_ms: float | None = Field( + default=None, + description=( + "Wall milliseconds this turn's MAIN-THREAD tool calls occupied: the UNION of their " + "bounded execution intervals, never their sum. Concurrent calls occupy the wall clock " + "once — one measured antigravity turn ran two overlapping `sleep 2` calls, which sum " + "to 4.1s of a 2.1s turn — so summing books the overlap twice and can drive the " + "four-bucket residual negative, destroying the disjointness the identity rests on. " + "MAIN THREAD ONLY, the same filter the generation subtraction and the head and tail " + "use: a sub-agent's own calls sit inside the spawning Agent call's interval, which " + "the union already covers. Written once by EventCollector.build_turn_record from the " + "single span set it computes for the turn, so no consumer has to reproduce union " + "arithmetic plus a sub-agent filter over raw command dicts. A call the harness timed " + "but did not BOUND contributes nothing — it cannot be placed on the timeline, so its " + "time reads as unaccounted (see evalboard/lib/timing.ts::toolExecutionMs, which " + "applies the identical policy). " + "None when the turn recorded no bounded span at all — never 0.0, which means spans " + "were recorded and occupied no measurable time. It is also None on a MID-STREAM " + "snapshot (a record built before the terminal event), where nothing was computed " + "rather than nothing measured; the two are indistinguishable here and deliberately " + "so, because a consumer's response to both is the same — derive it or show a dash." + ), + ) token_usage: TokenUsage | None = Field( default=None, description="Token usage for this turn (if available from agent SDK)" ) diff --git a/src/coder_eval/models/telemetry.py b/src/coder_eval/models/telemetry.py index 7079b2dac..c48614be1 100644 --- a/src/coder_eval/models/telemetry.py +++ b/src/coder_eval/models/telemetry.py @@ -229,7 +229,7 @@ class AssistantMessage(BaseModel): "surfaced the message with no measurable window (a rollout rebuild, or a sub-agent " "generation delivered as a tool result). " "WRITTEN BY THE COLLECTOR, not by the agent: a reducer publishes the RAW window it " - "measured, and streaming/collector.py::subtract_tool_time takes the UNION of the " + "measured, and timing.py::subtract_tool_time takes the UNION of the " "main-thread tool intervals back out of it, once, for every harness. So this equals " "completed_at - started_at only when no tool execution overlapped the window, and a " "reader of an agent's own AssistantMessage(...) call is NOT looking at the published " diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 5bba846f5..4f5ea832a 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -23,8 +23,7 @@ TaskExperimentSummary, TurnRecord, ) -from coder_eval.streaming.collector import main_thread_tool_spans -from coder_eval.timing import union_ms +from coder_eval.timing import main_thread_tool_spans, union_ms from .path_utils import TASK_JSON_FILENAME @@ -435,16 +434,28 @@ def _sum_measured(values: Iterable[float | None]) -> float | None: def _turn_tool_union_ms(turn: TurnRecord) -> float | None: """One turn's tool execution — the UNION of its main-thread command spans. - ``None`` when the turn recorded no bounded span at all, which is different - from a turn whose tools took no time. Never the sum: concurrent calls - occupy the wall clock once, and summing them books the overlap twice. - - The span SELECTION is ``streaming.collector.main_thread_tool_spans``, not a + PREFERS THE STORED VALUE. ``EventCollector.build_turn_record`` writes + ``TurnRecord.tool_union_ms`` from the single span set it measures all four + buckets against, so reading it is how this surface and the collector are + guaranteed to agree rather than merely observed to. The derivation below is + the LEGACY path: a ``task.json`` written before that field existed carries + neither it nor any way to recover it except by recomputing, and every such + run must stay renderable. + + Note the two paths cannot be distinguished by value — both return ``None`` + for a turn with no bounded span and a float otherwise — which is why the + stored one is checked with ``is not None`` rather than by truthiness: a + stored ``0.0`` is a measurement (spans were recorded and occupied no + measurable time) and must not fall through to a re-derivation. + + The span SELECTION is ``timing.main_thread_tool_spans``, not a copy of it. That rule (which commands count, and the sub-agent exclusion) is what the collector measures the generation subtraction and the head and tail against, so a second typed implementation here is how two surfaces come to publish two different tool totals for one run. """ + if turn.tool_union_ms is not None: + return turn.tool_union_ms spans = main_thread_tool_spans(turn.messages, turn.commands) return union_ms(spans) if spans else None diff --git a/src/coder_eval/streaming/collector.py b/src/coder_eval/streaming/collector.py index bae81206e..c4949d1d8 100644 --- a/src/coder_eval/streaming/collector.py +++ b/src/coder_eval/streaming/collector.py @@ -22,8 +22,6 @@ from __future__ import annotations -import math -from collections.abc import Iterable from datetime import datetime from coder_eval.models import ( @@ -41,176 +39,7 @@ ToolEndEvent, TurnStartEvent, ) -from coder_eval.timing import busy_ms, decompose_turn - - -def main_thread_tool_spans( - messages: Iterable[TranscriptMessage], commands: Iterable[CommandTelemetry] -) -> list[tuple[datetime, datetime]]: - """Bounded execution intervals of the MAIN THREAD's tool calls. - - The span set the generation subtraction, the head and the tail are all - measured against, so they cannot disagree about which calls exist. Shared - with ``reports_stats.turn_time_buckets``, which answers the same question - about a finished ``TurnRecord`` — a second typed copy of this rule is how - two report surfaces come to publish two different tool totals for one run. - (``scripts/timing/decompose_run.py`` keeps its own, over raw ``task.json`` - dicts rather than models; that is the sanctioned third reader, and - ``tests/test_timing_close_window.py::TestTheThreeToolUnionsAgree`` pins all - three together.) - - Sub-agent tools are excluded, and that used to be the gap: ``_overhead_ms`` - filtered its GENERATIONS to the main thread and then passed EVERY command, - so its claim to keep all four buckets measuring one thread was true only by - luck. It held because a child nests inside the parent Agent call, whose own - interval the union already covers — but Codex's recovered child tools carry - the CHILD's clock, so nothing made it true by construction. The evalboard's - twin (``toolExecutionMs``) does filter, so the two agreed by accident. - - A sub-agent's tool ids are reachable only through the messages that own - them: a child generation carries ``parent_tool_use_id``, and its - ``tool_use_ids`` are the calls it made. - - An inverted pair (``end`` before ``start``) is dropped here rather than - passed on. ``busy_ms`` would discard it anyway, but ``timing.union_ms`` - documents that it does NOT filter them because its callers do — so this is - the caller keeping that true. - """ - sub_agent_tool_ids = { - tool_id - for m in messages - if isinstance(m, AssistantMessage) and m.parent_tool_use_id is not None - for tool_id in m.tool_use_ids - } - return [ - (c.execution_started_at, c.execution_completed_at) - for c in commands - if c.execution_started_at is not None - and c.execution_completed_at is not None - and c.execution_completed_at >= c.execution_started_at - and c.tool_id not in sub_agent_tool_ids - ] - - -def subtract_tool_time( - messages: list[TranscriptMessage], - spans: list[tuple[datetime, datetime]], -) -> list[TranscriptMessage]: - """Take tool execution back out of the generation windows it overlapped. - - THE one place this happens. Five reducers used to do it themselves — four - through ``close_window`` as they flushed, claude-code once at finalization — - while the head and tail were already computed centrally, right here. That - asymmetry was the complexity, and every timing defect this branch fixed - lived in the per-reducer bookkeeping around the subtraction rather than in - the subtraction itself: when to reset a span list, when to clear a start - stamp, when to advance a mark. A reducer now publishes the RAW window and - keeps only the genuinely harness-shaped decision, which is where its window - opens. - - NON-MUTATING, and the reason is aliasing rather than repeated calls. Every - agent builds its terminal event as ``AgentEndEvent(messages=list(...))`` — - that copies the LIST, not the message objects — so writing in place would - reach back into the agent's own live state from the collector, which is - exactly the layering "the collector is the sole capture seam" exists to - prevent. ``model_copy`` keeps it one-directional. It is also unconditionally - safe for any caller that builds a record twice: ``EarlyStopWatcher`` holds - one collector across a turn's tool-call rounds and calls - ``build_turn_record`` on every one. - - GROUPED BY IDENTICAL BOUNDS, not by ``message_id``. Codex splits one window - across two sub-messages (thinking and action) that share ``started_at`` and - ``completed_at`` and divide the window by output-token share; subtracting - the group's overlap from each part separately would subtract it twice and - stop the parts summing to the window. Bounds identity covers that, and it - also covers OpenCode and Pi, which can legitimately carry - ``message_id is None`` — so keying on the id would silently collapse every - id-less message of a turn into one group. - - MAIN THREAD ONLY. A sub-agent generation (``parent_tool_use_id`` set) is - skipped: its own tools are not in this span set, and the Agent call that - spawned it already covers its whole run. - - A ``generation_duration_ms`` of ``None`` means no window was ever measured - (codex's rollout rebuild, claude's synthesized sub-agent terminal), so there - is nothing to subtract from and it passes through untouched — never - coerced to ``0.0`` (CE058). Every non-``AssistantMessage`` entry — a - simulation ``UserMessage``, the appended ``ReconciliationMessage`` — passes - through by identity. - - A window entirely covered by tool execution reaches ``0.0``, and that is a - measurement rather than an absence. - - THE GROUP'S RAW TOTAL MUST EQUAL THE SPAN ITS BOUNDS DESCRIBE, and this - function raises if it does not. That equality is the contract that lets - ``generation_duration_ms`` stay a PUBLISHED field rather than one the - collector derives from the bounds: a reducer publishes the raw window it - measured, so the duration is ``completed_at - started_at`` (or, for a group - Codex split across two sub-messages, sums to it). Deriving it here instead - was considered and cut — it would cost five reducers, a regeneration of - every golden and a rewrite of CE059, whose exemption keys on the kwarg being - present at the call site — and this assertion is the sensor that makes - deferring that safe. A mismatch means a reducer narrowed or widened a window - without moving its bounds, which is the drift - ``tests/_fixtures/golden_streams/_scrub.py::assert_timing_captured``'s - "bounds that span it" check catches one replay at a time. - - It OVERLAPS with CE061 and is deliberately kept anyway. All five reducers - build the window with ``timing.close_window(mark=…, now=…)`` and write - ``started_at=started, completed_at=now``, and CE061 — now exemption-free — - forces that shape statically, so the equality is largely true by - construction. What this adds is the runtime half: a reducer that bypasses - ``close_window`` in a way an import-level check cannot see, and a - third-party agent registered through the ``coder_eval.plugins`` SPI, which - lives outside ``src/coder_eval/agents/`` where no lint rule reaches it. It - is not load-bearing on its own. - - RAISING KILLS THE TURN, and that is accepted — the same trade - ``timing._require_same_awareness`` makes at this seam. The condition is - unreachable without a reducer bug; all five are exercised by the golden - corpus and by the ms-exact identity contract. - """ - # (index, raw window ms) per group. The raw value is captured HERE, where - # the message is already narrowed to AssistantMessage, so the apportioning - # loop below needs no second narrowing. - groups: dict[tuple[datetime, datetime], list[tuple[int, float]]] = {} - for index, message in enumerate(messages): - if not isinstance(message, AssistantMessage): - continue - raw = message.generation_duration_ms - if raw is None or message.parent_tool_use_id is not None: - continue - groups.setdefault((message.started_at, message.completed_at), []).append((index, raw)) - - out = list(messages) - for (started, completed), members in groups.items(): - raw_total = sum(raw for _, raw in members) - # Nothing to apportion, and dividing by it is a ZeroDivisionError. A - # group already at zero stays at zero. - if raw_total <= 0: - continue - bounds_ms = (completed - started).total_seconds() * 1000.0 - if not math.isclose(raw_total, bounds_ms, rel_tol=1e-9, abs_tol=1e-6): - raise ValueError( - f"generation_duration_ms: a group of {len(members)} message(s) bounded " - + f"{started} -> {completed} ({bounds_ms:.6f} ms) publishes {raw_total:.6f} ms of " - + "generation. A reducer publishes the RAW window it measured, so its duration is " - + "`completed_at - started_at` (or, across the sub-messages Codex splits one window " - + "into, sums to it) — tool execution comes back out HERE, once, for every harness. " - + "A disagreement means the reducer narrowed or widened a window without moving its " - + "bounds, which makes the duration and the bounds two answers to one question and " - + "breaks the four-bucket identity. Build the window with `timing.close_window` and " - + "write `completed_at=now` (CE061), rather than adjusting the duration in place." - ) - net = max(raw_total - busy_ms(spans, started, completed), 0.0) - assigned = 0.0 - for n, (index, raw) in enumerate(members): - # The last member takes the remainder so the parts reconstruct the - # group's net exactly, rather than drifting by the rounding. - share = net - assigned if n == len(members) - 1 else round(net * (raw / raw_total), 6) - out[index] = out[index].model_copy(update={"generation_duration_ms": share}) - assigned += share - return out +from coder_eval.timing import decompose_turn, main_thread_tool_spans, subtract_tool_time, union_ms class EventCollector: @@ -287,7 +116,7 @@ def _ordered_commands(self) -> list[CommandTelemetry]: return sorted(self._commands.values(), key=lambda c: c.sequence_number) def _overhead_ms( - self, messages: list[TranscriptMessage], tool_spans: list[tuple[datetime, datetime]] | None = None + self, messages: list[TranscriptMessage], tool_spans: list[tuple[datetime, datetime]] ) -> tuple[float | None, float | None]: """The turn's head and tail — the wall clock the generations do not cover. @@ -315,6 +144,13 @@ def _overhead_ms( messages after the parent's last flush. Positional access made the result depend on append order, which nothing enforces. + ``tool_spans`` is REQUIRED, never defaulted. Its one caller computes the + set once and hands the same object to both consumers; a fallback branch + here would build a SECOND set, which is precisely what the comment at + that call site says must never happen — the subtraction and the + head/tail have to agree about which calls exist or the buckets stop + being disjoint. + MAIN THREAD ONLY, the third restriction and the same rule its two sibling call sites already apply (``codex_agent._token_usage_from_messages`` and ``scripts/timing/decompose_run.py``). A sub-agent's generations @@ -338,13 +174,9 @@ def _overhead_ms( max(m.completed_at for m in generations), self._agent_start_at, self._agent_end.timestamp if self._agent_end is not None else None, - tool_spans if tool_spans is not None else self._main_thread_tool_spans(messages), + tool_spans, ) - def _main_thread_tool_spans(self, messages: list[TranscriptMessage]) -> list[tuple[datetime, datetime]]: - """This turn's main-thread tool spans, from the reduced ToolEnd stream.""" - return main_thread_tool_spans(messages, self._commands.values()) - @staticmethod def _reconciled_messages(messages: list[TranscriptMessage], usage: TokenUsage) -> list[TranscriptMessage]: """Append a ``ReconciliationMessage`` so the transcript's token buckets @@ -442,12 +274,18 @@ def build_turn_record(self) -> TurnRecord: # not load-bearing — `_overhead_ms` reads only the bounds, the # main-thread flag and whether the duration is `None`, none of which # `subtract_tool_time` changes. Do not add a comment claiming it is.) - tool_spans = self._main_thread_tool_spans(messages) + tool_spans = main_thread_tool_spans(messages, self._commands.values()) messages = subtract_tool_time(messages, tool_spans) if token_usage is not None: messages = self._reconciled_messages(messages, token_usage) startup_ms, teardown_ms = self._overhead_ms(messages, tool_spans) + # The turn's tool bucket, stored rather than left to be re-derived. It + # is the UNION (never the sum) of the SAME span set above, so all four + # buckets are measured against one selection. `None` when no bounded + # span was recorded — a turn that ran tools and timed none is not a + # turn whose tools took no time (CE058). + tool_union = union_ms(tool_spans) if tool_spans else None return TurnRecord( iteration=end.iteration or self._iteration, @@ -466,4 +304,5 @@ def build_turn_record(self) -> TurnRecord: crash_reason=end.crash_reason, harness_startup_ms=startup_ms, harness_teardown_ms=teardown_ms, + tool_union_ms=tool_union, ) diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index dce2290a7..788a77327 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -5,7 +5,7 @@ under ``agents/`` pulls in every agent, which imports ``streaming/``. NO harness subtracts tool execution from its own generation windows. Each -publishes the RAW window it measured, and ``streaming/collector.py::subtract_tool_time`` +publishes the RAW window it measured, and ``subtract_tool_time`` below takes the UNION of the tool intervals back out of them once, for all five, at the single capture seam — the same place the head and the tail are already computed. A reducer's only remaining timing decision is where its window @@ -24,9 +24,13 @@ does, and both suites replay it. """ +import math import time +from collections.abc import Iterable from datetime import datetime, timedelta +from coder_eval.models import AssistantMessage, CommandTelemetry, TranscriptMessage + class TurnClock: """One (wall, monotonic) pair per turn; every later stamp derives from it. @@ -181,7 +185,7 @@ def union_ms(spans: list[tuple[datetime, datetime]]) -> float: tail is shared. It does NOT filter ``end < start``. EVERY caller drops those while building - its span list — ``streaming.collector.main_thread_tool_spans`` (shared by + its span list — ``main_thread_tool_spans`` below (shared by the collector and the report layer), ``_scrub.py`` and ``decompose_run.py`` — so guarding again here would be a second rule about the same input in a second place. That reasoning holds only while it stays @@ -199,7 +203,7 @@ def close_window(*, mark: datetime, now: datetime, item_start: datetime | None = The shape all five reducers share. What it returns is the RAW window — tool execution is taken back out of it once, centrally, in - ``streaming/collector.py::subtract_tool_time``, which is the only place + ``subtract_tool_time`` below, which is the only place that arithmetic lives. It used to happen here too, per flush, and in claude-code at finalization; the per-reducer bookkeeping that required (a span list, its reset rule, the set of still-open calls) is where every @@ -310,3 +314,172 @@ def decompose_turn( elapsed = (agent_ended_at - last_completed_at).total_seconds() * 1000.0 tail = max(elapsed - busy_ms(spans, last_completed_at, agent_ended_at), 0.0) return head, tail + + +def main_thread_tool_spans( + messages: Iterable[TranscriptMessage], commands: Iterable[CommandTelemetry] +) -> list[tuple[datetime, datetime]]: + """Bounded execution intervals of the MAIN THREAD's tool calls. + + The span set the generation subtraction, the head and the tail are all + measured against, so they cannot disagree about which calls exist. Shared + with ``reports_stats.turn_time_buckets``, which answers the same question + about a finished ``TurnRecord`` — a second typed copy of this rule is how + two report surfaces come to publish two different tool totals for one run. + (``scripts/timing/decompose_run.py`` keeps its own, over raw ``task.json`` + dicts rather than models; that is the sanctioned third reader, and + ``tests/test_timing_close_window.py::TestTheThreeToolUnionsAgree`` pins all + three together.) + + Sub-agent tools are excluded, and that used to be the gap: ``_overhead_ms`` + filtered its GENERATIONS to the main thread and then passed EVERY command, + so its claim to keep all four buckets measuring one thread was true only by + luck. It held because a child nests inside the parent Agent call, whose own + interval the union already covers — but Codex's recovered child tools carry + the CHILD's clock, so nothing made it true by construction. The evalboard's + twin (``toolExecutionMs``) does filter, so the two agreed by accident. + + A sub-agent's tool ids are reachable only through the messages that own + them: a child generation carries ``parent_tool_use_id``, and its + ``tool_use_ids`` are the calls it made. + + An inverted pair (``end`` before ``start``) is dropped here rather than + passed on. ``busy_ms`` would discard it anyway, but ``timing.union_ms`` + documents that it does NOT filter them because its callers do — so this is + the caller keeping that true. + """ + sub_agent_tool_ids = { + tool_id + for m in messages + if isinstance(m, AssistantMessage) and m.parent_tool_use_id is not None + for tool_id in m.tool_use_ids + } + return [ + (c.execution_started_at, c.execution_completed_at) + for c in commands + if c.execution_started_at is not None + and c.execution_completed_at is not None + and c.execution_completed_at >= c.execution_started_at + and c.tool_id not in sub_agent_tool_ids + ] + + +def subtract_tool_time( + messages: list[TranscriptMessage], + spans: list[tuple[datetime, datetime]], +) -> list[TranscriptMessage]: + """Take tool execution back out of the generation windows it overlapped. + + THE one place this happens. Five reducers used to do it themselves — four + through ``close_window`` as they flushed, claude-code once at finalization — + while the head and tail were already computed centrally, right here. That + asymmetry was the complexity, and every timing defect this branch fixed + lived in the per-reducer bookkeeping around the subtraction rather than in + the subtraction itself: when to reset a span list, when to clear a start + stamp, when to advance a mark. A reducer now publishes the RAW window and + keeps only the genuinely harness-shaped decision, which is where its window + opens. + + NON-MUTATING, and the reason is aliasing rather than repeated calls. Every + agent builds its terminal event as ``AgentEndEvent(messages=list(...))`` — + that copies the LIST, not the message objects — so writing in place would + reach back into the agent's own live state from the collector, which is + exactly the layering "the collector is the sole capture seam" exists to + prevent. ``model_copy`` keeps it one-directional. It is also unconditionally + safe for any caller that builds a record twice: ``EarlyStopWatcher`` holds + one collector across a turn's tool-call rounds and calls + ``build_turn_record`` on every one. + + GROUPED BY IDENTICAL BOUNDS, not by ``message_id``. Codex splits one window + across two sub-messages (thinking and action) that share ``started_at`` and + ``completed_at`` and divide the window by output-token share; subtracting + the group's overlap from each part separately would subtract it twice and + stop the parts summing to the window. Bounds identity covers that, and it + also covers OpenCode and Pi, which can legitimately carry + ``message_id is None`` — so keying on the id would silently collapse every + id-less message of a turn into one group. + + MAIN THREAD ONLY. A sub-agent generation (``parent_tool_use_id`` set) is + skipped: its own tools are not in this span set, and the Agent call that + spawned it already covers its whole run. + + A ``generation_duration_ms`` of ``None`` means no window was ever measured + (codex's rollout rebuild, claude's synthesized sub-agent terminal), so there + is nothing to subtract from and it passes through untouched — never + coerced to ``0.0`` (CE058). Every non-``AssistantMessage`` entry — a + simulation ``UserMessage``, the appended ``ReconciliationMessage`` — passes + through by identity. + + A window entirely covered by tool execution reaches ``0.0``, and that is a + measurement rather than an absence. + + THE GROUP'S RAW TOTAL MUST EQUAL THE SPAN ITS BOUNDS DESCRIBE, and this + function raises if it does not. That equality is the contract that lets + ``generation_duration_ms`` stay a PUBLISHED field rather than one the + collector derives from the bounds: a reducer publishes the raw window it + measured, so the duration is ``completed_at - started_at`` (or, for a group + Codex split across two sub-messages, sums to it). Deriving it here instead + was considered and cut — it would cost five reducers, a regeneration of + every golden and a rewrite of CE059, whose exemption keys on the kwarg being + present at the call site — and this assertion is the sensor that makes + deferring that safe. A mismatch means a reducer narrowed or widened a window + without moving its bounds, which is the drift + ``tests/_fixtures/golden_streams/_scrub.py::assert_timing_captured``'s + "bounds that span it" check catches one replay at a time. + + It OVERLAPS with CE061 and is deliberately kept anyway. All five reducers + build the window with ``timing.close_window(mark=…, now=…)`` and write + ``started_at=started, completed_at=now``, and CE061 — now exemption-free — + forces that shape statically, so the equality is largely true by + construction. What this adds is the runtime half: a reducer that bypasses + ``close_window`` in a way an import-level check cannot see, and a + third-party agent registered through the ``coder_eval.plugins`` SPI, which + lives outside ``src/coder_eval/agents/`` where no lint rule reaches it. It + is not load-bearing on its own. + + RAISING KILLS THE TURN, and that is accepted — the same trade + ``timing._require_same_awareness`` makes at this seam. The condition is + unreachable without a reducer bug; all five are exercised by the golden + corpus and by the ms-exact identity contract. + """ + # (index, raw window ms) per group. The raw value is captured HERE, where + # the message is already narrowed to AssistantMessage, so the apportioning + # loop below needs no second narrowing. + groups: dict[tuple[datetime, datetime], list[tuple[int, float]]] = {} + for index, message in enumerate(messages): + if not isinstance(message, AssistantMessage): + continue + raw = message.generation_duration_ms + if raw is None or message.parent_tool_use_id is not None: + continue + groups.setdefault((message.started_at, message.completed_at), []).append((index, raw)) + + out = list(messages) + for (started, completed), members in groups.items(): + raw_total = sum(raw for _, raw in members) + # Nothing to apportion, and dividing by it is a ZeroDivisionError. A + # group already at zero stays at zero. + if raw_total <= 0: + continue + bounds_ms = (completed - started).total_seconds() * 1000.0 + if not math.isclose(raw_total, bounds_ms, rel_tol=1e-9, abs_tol=1e-6): + raise ValueError( + f"generation_duration_ms: a group of {len(members)} message(s) bounded " + + f"{started} -> {completed} ({bounds_ms:.6f} ms) publishes {raw_total:.6f} ms of " + + "generation. A reducer publishes the RAW window it measured, so its duration is " + + "`completed_at - started_at` (or, across the sub-messages Codex splits one window " + + "into, sums to it) — tool execution comes back out HERE, once, for every harness. " + + "A disagreement means the reducer narrowed or widened a window without moving its " + + "bounds, which makes the duration and the bounds two answers to one question and " + + "breaks the four-bucket identity. Build the window with `timing.close_window` and " + + "write `completed_at=now` (CE061), rather than adjusting the duration in place." + ) + net = max(raw_total - busy_ms(spans, started, completed), 0.0) + assigned = 0.0 + for n, (index, raw) in enumerate(members): + # The last member takes the remainder so the parts reconstruct the + # group's net exactly, rather than drifting by the rounding. + share = net - assigned if n == len(members) - 1 else round(net * (raw / raw_total), 6) + out[index] = out[index].model_copy(update={"generation_duration_ms": share}) + assigned += share + return out diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index 8afba06a2..b842cfa5d 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -32,6 +32,9 @@ # the value itself stays out of the snapshot. "harness_startup_ms", "harness_teardown_ms", + # The turn's third wall-clock bucket, and measured the same way, so it + # varies run to run for the same reason. + "tool_union_ms", # Cost is a rate-card-dependent float (and is backfilled from the rate # card on timeout/kill), so it is masked too — keeping the snapshot # rate-card-independent. The integer TOKEN buckets stay EXACT; those are @@ -298,7 +301,7 @@ def assert_timing_captured( "booked twice — most likely a tool that ran outside every generation window and was " "left in the head or tail as well as in the tool union, or a generation window that " "kept tool time it should have subtracted (see docs/agents/HARNESS_PARITY.md — the " - "subtraction happens once, in streaming/collector.py::subtract_tool_time, so a " + "subtraction happens once, in timing.py::subtract_tool_time, so a " "double-count is a span the collector saw twice or a reducer publishing a window it " "already narrowed)" ) diff --git a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json index 915390ff4..b29372c32 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_a_single_text_turn.json @@ -50,5 +50,6 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json index bda601266..e346caafb 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_b_tool_call_resolved.json @@ -81,5 +81,6 @@ "total_cost_usd": "", "uncached_input_tokens": 120 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json index 56138f54b..89bcd10fb 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_c_thinking_and_tool_same_generation.json @@ -90,5 +90,6 @@ "total_cost_usd": "", "uncached_input_tokens": 200 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json index 02f130642..f5bcc178f 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_d_orphaned_tool.json @@ -70,5 +70,6 @@ "total_cost_usd": "", "uncached_input_tokens": 90 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json index 1b69cc78a..e9ac990bf 100644 --- a/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json +++ b/tests/_fixtures/golden_streams/expected/antigravity_e_multi_generation.json @@ -104,5 +104,6 @@ "total_cost_usd": "", "uncached_input_tokens": 330 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json index 7dd57bd45..dd99b9f5b 100644 --- a/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/claude_a_single_text_turn.json @@ -55,5 +55,6 @@ "total_cost_usd": "", "uncached_input_tokens": 50 }, + "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json index d94df272d..42b675546 100644 --- a/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json +++ b/tests/_fixtures/golden_streams/expected/claude_b_tool_use_result.json @@ -77,5 +77,6 @@ "total_cost_usd": "", "uncached_input_tokens": 80 }, + "tool_union_ms": "", "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json index 0644c58c3..80de2a507 100644 --- a/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json +++ b/tests/_fixtures/golden_streams/expected/claude_c_multi_emission_delta.json @@ -131,5 +131,6 @@ "total_cost_usd": "", "uncached_input_tokens": 237 }, + "tool_union_ms": "", "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json index 37c18d97f..e9fa5477f 100644 --- a/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json +++ b/tests/_fixtures/golden_streams/expected/claude_d_subagent_terminal.json @@ -104,5 +104,6 @@ "total_cost_usd": "", "uncached_input_tokens": 390 }, + "tool_union_ms": "", "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json index 88ee7a648..7b46dede4 100644 --- a/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json +++ b/tests/_fixtures/golden_streams/expected/claude_e_model_usage_and_backfill.json @@ -63,5 +63,6 @@ "total_cost_usd": "", "uncached_input_tokens": 500 }, + "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json index 9195adde8..f5f8cd9b2 100644 --- a/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/claude_f_orphaned_tool.json @@ -78,5 +78,6 @@ "total_cost_usd": "", "uncached_input_tokens": 60 }, + "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json index 4e60e81d1..69b9cfeee 100644 --- a/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json +++ b/tests/_fixtures/golden_streams/expected/claude_g_crash_format_placeholder.json @@ -15,5 +15,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json index c87fb5be2..8eee6edfb 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json +++ b/tests/_fixtures/golden_streams/expected/claude_h1_timeout_process_error.json @@ -15,5 +15,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json index bf0c67150..997a23868 100644 --- a/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json +++ b/tests/_fixtures/golden_streams/expected/claude_h2_process_error_crash.json @@ -15,5 +15,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json index e0a00c6bd..f44f93cec 100644 --- a/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json +++ b/tests/_fixtures/golden_streams/expected/claude_i_in_loop_deadline_break.json @@ -43,5 +43,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_union_ms": null, "user_input": "do the thing" } diff --git a/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json b/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json index b3b0e23e6..c755ee0c8 100644 --- a/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json +++ b/tests/_fixtures/golden_streams/expected/codex_a_agent_message_only.json @@ -50,5 +50,6 @@ "total_cost_usd": "", "uncached_input_tokens": 92 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json index df6526577..b57169b84 100644 --- a/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json +++ b/tests/_fixtures/golden_streams/expected/codex_b_command_execution.json @@ -81,5 +81,6 @@ "total_cost_usd": "", "uncached_input_tokens": 120 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json b/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json index 93b837621..77a89092f 100644 --- a/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json +++ b/tests/_fixtures/golden_streams/expected/codex_c_reasoning_placeholder.json @@ -77,5 +77,6 @@ "total_cost_usd": "", "uncached_input_tokens": 92 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json index 320923796..e239e5391 100644 --- a/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json +++ b/tests/_fixtures/golden_streams/expected/codex_d_cross_flush_is_error.json @@ -72,5 +72,6 @@ "total_cost_usd": "", "uncached_input_tokens": 90 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json index 81459ad05..dcaaf846c 100644 --- a/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json +++ b/tests/_fixtures/golden_streams/expected/codex_e_orphan_tool.json @@ -65,5 +65,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json index b1f83b099..3d8031b04 100644 --- a/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json +++ b/tests/_fixtures/golden_streams/expected/codex_f_collab_fallback.json @@ -123,5 +123,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json index 752850127..c73dd11b0 100644 --- a/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json +++ b/tests/_fixtures/golden_streams/expected/codex_g_items_rebuild.json @@ -43,5 +43,6 @@ "result_summary": null, "timestamp": "", "token_usage": null, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json b/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json index 4828acb6c..b0414c980 100644 --- a/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json +++ b/tests/_fixtures/golden_streams/expected/codex_h_no_turn_completed_crash.json @@ -50,5 +50,6 @@ "total_cost_usd": "", "uncached_input_tokens": 92 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json index bc1b0deaf..7a6758156 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/opencode_a_single_text_turn.json @@ -55,5 +55,6 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json index 8abfd4996..7953c6f08 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/opencode_b_tool_call_resolved.json @@ -104,5 +104,6 @@ "total_cost_usd": "", "uncached_input_tokens": 95 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json index ee573afc1..01eeb86d0 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json +++ b/tests/_fixtures/golden_streams/expected/opencode_c_multi_step_tiling.json @@ -104,5 +104,6 @@ "total_cost_usd": "", "uncached_input_tokens": 150 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json index 3ebb56912..b791f9ee7 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/opencode_d_orphaned_tool.json @@ -86,5 +86,6 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json index a757b0c84..66059b14a 100644 --- a/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json +++ b/tests/_fixtures/golden_streams/expected/opencode_e_error_after_generation.json @@ -55,5 +55,6 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json index f83268c0c..d794274ae 100644 --- a/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json +++ b/tests/_fixtures/golden_streams/expected/pi_a_single_text_turn.json @@ -55,5 +55,6 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json index f256747cf..06e3816e5 100644 --- a/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json +++ b/tests/_fixtures/golden_streams/expected/pi_b_tool_call_resolved.json @@ -153,5 +153,6 @@ "total_cost_usd": "", "uncached_input_tokens": 997 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json index 2a06910a3..deebf4700 100644 --- a/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json +++ b/tests/_fixtures/golden_streams/expected/pi_c_multi_turn_tiling.json @@ -104,5 +104,6 @@ "total_cost_usd": "", "uncached_input_tokens": 150 }, + "tool_union_ms": "", "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json index 87f6426ec..c5e8d127e 100644 --- a/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json +++ b/tests/_fixtures/golden_streams/expected/pi_d_orphaned_tool.json @@ -86,5 +86,6 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json index 33a4d72e4..f860de537 100644 --- a/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json +++ b/tests/_fixtures/golden_streams/expected/pi_e_error_after_generation.json @@ -72,5 +72,6 @@ "total_cost_usd": "", "uncached_input_tokens": 100 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json index c7d047901..db88ceeca 100644 --- a/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json +++ b/tests/_fixtures/golden_streams/expected/pi_f_duplicate_turn_end.json @@ -72,5 +72,6 @@ "total_cost_usd": "", "uncached_input_tokens": 110 }, + "tool_union_ms": null, "user_input": "do it" } diff --git a/tests/lint/rules/ce058_no_timing_literal.py b/tests/lint/rules/ce058_no_timing_literal.py index 2abe303f5..5647811d6 100644 --- a/tests/lint/rules/ce058_no_timing_literal.py +++ b/tests/lint/rules/ce058_no_timing_literal.py @@ -61,9 +61,18 @@ # arms need a leading segment for the same reason the `_duration_ms` arm does: # the shipped fields are `harness_*`, and a bare `startup_ms` is more likely a # budget than a measurement. +# +# THREE field families, not two. `tool_union_ms` is the turn's third wall-clock +# bucket, on the same model and under the same None-vs-0.0 contract as the +# `harness_*` pair — and it matched NO arm above, so `TurnRecord(tool_union_ms=0.0)` +# would have been invisible even though `TurnRecord` is already in +# `_TIMING_CONSTRUCTORS`. Naming the field `tool_union_duration_ms` to inherit +# the generic `_duration_ms` arm for free was considered and rejected: the two +# fields beside it needed their own arm for exactly this reason, and one +# spelling across the four buckets is worth two lines of regex. _TIMING_NAME = re.compile( r"^(duration_ms|generation_duration_ms|total_command_time_ms|avg_command_time_ms" - r"|[a-z_]*_duration_ms|[a-z_]*_(?:startup|teardown)_ms)$" + r"|[a-z_]*_duration_ms|[a-z_]*_(?:startup|teardown)_ms|[a-z_]*_union_ms)$" ) # The constructors that carry a timing field. Keying on the callee name is what diff --git a/tests/lint/rules/ce061_window_via_close_window.py b/tests/lint/rules/ce061_window_via_close_window.py index eacd99bb3..47e49a3e3 100644 --- a/tests/lint/rules/ce061_window_via_close_window.py +++ b/tests/lint/rules/ce061_window_via_close_window.py @@ -39,7 +39,7 @@ finalization, because a call issued by an earlier emission is still running when the next window closes — and forcing that into ``close_window`` would have meant a mode flag on a helper whose whole value is having one shape. Moving the -subtraction to ``EventCollector.subtract_tool_time`` dissolved the exception: +subtraction to ``timing.subtract_tool_time`` dissolved the exception: the collector is already the place where every span is known, so claude-code needs no separate pass and calls the same shrunken helper as the other four. ``tests/test_custom_lint.py::TestCE061WindowViaCloseWindow::test_the_rule_is_now_exemption_free`` @@ -131,7 +131,7 @@ def visit_Call(self, node: ast.Call) -> None: f"never imports {_TIMING_MODULE}.{_HELPER} — so it is computing a generation " "window of its own. Every window is the same geometry: tile from the mark, and " "keep a backwards item stamp from inverting the span. Publish that RAW span; do " - "NOT subtract tool time here — EventCollector.subtract_tool_time does it once, " + "NOT subtract tool time here — coder_eval.timing.subtract_tool_time does it once, " "for every harness, and doing it in the reducer too takes it out twice (CE063 " "guards that half). Pi got the mark wrong by measuring from its own turn start, " "and nothing caught it because the golden identity check is one-sided; " diff --git a/tests/lint/rules/ce063_no_busy_ms_in_agents.py b/tests/lint/rules/ce063_no_busy_ms_in_agents.py index 6a6d901a2..18d0e1b69 100644 --- a/tests/lint/rules/ce063_no_busy_ms_in_agents.py +++ b/tests/lint/rules/ce063_no_busy_ms_in_agents.py @@ -1,7 +1,7 @@ """CE063: a reducer may not compute its own tool subtraction. Tool execution comes out of a generation window in exactly ONE place: -``coder_eval.streaming.collector.subtract_tool_time``. Before that, five +``coder_eval.timing.subtract_tool_time``. Before that, five reducers each did it themselves — four through ``close_window`` as they flushed, claude-code once at finalization — while the head and the tail were already computed centrally at the collector seam. That asymmetry is where every @@ -63,7 +63,7 @@ _MESSAGE = ( f"imports '{_BANNED}', but a reducer does not subtract tool time any more — " - "coder_eval.streaming.collector.subtract_tool_time does it once, for every harness, " + "coder_eval.timing.subtract_tool_time does it once, for every harness, " "at the single capture seam. Publish the RAW window (close_window gives you its bounds " "and span) and let the collector clip the tool union out of it. Subtracting here too " "takes it out twice and silently under-reports generation on this harness alone." diff --git a/tests/test_codex_agent.py b/tests/test_codex_agent.py index e15b1dee2..41076cb63 100644 --- a/tests/test_codex_agent.py +++ b/tests/test_codex_agent.py @@ -2413,7 +2413,7 @@ class TestFlushMessageWindowBounds: the emission's own first stamp (`item_start`), whose `min()` against the mark is the backwards-clock defence. The tool-span arguments this class also used to cover are gone — the subtraction moved to - `EventCollector.subtract_tool_time`, and + `timing.subtract_tool_time`, and `tests/test_event_collector.py::TestSubtractToolTime` pins it there. """ @@ -2471,7 +2471,7 @@ def test_the_published_window_is_raw_and_ignores_a_call_still_open(self): """The reducer publishes the RAW span; the collector subtracts. It used to bound a still-open call at the window's end and take that - slice out here. `EventCollector.subtract_tool_time` sees every span at + slice out here. `timing.subtract_tool_time` sees every span at once, so a call is subtracted from the windows its REAL interval overlaps once it resolves — no boundary approximation, and nothing for this reducer to remember. A call that never resolves has no diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 64b1c6b18..4da38090b 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4041,7 +4041,7 @@ def test_the_rule_is_now_exemption_free(self): the exact set rather than merely that it shrank. It has earned that twice: antigravity carried a TEMPORARY suppression until it moved onto `close_window`, and claude-code carried a permanent one until the tool - subtraction moved to `EventCollector.subtract_tool_time` — at which + subtraction moved to `timing.subtract_tool_time` — at which point it could call the same shrunken helper as the other four. This test is what failed each time the reason expired. """ @@ -4501,6 +4501,28 @@ def test_ignores_a_name_that_merely_starts_with_startup(self): assert not self._run("cfg = TurnRecord(startup_ms_limit=0)") assert not self._run("x = startup_ms_limit or 0") + # The tool-union family — the turn's THIRD wall-clock bucket, on the same + # model and under the same contract, and matching no arm of the regex until + # it was widened for it. + def test_flags_a_zero_tool_union(self): + assert self._run("rec = TurnRecord(iteration=0, tool_union_ms=0.0)") + + def test_flags_the_tool_union_coalesce(self): + assert self._run("x = rec.tool_union_ms or 0") + + def test_allows_an_unmeasured_tool_union(self): + assert not self._run("rec = TurnRecord(iteration=0, tool_union_ms=None)") + + def test_allows_a_measured_tool_union(self): + assert not self._run("rec = TurnRecord(iteration=0, tool_union_ms=union_ms(spans))") + + def test_ignores_a_bare_union_ms(self): + # `union_ms` is the ARITHMETIC helper, not a published bucket, and it + # returns 0.0 for an empty span list by contract. The family needs a + # leading segment for the same reason `_startup_ms` does. + assert not self._run("x = union_ms(spans) or 0") + assert not self._run("cfg = TurnRecord(tool_union_ms_limit=0)") + # Scope + suppression. def test_is_out_of_scope_outside_src(self): assert not self._run( @@ -4674,7 +4696,7 @@ class TestCE063NoBusyMsInAgents: """CE063 flags a reducer that would subtract tool time itself. The subtraction lives once, in - `coder_eval.streaming.collector.subtract_tool_time`. A reducer that also + `coder_eval.timing.subtract_tool_time`. A reducer that also does it has its tool time taken out TWICE — once by itself, once by the collector — which under-reports generation on that harness alone. """ diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index 765e53578..ba2ced513 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -18,7 +18,7 @@ TokenUsage, TurnRecord, ) -from coder_eval.streaming.collector import EventCollector, subtract_tool_time +from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -26,7 +26,7 @@ ToolEndEvent, TurnStartEvent, ) -from coder_eval.timing import union_ms +from coder_eval.timing import subtract_tool_time, union_ms TASK_ID = "collector-test" @@ -216,6 +216,9 @@ class TestFullFieldParity: # AgentEndEvent, because no agent computes them. "harness_startup_ms", "harness_teardown_ms", + # The union of the same span set those two are measured against, + # computed once at the seam for the same reason. + "tool_union_ms", } def _full_agent_end(self) -> AgentEndEvent: @@ -956,6 +959,156 @@ def test_a_sub_agent_message_is_excluded_even_with_placeholder_bounds(self): assert out[0].generation_duration_ms == pytest.approx(900.0) +class TestTheToolUnionIsStored: + """`TurnRecord.tool_union_ms`: the turn's tool bucket, written once. + + It is the one bucket a dict consumer cannot cheaply reproduce — union + arithmetic plus the sub-agent filter — so it is stored rather than left to + four surfaces to re-derive. The generation total deliberately is NOT: that + is a one-line sum over the message stream, and the reconciliation entry + exists precisely so a consumer sums the stream instead of reading a + separate aggregate. + """ + + BASE: ClassVar[datetime] = datetime(2026, 9, 11, 9, 0, 0) + + @classmethod + def _at(cls, ms: float) -> datetime: + return cls.BASE + timedelta(milliseconds=ms) + + def _tool(self, tool_id: str, lo: float, hi: float) -> ToolEndEvent: + return ToolEndEvent( + task_id=TASK_ID, + tool=CommandTelemetry( + tool_id=tool_id, + tool_name="Bash", + timestamp=self._at(lo), + sequence_number=0, + execution_started_at=self._at(lo), + execution_completed_at=self._at(hi), + result_status="success", + ), + ) + + def _record(self, messages, tools=()) -> TurnRecord: + collector = EventCollector() + _feed( + collector, + [ + AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1, timestamp=self._at(0)), + *tools, + AgentEndEvent( + task_id=TASK_ID, + usage=TokenUsage(output_tokens=1), + messages=list(messages), + timestamp=self._at(10_000), + ), + ], + ) + return collector.build_turn_record() + + def _msg(self, lo: float, hi: float, **kwargs) -> AssistantMessage: + return AssistantMessage( + started_at=self._at(lo), + completed_at=self._at(hi), + generation_duration_ms=_span_ms(self._at(lo), self._at(hi)), + **kwargs, + ) + + def test_overlapping_calls_record_their_union_not_their_sum(self): + """The property that justifies storing the field at all. + + Two 2 s calls overlapping almost entirely occupy ~2.1 s of wall clock, + not 4.1 s. A consumer summing `duration_ms` reports more tool time in + one message than the whole task's tool bucket, which is impossible on + its face — measured on a live antigravity turn. + """ + rec = self._record( + [self._msg(0, 9_000)], + tools=[self._tool("t1", 1_000, 3_000), self._tool("t2", 1_100, 3_100)], + ) + assert rec.tool_union_ms == pytest.approx(2_100.0) + + def test_sequential_calls_record_their_total(self): + rec = self._record( + [self._msg(0, 9_000)], + tools=[self._tool("t1", 1_000, 2_000), self._tool("t2", 4_000, 4_500)], + ) + assert rec.tool_union_ms == pytest.approx(1_500.0) + + def test_a_sub_agent_tool_is_excluded(self): + """MAIN THREAD ONLY — the same filter the other three buckets use. + + The spawning Agent call's own interval already spans the child's whole + run, so counting the child's tools books that time twice. + """ + rec = self._record( + [ + self._msg(0, 9_000), + self._msg(1_000, 1_400, parent_tool_use_id="agent-call", tool_use_ids=["child-1"]), + ], + tools=[self._tool("child-1", 1_000, 1_400)], + ) + assert rec.tool_union_ms is None, "a turn whose only bounded span belongs to a child measured none" + + def test_a_turn_with_no_bounded_span_records_none_not_zero(self): + """`None` means no span was recorded; `0.0` would mean spans took no time.""" + unbounded = ToolEndEvent( + task_id=TASK_ID, + tool=CommandTelemetry( + tool_id="t1", + tool_name="Bash", + timestamp=self._at(1_000), + duration_ms=500.0, + result_status="success", + ), + ) + rec = self._record([self._msg(0, 9_000)], tools=[unbounded]) + assert rec.tool_union_ms is None + + def test_a_zero_length_bounded_span_records_zero_not_none(self): + """The other side of the same distinction: this one WAS measured.""" + rec = self._record([self._msg(0, 9_000)], tools=[self._tool("t1", 2_000, 2_000)]) + assert rec.tool_union_ms == 0.0 + + def test_a_mid_stream_record_leaves_it_unset(self): + """No terminal event means no span set was built, so nothing was measured.""" + collector = EventCollector() + _feed(collector, [AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1, timestamp=self._at(0))]) + assert collector.build_turn_record().tool_union_ms is None + + def test_the_span_set_is_computed_exactly_once(self): + """All four buckets must be measured against ONE selection. + + `_overhead_ms` used to accept `tool_spans=None` and fall back to + building its own set — a second selection, which is what the comment at + the single call site says must never happen. The parameter is now + required, so the fallback is unrepresentable rather than merely unused. + """ + import inspect + + parameter = inspect.signature(EventCollector._overhead_ms).parameters["tool_spans"] + assert parameter.default is inspect.Parameter.empty + + def test_two_builds_agree(self): + """`EarlyStopWatcher` holds one collector across a turn's rounds.""" + collector = EventCollector() + _feed( + collector, + [ + AgentStartEvent(task_id=TASK_ID, prompt="go", iteration=1, timestamp=self._at(0)), + self._tool("t1", 1_000, 3_000), + AgentEndEvent( + task_id=TASK_ID, + usage=TokenUsage(output_tokens=1), + messages=[self._msg(0, 9_000)], + timestamp=self._at(10_000), + ), + ], + ) + assert collector.build_turn_record().tool_union_ms == collector.build_turn_record().tool_union_ms + + class TestBuildTurnRecordIsIdempotent: """Building the record twice must give the same numbers. diff --git a/tests/test_models.py b/tests/test_models.py index 097840944..d4c907895 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -644,3 +644,46 @@ def test_a_historical_record_still_validates(self): ).generation_duration_ms == 1000.0 ) + + +class TestTurnRecordToolUnion: + """`tool_union_ms` is the turn's third wall-clock bucket, under the same contract. + + `None` means no bounded tool span was recorded; `0.0` means spans were + recorded and occupied no measurable time. It sits flat beside + `harness_startup_ms` / `harness_teardown_ms` rather than nested, because + those two are already consumed by name by the evalboard and the two timing + sensors, and `TurnRecord` is the shape `task.json` publishes. + """ + + @staticmethod + def _record(**overrides): + from coder_eval.models import TurnRecord + + return TurnRecord(iteration=1, user_input="go", agent_output="done", **overrides) + + def test_omitting_it_yields_none_not_zero(self): + assert self._record().tool_union_ms is None + + def test_a_measured_zero_is_still_legal(self): + assert self._record(tool_union_ms=0.0).tool_union_ms == 0.0 + + def test_round_trip_preserves_none_and_a_value(self): + from coder_eval.models import TurnRecord + + assert TurnRecord.model_validate(self._record().model_dump()).tool_union_ms is None + assert TurnRecord.model_validate(self._record(tool_union_ms=250.5).model_dump()).tool_union_ms == 250.5 + + def test_a_record_predating_the_field_still_validates(self): + """The legacy path: a `task.json` written before the field existed. + + `TurnRecord` declares no `model_config`, so pydantic's default + `extra="ignore"` applies and an absent optional validates to `None` — + which is what routes the report layer to its derive-from-commands + fallback instead of reading a value that is not there. + """ + from coder_eval.models import TurnRecord + + raw = self._record(tool_union_ms=250.5).model_dump() + raw.pop("tool_union_ms") + assert TurnRecord.model_validate(raw).tool_union_ms is None diff --git a/tests/test_pi_agent.py b/tests/test_pi_agent.py index a68d903d1..f15e14457 100644 --- a/tests/test_pi_agent.py +++ b/tests/test_pi_agent.py @@ -1109,7 +1109,7 @@ class TestGenerationWindowExcludesToolExecution: """A tool running inside a turn is not model time — asserted where it is now DECIDED. The reducer no longer subtracts anything. It publishes the RAW window, and - `EventCollector.subtract_tool_time` takes the tool union back out of it + `timing.subtract_tool_time` takes the tool union back out of it once, for all five harnesses. So these cases drive the reducer and then a real collector, and assert the PUBLISHED number — the one that reaches `task.json` — rather than an intermediate the reducer used to own. @@ -1304,7 +1304,7 @@ class TestToolSpansSurviveTheTurnBoundary: This used to be a bookkeeping problem: a per-turn span list, cleared at `turn_start` — after the window it feeds had already opened at the mark — so a call closing in the gap had its span wiped before the flush could - subtract it. That list is gone. `EventCollector.subtract_tool_time` sees + subtract it. That list is gone. `timing.subtract_tool_time` sees every span at once and clips each to the windows it overlaps, so the property now holds by construction rather than by a reset rule. @@ -1448,7 +1448,7 @@ def test_an_unresolved_orphan_is_not_given_a_completion_or_a_duration(self): `execution_completed_at` manufactures a bound, and the `duration_ms` derived from it is the distance to whenever the sweep happened to run. The pair then reads as a measured span that - `EventCollector.subtract_tool_time` takes back out of a generation + `timing.subtract_tool_time` takes back out of a generation window the tool never occupied. `execution_started_at` IS kept: the CLI really did emit that start, and one bound alone forms no span. Same rule as claude-code's `_finalize_commands` — unknown status and unknown diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py index fcc2ade08..161a1d376 100644 --- a/tests/test_reports_html.py +++ b/tests/test_reports_html.py @@ -1374,6 +1374,7 @@ def _turn( generations: list[tuple[float, float, float | None]], tools: tuple[float, float] | None = None, sub_agent: tuple[float, float, float] | None = None, + tool_union_ms: float | None = None, ) -> TurnRecord: messages: list = [ AssistantMessage(started_at=cls._at(lo), completed_at=cls._at(hi), generation_duration_ms=gen) @@ -1421,6 +1422,10 @@ def _turn( messages=messages, harness_startup_ms=startup, harness_teardown_ms=teardown, + # Left UNSET by default, which is the LEGACY shape: every test in + # this class that does not pass it exercises the derive-from-commands + # fallback, and the parity test below pins the two paths together. + tool_union_ms=tool_union_ms, ) def test_each_bucket_is_summed_across_turns(self): @@ -1548,6 +1553,64 @@ def test_sub_agent_generations_and_their_tools_are_excluded(self): assert buckets.generation_ms == pytest.approx(800.0), "the child's 400ms is not main-thread generation" assert buckets.tool_ms == pytest.approx(200.0), "and its tool is not a main-thread span" + def test_a_stored_tool_union_renders_the_same_grid_as_a_derived_one(self): + """The two paths must be indistinguishable, or a legacy run reads differently. + + Everything else in this class leaves `tool_union_ms` unset, so the + suite already covers the fallback; this is the control that the STORED + path — which every run recorded from now on takes — reaches the same + cell. + """ + from coder_eval.reports_stats import turn_time_buckets + + kwargs = {"startup": 500.0, "teardown": 100.0, "generations": [(500, 1500, 800.0)], "tools": (600, 800)} + legacy = _make_result(iterations=[self._turn(**kwargs)]) + stored = _make_result(iterations=[self._turn(**kwargs, tool_union_ms=200.0)]) + + assert turn_time_buckets(legacy) == turn_time_buckets(stored) + assert self._stat(HTMLReportGenerator().generate_task_html(legacy), "Tool exec") == self._stat( + HTMLReportGenerator().generate_task_html(stored), "Tool exec" + ) + + def test_a_stored_measured_zero_is_not_re_derived(self): + """`0.0` is a measurement and must not fall through to the fallback. + + The fallback would find this turn's bounded command and report 200ms, + so reading the stored value with truthiness instead of `is not None` + would silently replace a measurement with a re-derivation. + """ + from coder_eval.reports_stats import turn_time_buckets + + result = _make_result( + iterations=[ + self._turn( + startup=500.0, + teardown=100.0, + generations=[(500, 1500, 800.0)], + tools=(600, 800), + tool_union_ms=0.0, + ) + ] + ) + assert turn_time_buckets(result).tool_ms == 0.0 + + def test_a_legacy_record_missing_the_field_entirely_still_validates(self): + """A `task.json` written before the field existed must stay renderable. + + `TurnRecord` declares no `model_config`, so pydantic's default + `extra="ignore"` applies and an absent optional validates to `None` — + which is what routes it to the fallback. + """ + turn = self._turn(startup=500.0, teardown=100.0, generations=[(500, 1500, 800.0)], tools=(600, 800)) + raw = turn.model_dump() + raw.pop("tool_union_ms") + restored = TurnRecord.model_validate(raw) + assert restored.tool_union_ms is None + + from coder_eval.reports_stats import turn_time_buckets + + assert turn_time_buckets(_make_result(iterations=[restored])).tool_ms == pytest.approx(200.0) + def test_the_existing_four_stats_are_unchanged(self): result = _make_result(iterations=[self._turn(startup=500.0, teardown=100.0, generations=[(500, 1500, 800.0)])]) html = HTMLReportGenerator().generate_task_html(result) diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index e523fdf3e..2dd07ad90 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -61,7 +61,7 @@ parse_agent_config, ) from coder_eval.streaming.callbacks import CompositeStreamCallback -from coder_eval.streaming.collector import EventCollector, main_thread_tool_spans +from coder_eval.streaming.collector import EventCollector from coder_eval.streaming.events import ( AgentEndEvent, AgentEndStatus, @@ -69,7 +69,7 @@ ToolEndEvent, ToolEndStatus, ) -from coder_eval.timing import union_ms +from coder_eval.timing import main_thread_tool_spans, union_ms # The two CLI harnesses (opencode, codex) report their stamps as epoch @@ -156,6 +156,18 @@ def assert_identity_closes(turn: Turn) -> None: tool_ms = union_ms(main_thread_tool_spans(record.messages, record.commands)) assert record.harness_startup_ms is not None, "a turn that generated has a measured head" assert record.harness_teardown_ms is not None, "a turn that generated has a measured tail" + # The STORED bucket must equal the one just computed independently. Without + # this the ms-exact sensor would cover three of the four buckets and read + # the fourth from a re-derivation, leaving the published field unchecked on + # every harness — which is how a stored value and its consumers drift. + # `None` only when no bounded span exists, in which case the union is 0.0. + stored_tool_ms = record.tool_union_ms if record.tool_union_ms is not None else 0.0 + assert stored_tool_ms == pytest.approx(tool_ms), ( + f"TurnRecord.tool_union_ms is {record.tool_union_ms}, but this turn's main-thread " + f"command spans union to {tool_ms:.4f} ms. The collector writes the field from the same " + "span set it measures the head and the tail against, so a disagreement means the stored " + "value and the selection rule have come apart." + ) bucket_sum = record.harness_startup_ms + generation_ms + tool_ms + record.harness_teardown_ms assert bucket_sum == pytest.approx(span_ms), ( diff --git a/tests/test_timing_union_parity.py b/tests/test_timing_union_parity.py index b63b9de75..726ed7080 100644 --- a/tests/test_timing_union_parity.py +++ b/tests/test_timing_union_parity.py @@ -21,8 +21,7 @@ import pytest from coder_eval.models import CommandTelemetry -from coder_eval.streaming.collector import main_thread_tool_spans -from coder_eval.timing import busy_ms, union_ms +from coder_eval.timing import busy_ms, main_thread_tool_spans, union_ms _FIXTURE = Path(__file__).parent / "_fixtures" / "timing_union_cases.json" From 8fa3e53299ef74110d1ed522bca36ef2a0bb20f3 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:55:22 -0700 Subject: [PATCH 46/54] =?UTF-8?q?refactor(timing):=206/8=20=E2=80=94=20the?= =?UTF-8?q?=20two=20sensors=20share=20one=20selection=20rule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_scrub.py` and `decompose_run.py` each carried their own sub-agent-id derivation, stamp parser and span builder — three copies of one rule with the typed original, agreeing only because someone kept checking. They agreed by luck once at a real cost: the collector filtered its GENERATIONS to the main thread and then passed EVERY command as a tool span, and nothing failed, because a child nests inside the parent Agent call whose interval the union already covers. Codex's recovered child tools carry the CHILD's clock, so the nesting was never guaranteed. Both now validate the raw dict into a `TurnRecord` and call the one typed selector. That costs NO new production code: `TurnRecord` declares no `model_config`, so pydantic's default `extra="ignore"` applies, and a raw `task.json` turn validates — measured on 2466 turns across 2248 files, 17 days and all six agent types, zero failures. Verified again here over 400 real records, still zero. They stay SENSORS rather than restatements. Each still builds its own span set and computes its own union; what is now shared is the selection and `union_ms`, and what is not is the bookkeeping around them, which is where every timing defect on this branch actually lived. On top of that each CROSS-CHECKS the stored `tool_union_ms` against what it independently computed, skipping when the field is absent — which is every record written before it existed. A disagreement is its own named breach in the live gate, and it exits non-zero independently of `--max-residual-pct`, because a finding that cannot fail the gate is prose. The corpus moves to `scripts/timing/corpus/` and is re-scrubbed to be model-valid. Two of its five records deliberately preserve defects the live code no longer has, and stale-by-design data under `tests/_fixtures/` invites the next reader to point a test at it and pin a fixed defect as expected behaviour; the README now says outright that no test may read the directory. The re-scrub restores what the original one had stripped below what the model requires — `user_input`, `agent_output` and each command's `timestamp`, as neutral placeholders, none of them a wall-clock magnitude. The script's table over the moved corpus is byte-identical to its pre-move output. Three fixture families had to become model-valid for the same reason, which is the visible cost of the sensors no longer accepting anything dict-shaped. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- .claude/harness-candidates.md | 30 +++ docs/agents/HARNESS_PARITY.md | 14 +- .../timing/corpus}/README.md | 15 +- .../timing/corpus}/antigravity.json | 40 ++- .../timing/corpus}/claude-code.json | 40 ++- .../timing/corpus}/codex.json | 28 +- .../timing/corpus}/opencode.json | 46 ++-- .../timing/corpus}/pi.json | 40 ++- scripts/timing/decompose_run.py | 130 ++++++---- tests/_fixtures/golden_streams/_scrub.py | 72 +++--- tests/test_agent_golden_master.py | 9 +- tests/test_timing_close_window.py | 239 ++++++++++++++---- 12 files changed, 500 insertions(+), 203 deletions(-) rename {tests/_fixtures/timing_runs => scripts/timing/corpus}/README.md (73%) rename {tests/_fixtures/timing_runs => scripts/timing/corpus}/antigravity.json (80%) rename {tests/_fixtures/timing_runs => scripts/timing/corpus}/claude-code.json (86%) rename {tests/_fixtures/timing_runs => scripts/timing/corpus}/codex.json (83%) rename {tests/_fixtures/timing_runs => scripts/timing/corpus}/opencode.json (82%) rename {tests/_fixtures/timing_runs => scripts/timing/corpus}/pi.json (81%) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 6a41b7c16..7069c7c33 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -827,3 +827,33 @@ re-derive from scratch. wrong instant. REVISIT IF: an inversion is observed on a live run after CE064, which would mean a basis is still mixed somewhere the rule cannot see (the plugin SPI, or a harness whose spans come from a CLI). + +- [ ] **`test_codex_golden[a_agent_message_only]` is FLAKY, ~5% — measured, and + pre-existing.** Forty consecutive runs on an unmodified tree (`-n 0`): 2 + failures, `"no assistant message reports a positive generation window with + bounds that span it"` with `(0.0, '...164797', '...164797')`. The replay + finishes faster than `datetime`'s 1 us resolution, so codex's rebased item + stamps can collapse to one instant and the window rounds to `0.0` — which + `assert_timing_captured`'s `expect_generation_window` arm then correctly + refuses. Surfaced (not caused) by the turn-timing consolidation, which runs + that file repeatedly. Not fixed here because the fix is in the codex fixture's + stamp rebasing (`_rebase_notifications`), which is its own change with its own + risk of ratifying whatever it then produces; the honest options are to give + the fixture's items a floor above the clock's resolution, or to give the + scenario `expect_generation_window=False` and say why. Caught in: the + turn-timing consolidation, Phase 6. + +- [ ] **CE058 misses a sixth form: ` if else `.** + Its five forms are a zero constructor keyword, `x or 0`, `x if x is not None + else 0.0`, an `if x is None: x = 0.0` assignment, and a `model_copy(update=)` + dict. Form 3 keys on an `is None` / `is not None` COMPARISON, so the shape the + single production writer of `tool_union_ms` actually uses — + `union_ms(tool_spans) if tool_spans else None`, a truthiness test on a list — + is invisible to the rule in either polarity. Nothing ships wrong today (that + line correctly writes `None`, and `test_a_turn_with_no_bounded_span_records_none_not_zero` + covers it), but an author flipping it to `else 0.0` would publish "measured, + and instant" with the rule silent. Candidate: a form that fires on an + `ast.IfExp` whose `orelse` is a numeric literal and whose assignment target — + or enclosing timing-constructor keyword — matches `_TIMING_NAME`. Deferred + because the target-name resolution is new machinery rather than a variant of + an existing form. Caught in: the turn-timing consolidation, Phase 5 review. diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index d6fb384e8..4b1cabd1d 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -65,7 +65,7 @@ sets that type); run it by hand for the others. **`generation_duration_ms` is model-generation time, not `completed_at − started_at`.** All five harnesses can have tool execution inside a generation window, and it is subtracted out of every one of them — **once, centrally**, by -`streaming/collector.py::subtract_tool_time`. No reducer does it itself; each +`timing.py::subtract_tool_time`. No reducer does it itself; each publishes the raw window (see the two sections below). Every harness has the problem: Antigravity reports a `Step` for the tool and only a later `usage_metadata` `Step` cuts the message; Codex's message window is seeded from @@ -98,7 +98,7 @@ publishing a measured `generation_duration_ms` to import the helper, and is now needs it. **Tool execution comes out of the windows ONCE, at the collector.** -`streaming/collector.py::subtract_tool_time` takes the union of the main-thread +`timing.py::subtract_tool_time` takes the union of the main-thread tool intervals, clipped to each window, out of the raw spans the reducers publish. Before, that happened five times in five places — four inside `close_window` as the reducer flushed, claude-code once at finalization — while @@ -205,7 +205,7 @@ emission's window runs concurrently with a tool already timing. On a task issuing five parallel writes, five reads and two concurrent `Bash` calls the overlap was 482 ms and 340 ms on two ~18-25 s turns, and the four-bucket residual came out at exactly `-481 ms` and `-339 ms`. (That run is pinned at -`tests/_fixtures/timing_runs/claude-code.json`, which still reconciles at +`scripts/timing/corpus/claude-code.json`, which still reconciles at -481 ms — it is a RECORD of the defect, not of current behaviour; see the README there.) @@ -241,7 +241,13 @@ generation window) and **tail** (last window → turn end) are booked as `EventCollector` seam by `coder_eval/timing.py::decompose_turn`. The tool term is the **union** of the command intervals, for the same reason the subtraction above is — Pi resolved a `Write` and a `Bash` overlapping by 18.4 ms in one -measured turn, and summing their durations books that overlap twice. The head +measured turn, and summing their durations books that overlap twice — and it is +the THIRD stored bucket, `TurnRecord.tool_union_ms`, written at the same seam +from the same span set the head and the tail are measured against. Generation is +deliberately not stored: it is a one-line sum over the message stream, and the +reconciliation entry exists so a consumer sums that stream rather than reading a +separate aggregate. The tool union is the opposite case — union arithmetic plus +a sub-agent filter — which is what a dict consumer cannot cheaply reproduce. The head and tail exclude tool execution by that same rule and that same helper, which is what keeps the four buckets disjoint: a tool is not confined to a generation window (Antigravity force-closes an orphan at finalization, inside diff --git a/tests/_fixtures/timing_runs/README.md b/scripts/timing/corpus/README.md similarity index 73% rename from tests/_fixtures/timing_runs/README.md rename to scripts/timing/corpus/README.md index 838b0c219..ca8edc43a 100644 --- a/tests/_fixtures/timing_runs/README.md +++ b/scripts/timing/corpus/README.md @@ -2,11 +2,20 @@ One scrubbed, representative `task.json` per harness, from a real `tasks/timing-parallel-tools` run. Prompts, outputs, tokens and cost are -stripped; only the wall-clock fields `scripts/timing/decompose_run.py` reads -survive. +stripped; the wall-clock fields `scripts/timing/decompose_run.py` reads survive, +plus the handful `TurnRecord` requires (`user_input`, `agent_output`, each +command's `timestamp`) as neutral placeholders — the script validates each turn +into the model so it can call production's own span selector, and the original +scrub had left these records below what that requires. + +**NO TEST MAY READ THIS DIRECTORY.** That is why it lives under `scripts/` and +not under `tests/_fixtures/`. Two of the five records deliberately preserve +defects the live code no longer has (see below); a green assertion over them +would pin a fixed defect as expected behaviour, and a reader who finds +stale-by-design data under `tests/` has every reason to point a test at it. ``` -uv run python scripts/timing/decompose_run.py tests/_fixtures/timing_runs/*.json --min-turn-ms 0 +uv run python scripts/timing/decompose_run.py scripts/timing/corpus/*.json --min-turn-ms 0 ``` ## What this is NOT diff --git a/tests/_fixtures/timing_runs/antigravity.json b/scripts/timing/corpus/antigravity.json similarity index 80% rename from tests/_fixtures/timing_runs/antigravity.json rename to scripts/timing/corpus/antigravity.json index a662f2f9a..e2ed7f7a9 100644 --- a/tests/_fixtures/timing_runs/antigravity.json +++ b/scripts/timing/corpus/antigravity.json @@ -38,7 +38,8 @@ "execution_started_at": "2026-09-11T21:25:55.122510", "tool_id": "b7747e76d7fb722e11ea267731099383:2", "result_status": "success", - "duration_ms": 4.08 + "duration_ms": 4.08, + "timestamp": "2026-09-11T21:25:55.122510" }, { "tool_name": "Edit", @@ -46,7 +47,8 @@ "execution_started_at": "2026-09-11T21:25:55.452684", "tool_id": "b7747e76d7fb722e11ea267731099383:3", "result_status": "success", - "duration_ms": 2.7039999999999997 + "duration_ms": 2.7039999999999997, + "timestamp": "2026-09-11T21:25:55.452684" }, { "tool_name": "Edit", @@ -54,7 +56,8 @@ "execution_started_at": "2026-09-11T21:25:55.796438", "tool_id": "b7747e76d7fb722e11ea267731099383:4", "result_status": "success", - "duration_ms": 2.5260000000000002 + "duration_ms": 2.5260000000000002, + "timestamp": "2026-09-11T21:25:55.796438" }, { "tool_name": "Edit", @@ -62,7 +65,8 @@ "execution_started_at": "2026-09-11T21:25:56.099180", "tool_id": "b7747e76d7fb722e11ea267731099383:5", "result_status": "success", - "duration_ms": 2.733 + "duration_ms": 2.733, + "timestamp": "2026-09-11T21:25:56.099180" }, { "tool_name": "Edit", @@ -70,7 +74,8 @@ "execution_started_at": "2026-09-11T21:25:56.133426", "tool_id": "b7747e76d7fb722e11ea267731099383:6", "result_status": "success", - "duration_ms": 7.332 + "duration_ms": 7.332, + "timestamp": "2026-09-11T21:25:56.133426" }, { "tool_name": "Read", @@ -78,7 +83,8 @@ "execution_started_at": "2026-09-11T21:25:59.323643", "tool_id": "b7747e76d7fb722e11ea267731099383:8", "result_status": "success", - "duration_ms": 4.558000000000001 + "duration_ms": 4.558000000000001, + "timestamp": "2026-09-11T21:25:59.323643" }, { "tool_name": "Read", @@ -86,7 +92,8 @@ "execution_started_at": "2026-09-11T21:25:59.549154", "tool_id": "b7747e76d7fb722e11ea267731099383:9", "result_status": "success", - "duration_ms": 1.613 + "duration_ms": 1.613, + "timestamp": "2026-09-11T21:25:59.549154" }, { "tool_name": "Read", @@ -94,7 +101,8 @@ "execution_started_at": "2026-09-11T21:25:59.800660", "tool_id": "b7747e76d7fb722e11ea267731099383:10", "result_status": "success", - "duration_ms": 1.1720000000000002 + "duration_ms": 1.1720000000000002, + "timestamp": "2026-09-11T21:25:59.800660" }, { "tool_name": "Read", @@ -102,7 +110,8 @@ "execution_started_at": "2026-09-11T21:26:00.086176", "tool_id": "b7747e76d7fb722e11ea267731099383:11", "result_status": "success", - "duration_ms": 0.832 + "duration_ms": 0.832, + "timestamp": "2026-09-11T21:26:00.086176" }, { "tool_name": "Read", @@ -110,7 +119,8 @@ "execution_started_at": "2026-09-11T21:26:00.394123", "tool_id": "b7747e76d7fb722e11ea267731099383:12", "result_status": "success", - "duration_ms": 1.738 + "duration_ms": 1.738, + "timestamp": "2026-09-11T21:26:00.394123" }, { "tool_name": "Bash", @@ -118,7 +128,8 @@ "execution_started_at": "2026-09-11T21:26:00.725396", "tool_id": "b7747e76d7fb722e11ea267731099383:13", "result_status": "success", - "duration_ms": 2043.1840000000002 + "duration_ms": 2043.1840000000002, + "timestamp": "2026-09-11T21:26:00.725396" }, { "tool_name": "Bash", @@ -126,9 +137,12 @@ "execution_started_at": "2026-09-11T21:26:00.773556", "tool_id": "b7747e76d7fb722e11ea267731099383:14", "result_status": "success", - "duration_ms": 2084.5950000000003 + "duration_ms": 2084.5950000000003, + "timestamp": "2026-09-11T21:26:00.773556" } - ] + ], + "user_input": "", + "agent_output": "" } ] } diff --git a/tests/_fixtures/timing_runs/claude-code.json b/scripts/timing/corpus/claude-code.json similarity index 86% rename from tests/_fixtures/timing_runs/claude-code.json rename to scripts/timing/corpus/claude-code.json index 1eff89cca..97c6b73f3 100644 --- a/tests/_fixtures/timing_runs/claude-code.json +++ b/scripts/timing/corpus/claude-code.json @@ -129,7 +129,8 @@ "execution_started_at": "2026-09-11T07:14:01.011427", "tool_id": "toolu_01BqWiy1hXCaepichHH9dnXC", "result_status": "success", - "duration_ms": 21.16704103536904 + "duration_ms": 21.16704103536904, + "timestamp": "2026-09-11T07:14:01.011427" }, { "tool_name": "Write", @@ -137,7 +138,8 @@ "execution_started_at": "2026-09-11T07:14:01.672806", "tool_id": "toolu_01YN7oVP3MEuUGhNgtoYXDhV", "result_status": "success", - "duration_ms": 3.7791249342262745 + "duration_ms": 3.7791249342262745, + "timestamp": "2026-09-11T07:14:01.672806" }, { "tool_name": "Write", @@ -145,7 +147,8 @@ "execution_started_at": "2026-09-11T07:14:02.386537", "tool_id": "toolu_01JNtoHGHMzD4QamxrePBR9T", "result_status": "success", - "duration_ms": 16.864625038579106 + "duration_ms": 16.864625038579106, + "timestamp": "2026-09-11T07:14:02.386537" }, { "tool_name": "Write", @@ -153,7 +156,8 @@ "execution_started_at": "2026-09-11T07:14:03.064935", "tool_id": "toolu_01DaWJKb1Hm6bxcpiKjVzeYd", "result_status": "success", - "duration_ms": 5.963291972875595 + "duration_ms": 5.963291972875595, + "timestamp": "2026-09-11T07:14:03.064935" }, { "tool_name": "Write", @@ -161,7 +165,8 @@ "execution_started_at": "2026-09-11T07:14:03.747296", "tool_id": "toolu_014MD2sCpV9RhaF9TjAQccFT", "result_status": "success", - "duration_ms": 20.77529113739729 + "duration_ms": 20.77529113739729, + "timestamp": "2026-09-11T07:14:03.747296" }, { "tool_name": "Read", @@ -169,7 +174,8 @@ "execution_started_at": "2026-09-11T07:14:04.328605", "tool_id": "toolu_01M2q1dfDu3Vsxte9dSv6AhA", "result_status": "success", - "duration_ms": 9.594874922186136 + "duration_ms": 9.594874922186136, + "timestamp": "2026-09-11T07:14:04.328605" }, { "tool_name": "Read", @@ -177,7 +183,8 @@ "execution_started_at": "2026-09-11T07:14:04.917406", "tool_id": "toolu_01FueR7TLLvWTEWXEbfGErzE", "result_status": "success", - "duration_ms": 3.019041148945689 + "duration_ms": 3.019041148945689, + "timestamp": "2026-09-11T07:14:04.917406" }, { "tool_name": "Read", @@ -185,7 +192,8 @@ "execution_started_at": "2026-09-11T07:14:05.486085", "tool_id": "toolu_017wr4qR5X22aegAGbtx9PVE", "result_status": "success", - "duration_ms": 7.906709099188447 + "duration_ms": 7.906709099188447, + "timestamp": "2026-09-11T07:14:05.486085" }, { "tool_name": "Read", @@ -193,7 +201,8 @@ "execution_started_at": "2026-09-11T07:14:06.071847", "tool_id": "toolu_01HDYj2UeY1Nagd1ecjbs81V", "result_status": "success", - "duration_ms": 6.425125058740377 + "duration_ms": 6.425125058740377, + "timestamp": "2026-09-11T07:14:06.071847" }, { "tool_name": "Read", @@ -201,7 +210,8 @@ "execution_started_at": "2026-09-11T07:14:06.671314", "tool_id": "toolu_01UbZp7DRorw4cUbo5PB2Bia", "result_status": "success", - "duration_ms": 6.9256669376045465 + "duration_ms": 6.9256669376045465, + "timestamp": "2026-09-11T07:14:06.671314" }, { "tool_name": "Bash", @@ -209,7 +219,8 @@ "execution_started_at": "2026-09-11T07:14:07.253341", "tool_id": "toolu_0194Py8X1atsEPAA5WAWVjyT", "result_status": "success", - "duration_ms": 2273.930750088766 + "duration_ms": 2273.930750088766, + "timestamp": "2026-09-11T07:14:07.253341" }, { "tool_name": "Bash", @@ -217,9 +228,12 @@ "execution_started_at": "2026-09-11T07:14:07.734993", "tool_id": "toolu_018Nbqg6YSdYk6fb9c4gWAGy", "result_status": "success", - "duration_ms": 1852.037207921967 + "duration_ms": 1852.037207921967, + "timestamp": "2026-09-11T07:14:07.734993" } - ] + ], + "user_input": "", + "agent_output": "" } ] } diff --git a/tests/_fixtures/timing_runs/codex.json b/scripts/timing/corpus/codex.json similarity index 83% rename from tests/_fixtures/timing_runs/codex.json rename to scripts/timing/corpus/codex.json index 0042aaf67..e9d6126de 100644 --- a/tests/_fixtures/timing_runs/codex.json +++ b/scripts/timing/corpus/codex.json @@ -45,7 +45,8 @@ "execution_started_at": "2026-09-11T07:14:25.035000", "tool_id": "call_MFDN1SquQKjnPMAC7cdID6es", "result_status": "success", - "duration_ms": 9.0 + "duration_ms": 9.0, + "timestamp": "2026-09-11T07:14:25.035000" }, { "tool_name": "Bash", @@ -53,7 +54,8 @@ "execution_started_at": "2026-09-11T07:14:31.452000", "tool_id": "call_XoyG0xCaUZkVh4gs4HscEQY6", "result_status": "success", - "duration_ms": 0.0 + "duration_ms": 0.0, + "timestamp": "2026-09-11T07:14:31.452000" }, { "tool_name": "Bash", @@ -61,7 +63,8 @@ "execution_started_at": "2026-09-11T07:14:31.474000", "tool_id": "call_3xwdnJ9958Qws766mtNMUmum", "result_status": "success", - "duration_ms": 0.0 + "duration_ms": 0.0, + "timestamp": "2026-09-11T07:14:31.474000" }, { "tool_name": "Bash", @@ -69,7 +72,8 @@ "execution_started_at": "2026-09-11T07:14:31.480000", "tool_id": "call_kQU4wxYIqLLd7MxdhVBXt1fz", "result_status": "success", - "duration_ms": 0.0 + "duration_ms": 0.0, + "timestamp": "2026-09-11T07:14:31.480000" }, { "tool_name": "Bash", @@ -77,7 +81,8 @@ "execution_started_at": "2026-09-11T07:14:31.480000", "tool_id": "call_EHI6niBRHKpW9GSxYgbSGPQn", "result_status": "success", - "duration_ms": 2072.0 + "duration_ms": 2072.0, + "timestamp": "2026-09-11T07:14:31.480000" }, { "tool_name": "Bash", @@ -85,7 +90,8 @@ "execution_started_at": "2026-09-11T07:14:31.483000", "tool_id": "call_5UNrspGHKnOkwpcR5PedDt5B", "result_status": "success", - "duration_ms": 0.0 + "duration_ms": 0.0, + "timestamp": "2026-09-11T07:14:31.483000" }, { "tool_name": "Bash", @@ -93,7 +99,8 @@ "execution_started_at": "2026-09-11T07:14:31.487000", "tool_id": "call_xOqzas8ctiA0ri1wIsfiJobK", "result_status": "success", - "duration_ms": 0.0 + "duration_ms": 0.0, + "timestamp": "2026-09-11T07:14:31.487000" }, { "tool_name": "Bash", @@ -101,9 +108,12 @@ "execution_started_at": null, "tool_id": "call_zAeVW6TDp7vxHy3HilRTAa3i", "result_status": null, - "duration_ms": null + "duration_ms": null, + "timestamp": "2026-09-11T07:14:22.104000" } - ] + ], + "user_input": "", + "agent_output": "" } ] } diff --git a/tests/_fixtures/timing_runs/opencode.json b/scripts/timing/corpus/opencode.json similarity index 82% rename from tests/_fixtures/timing_runs/opencode.json rename to scripts/timing/corpus/opencode.json index 749e62e37..ec76a0c6e 100644 --- a/tests/_fixtures/timing_runs/opencode.json +++ b/scripts/timing/corpus/opencode.json @@ -59,7 +59,8 @@ "execution_started_at": "2026-09-11T20:13:55.115000", "tool_id": "toolu_01WfZGLMB5HLRS9E15jLYJq8", "result_status": "success", - "duration_ms": 4.0 + "duration_ms": 4.0, + "timestamp": "2026-09-11T20:13:55.115000" }, { "tool_name": "Write", @@ -67,7 +68,8 @@ "execution_started_at": "2026-09-11T20:13:56.968000", "tool_id": "toolu_01WLeJLQCihZjNtCgwCi1xpL", "result_status": "success", - "duration_ms": 11.0 + "duration_ms": 11.0, + "timestamp": "2026-09-11T20:13:56.968000" }, { "tool_name": "Write", @@ -75,7 +77,8 @@ "execution_started_at": "2026-09-11T20:13:57.678000", "tool_id": "toolu_019tjDMgudrqyntWUAh8Aupy", "result_status": "success", - "duration_ms": 6.0 + "duration_ms": 6.0, + "timestamp": "2026-09-11T20:13:57.678000" }, { "tool_name": "Write", @@ -83,7 +86,8 @@ "execution_started_at": "2026-09-11T20:13:58.376000", "tool_id": "toolu_013J2dP6wxbm4q1acY5PXr6s", "result_status": "success", - "duration_ms": 9.0 + "duration_ms": 9.0, + "timestamp": "2026-09-11T20:13:58.376000" }, { "tool_name": "Write", @@ -91,7 +95,8 @@ "execution_started_at": "2026-09-11T20:13:59.075000", "tool_id": "toolu_01DZAPiHZrcfEiDeUDVdur6H", "result_status": "success", - "duration_ms": 5.0 + "duration_ms": 5.0, + "timestamp": "2026-09-11T20:13:59.075000" }, { "tool_name": "Write", @@ -99,7 +104,8 @@ "execution_started_at": "2026-09-11T20:13:59.751000", "tool_id": "toolu_01S2Kd2KUueP11tuBZ3hX53j", "result_status": "success", - "duration_ms": 9.0 + "duration_ms": 9.0, + "timestamp": "2026-09-11T20:13:59.751000" }, { "tool_name": "Read", @@ -107,7 +113,8 @@ "execution_started_at": "2026-09-11T20:14:01.506000", "tool_id": "toolu_01RbNdCyWmL3XcWUZ1BZBMVs", "result_status": "success", - "duration_ms": 12.0 + "duration_ms": 12.0, + "timestamp": "2026-09-11T20:14:01.506000" }, { "tool_name": "Read", @@ -115,7 +122,8 @@ "execution_started_at": "2026-09-11T20:14:02.079000", "tool_id": "toolu_01SDGHLgG9HR9aCieXd81Wz3", "result_status": "success", - "duration_ms": 10.0 + "duration_ms": 10.0, + "timestamp": "2026-09-11T20:14:02.079000" }, { "tool_name": "Read", @@ -123,7 +131,8 @@ "execution_started_at": "2026-09-11T20:14:02.654000", "tool_id": "toolu_01XER7fGNDxwMaaucfPhKS7C", "result_status": "success", - "duration_ms": 6.0 + "duration_ms": 6.0, + "timestamp": "2026-09-11T20:14:02.654000" }, { "tool_name": "Read", @@ -131,7 +140,8 @@ "execution_started_at": "2026-09-11T20:14:03.232000", "tool_id": "toolu_01TFRCQq7bYkiBwHvSpc3Cxr", "result_status": "success", - "duration_ms": 7.0 + "duration_ms": 7.0, + "timestamp": "2026-09-11T20:14:03.232000" }, { "tool_name": "Read", @@ -139,7 +149,8 @@ "execution_started_at": "2026-09-11T20:14:03.965000", "tool_id": "toolu_01MHcPMKPGMihm8MkmhAWBtu", "result_status": "success", - "duration_ms": 9.0 + "duration_ms": 9.0, + "timestamp": "2026-09-11T20:14:03.965000" }, { "tool_name": "Bash", @@ -147,7 +158,8 @@ "execution_started_at": "2026-09-11T20:14:06.125000", "tool_id": "toolu_014B6sD12tnYcpCAHbC7YJbS", "result_status": "success", - "duration_ms": 56.0 + "duration_ms": 56.0, + "timestamp": "2026-09-11T20:14:06.125000" }, { "tool_name": "Bash", @@ -155,7 +167,8 @@ "execution_started_at": "2026-09-11T20:14:05.681000", "tool_id": "toolu_01KCeAUpoHtjG2ssT85e4o9c", "result_status": "success", - "duration_ms": 2064.0 + "duration_ms": 2064.0, + "timestamp": "2026-09-11T20:14:05.681000" }, { "tool_name": "TodoWrite", @@ -163,9 +176,12 @@ "execution_started_at": "2026-09-11T20:14:10.247000", "tool_id": "toolu_01GMeN29huU3tN1BHpusugFh", "result_status": "success", - "duration_ms": 2.0 + "duration_ms": 2.0, + "timestamp": "2026-09-11T20:14:10.247000" } - ] + ], + "user_input": "", + "agent_output": "" } ] } diff --git a/tests/_fixtures/timing_runs/pi.json b/scripts/timing/corpus/pi.json similarity index 81% rename from tests/_fixtures/timing_runs/pi.json rename to scripts/timing/corpus/pi.json index e6a2a5924..cb0143c8f 100644 --- a/tests/_fixtures/timing_runs/pi.json +++ b/scripts/timing/corpus/pi.json @@ -45,7 +45,8 @@ "execution_started_at": "2026-09-11T07:42:08.389754", "tool_id": "toolu_016h4AfvKbh7JeGEBNGX9XXf", "result_status": "success", - "duration_ms": 16.657 + "duration_ms": 16.657, + "timestamp": "2026-09-11T07:42:08.389754" }, { "tool_name": "Write", @@ -53,7 +54,8 @@ "execution_started_at": "2026-09-11T07:42:08.398691", "tool_id": "toolu_01PBQpT5a3TeTqKhkyNP8x2t", "result_status": "success", - "duration_ms": 7.073 + "duration_ms": 7.073, + "timestamp": "2026-09-11T07:42:08.398691" }, { "tool_name": "Write", @@ -61,7 +63,8 @@ "execution_started_at": "2026-09-11T07:42:08.399067", "tool_id": "toolu_01G3VRUbZC8WUetG97MvmRML", "result_status": "success", - "duration_ms": 7.756 + "duration_ms": 7.756, + "timestamp": "2026-09-11T07:42:08.399067" }, { "tool_name": "Write", @@ -69,7 +72,8 @@ "execution_started_at": "2026-09-11T07:42:08.399201", "tool_id": "toolu_01C9J9jQya5aZZQvvARyi2Cn", "result_status": "success", - "duration_ms": 7.3790000000000004 + "duration_ms": 7.3790000000000004, + "timestamp": "2026-09-11T07:42:08.399201" }, { "tool_name": "Write", @@ -77,7 +81,8 @@ "execution_started_at": "2026-09-11T07:42:08.399309", "tool_id": "toolu_01DitFddEYKRKVBhSzeeGqZB", "result_status": "success", - "duration_ms": 7.414 + "duration_ms": 7.414, + "timestamp": "2026-09-11T07:42:08.399309" }, { "tool_name": "Read", @@ -85,7 +90,8 @@ "execution_started_at": "2026-09-11T07:42:10.813486", "tool_id": "toolu_0171JeX8QfzobMv3YZxqafuM", "result_status": "success", - "duration_ms": 11.376000000000001 + "duration_ms": 11.376000000000001, + "timestamp": "2026-09-11T07:42:10.813486" }, { "tool_name": "Read", @@ -93,7 +99,8 @@ "execution_started_at": "2026-09-11T07:42:10.814175", "tool_id": "toolu_01PoWyWCXneLVWEJoBbRTiYQ", "result_status": "success", - "duration_ms": 10.36 + "duration_ms": 10.36, + "timestamp": "2026-09-11T07:42:10.814175" }, { "tool_name": "Read", @@ -101,7 +108,8 @@ "execution_started_at": "2026-09-11T07:42:10.814507", "tool_id": "toolu_01NVqJypM1gJY6To9KfihgJL", "result_status": "success", - "duration_ms": 8.299000000000001 + "duration_ms": 8.299000000000001, + "timestamp": "2026-09-11T07:42:10.814507" }, { "tool_name": "Read", @@ -109,7 +117,8 @@ "execution_started_at": "2026-09-11T07:42:10.815282", "tool_id": "toolu_01MgTWDsatAnY18osCK4qTTr", "result_status": "success", - "duration_ms": 8.465 + "duration_ms": 8.465, + "timestamp": "2026-09-11T07:42:10.815282" }, { "tool_name": "Read", @@ -117,7 +126,8 @@ "execution_started_at": "2026-09-11T07:42:10.815562", "tool_id": "toolu_01ACtUYSTWuGSkr34qH2Lvqd", "result_status": "success", - "duration_ms": 9.418 + "duration_ms": 9.418, + "timestamp": "2026-09-11T07:42:10.815562" }, { "tool_name": "Bash", @@ -125,7 +135,8 @@ "execution_started_at": "2026-09-11T07:42:12.548936", "tool_id": "toolu_01SGnPjtpNCtQnCrxwKLvqQ4", "result_status": "success", - "duration_ms": 2049.098 + "duration_ms": 2049.098, + "timestamp": "2026-09-11T07:42:12.548936" }, { "tool_name": "Bash", @@ -133,9 +144,12 @@ "execution_started_at": "2026-09-11T07:42:12.549868", "tool_id": "toolu_016kzccEcHbwwo1P9USYsXKM", "result_status": "success", - "duration_ms": 76.719 + "duration_ms": 76.719, + "timestamp": "2026-09-11T07:42:12.549868" } - ] + ], + "user_input": "", + "agent_output": "" } ] } diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index 6c3bd68c7..bc2b11e94 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -20,8 +20,8 @@ Not wired into `make`: it needs live runs, not fixtures. NOTE `scripts/` is outside the Makefile's LINT_PATHS, so this file is neither formatted nor -ruff-checked — keep it small and dependency-free (stdlib plus the one shared -`union_ms` import, so the union rule has a single definition). +ruff-checked — keep it small and dependency-free (stdlib plus the shared +timing helpers and `TurnRecord`, so the union rule has a single definition). """ from __future__ import annotations @@ -34,66 +34,61 @@ from datetime import datetime from pathlib import Path -from coder_eval.timing import union_ms +from pydantic import ValidationError +from coder_eval.models import TurnRecord +from coder_eval.timing import main_thread_tool_spans, union_ms -def _parse(stamp: object) -> datetime | None: - if not isinstance(stamp, str): - return None - try: - return datetime.fromisoformat(stamp) - except ValueError: - return None +# A stored `tool_union_ms` and the union computed here should be the same +# number; JSON round-tripping is the only slack, so the tolerance is absolute +# and tiny. +_UNION_TOLERANCE_MS = 1e-6 -def _sub_agent_tool_ids(turn: dict) -> set: - """Tool ids owned by a SUB-AGENT generation, which the main thread excludes. - Derived the only way it can be: a child generation carries - `parent_tool_use_id`, and its `tool_use_ids` are the calls it made. +def _tool_ms(turn: dict) -> float: + """Wall ms this turn's MAIN-THREAD tools occupied — the UNION, not the sum. - This MUST match `EventCollector._main_thread_tool_spans`, which applies the - same filter when it computes the head, the tail and the generation - subtraction. The two used to disagree — the collector passed every command - while filtering its generations — and they agreed only by luck, because a - child nests inside the parent Agent call whose interval the union already - covers. Codex's recovered child tools carry the CHILD's clock, so the - nesting is not guaranteed, and a gate computing a different tool total than - the harness reports a residual that is an artifact of the disagreement - rather than a bucket error. This is the only two-sided live sensor for the - identity, so that is the worst place for the two to drift. + Validates the raw dict into a `TurnRecord` and calls the SAME typed + selector the collector uses, rather than reimplementing the selection rule + (which commands count, the sub-agent exclusion, the stamp parse, the + `end >= start` filter) over dicts. Three copies of that rule existed and + they agreed only because someone kept checking; Pi resolved a `Write` and a + `Bash` overlapping by 18.4 ms in one measured turn, and a selector that + disagreed with the collector's would report a residual that is an artifact + of the disagreement rather than a bucket error. This is the only two-sided + live sensor for the identity, so that is the worst place for a copy. + + What is shared is the SELECTION and `union_ms`. What is NOT shared is the + bookkeeping around them — this still builds its own span set and computes + its own union, so it stays a sensor rather than a restatement of the + producer's answer. Do not "simplify" it into reading `tool_union_ms`; the + cross-check below is how that field is verified, not how this is computed. """ - ids = set() - for message in turn.get("messages") or []: - if message.get("role") == "assistant" and message.get("parent_tool_use_id") is not None: - ids.update(message.get("tool_use_ids") or []) - return ids + record = TurnRecord.model_validate(turn) + return union_ms(main_thread_tool_spans(record.messages, record.commands)) -def _tool_ms(turn: dict) -> float: - """Wall ms this turn's MAIN-THREAD tools occupied — the UNION, not the sum. +def _union_breach(turn: dict) -> str | None: + """The stored `tool_union_ms` disagrees with what we just computed, if so. - The same rule `coder_eval.timing.union_ms` applies when the collector - subtracts tool time out of a generation window, and it has to be the same - rule here or the identity does not close: Pi resolved a `Write` and a `Bash` - that overlapped by 18.4 ms in one measured turn, and summing their durations - booked that overlap twice, which is precisely the 18.3 ms residual that - found this. A command with no recorded bounds cannot be placed on the - timeline at all, so it contributes nothing rather than being summed in - blind — see docs/agents/HARNESS_PARITY.md's Delegate divergence. Sub-agent - tools are excluded for the same reason their generations are; see - `_sub_agent_tool_ids`. + Skips when the field is absent: `decompose_run.py` reads corpora recorded + before `TurnRecord` carried it, so that is the common case here rather than + an exotic one. A DISAGREEMENT is its own named breach, distinct from a + residual breach — a residual says the buckets do not tile the turn, this + says the producer and an independent recomputation of one bucket do not + agree about its value, which is a different fault with a different fix. """ - excluded = _sub_agent_tool_ids(turn) - spans = [] - for command in turn.get("commands") or []: - if command.get("tool_id") in excluded: - continue - start = _parse(command.get("execution_started_at")) - end = _parse(command.get("execution_completed_at")) - if start is not None and end is not None and end >= start: - spans.append((start, end)) - return union_ms(spans) + stored = turn.get("tool_union_ms") + if not isinstance(stored, (int, float)): + return None + computed = _tool_ms(turn) + if abs(stored - computed) <= _UNION_TOLERANCE_MS: + return None + return ( + f"tool_union_ms={stored:.6f}ms but this turn's main-thread command spans union to " + f"{computed:.6f}ms (off by {stored - computed:+.6f}ms)" + ) def _turn_buckets(turn: dict) -> tuple[float, float, float, float, float] | None: @@ -183,6 +178,12 @@ def main(argv: list[str]) -> int: # reason as the two above: an exclusion nobody can see understates how much # of the corpus the gate actually looked at. skipped_untimed = 0 + # A turn whose STORED tool bucket disagrees with the union recomputed here. + # Its own list, not folded into the residual breaches: a residual says the + # buckets do not tile the turn; this says the producer and an independent + # recomputation of one bucket disagree about its value. + union_breaches: list[tuple[str, Path, int, str]] = [] + invalid: list[tuple[Path, int, int]] = [] for path in args.task_json: try: record = json.loads(path.read_text(encoding="utf-8")) @@ -210,10 +211,22 @@ def main(argv: list[str]) -> int: skipped_crashed += int(crashed) skipped_no_window += int(no_window) continue - buckets = _turn_buckets(turn) + try: + buckets = _turn_buckets(turn) + except ValidationError as exc: + # A real record failing TurnRecord validation is a FINDING, not + # a nuisance: measured across the whole run history on disk, + # 2466 of 2466 turns validated. Name it and move on rather than + # falling back to dict access, which is the second selection + # path this script just removed. + invalid.append((path, index, exc.error_count())) + continue if buckets is None: skipped_untimed += 1 continue + breach = _union_breach(turn) + if breach is not None: + union_breaches.append((harness, path, index, breach)) by_harness[harness].append((path, index, buckets)) if not by_harness: @@ -287,12 +300,25 @@ def main(argv: list[str]) -> int: f"{skipped_short} short (< {args.min_turn_ms:.0f}ms)" ) + if invalid: + print(f"\n{len(invalid)} turn(s) failed TurnRecord validation:", file=sys.stderr) + for path, index, count in invalid: + print(f" {path} turn {index}: {count} error(s)", file=sys.stderr) + if union_breaches: + print(f"\nSTORED TOOL UNION disagrees on {len(union_breaches)} turn(s):", file=sys.stderr) + for harness, path, index, detail in union_breaches: + print(f" {harness:<14} {path} turn {index}: {detail}", file=sys.stderr) + if not gateable_total: # A gate that passes because it measured nothing is the exact failure # this script exists to remove, so it only passes when none was asked for. print("no gateable turns", file=sys.stderr) return 1 if args.max_residual_pct is not None else 0 + if union_breaches or invalid: + # Independent of --max-residual-pct: neither is a residual question, and + # a disagreement about a stored bucket is exactly what a gate is for. + return 1 if args.max_residual_pct is None: return 0 if not breaches: diff --git a/tests/_fixtures/golden_streams/_scrub.py b/tests/_fixtures/golden_streams/_scrub.py index b842cfa5d..0c19a806f 100644 --- a/tests/_fixtures/golden_streams/_scrub.py +++ b/tests/_fixtures/golden_streams/_scrub.py @@ -2,10 +2,10 @@ from __future__ import annotations -from datetime import datetime from typing import Any -from coder_eval.timing import union_ms +from coder_eval.models import TurnRecord +from coder_eval.timing import main_thread_tool_spans, union_ms SCRUB_PLACEHOLDER = "" @@ -120,46 +120,30 @@ def assert_reconciliation(record: dict[str, Any]) -> None: _IDENTITY_SHARE = 0.20 -def _sub_agent_tool_ids(record: dict[str, Any]) -> set[str]: - """Tool ids owned by a SUB-AGENT generation. - - Must match `EventCollector._main_thread_tool_spans` and - `scripts/timing/decompose_run.py::_sub_agent_tool_ids`: all three recompute - the tool union for the same identity, so a filter applied by one and not - the others reports a residual that is an artifact of the disagreement. - """ - ids: set[str] = set() - for message in record.get("messages") or []: - if message.get("role") == "assistant" and message.get("parent_tool_use_id") is not None: - ids.update(message.get("tool_use_ids") or []) - return ids - - def _tool_union_ms(record: dict[str, Any]) -> float: """Wall ms this turn's MAIN-THREAD tools occupied — the union, never the sum. - Sub-agent tools are excluded for the same reason their generations are: the - spawning Agent call's own interval already spans the child's whole run. + Validates the raw dump into a ``TurnRecord`` and calls the SAME typed + selector the collector uses, rather than reimplementing the selection rule + (the sub-agent-id derivation, the stamp parse, the ``end >= start`` filter) + over dicts. Three copies of that rule existed and agreed only because + someone kept checking; the collector's own version once passed every command + while filtering only its generations, and the two agreed by luck. + + What is shared with production is the SELECTION and ``union_ms``. What is + NOT shared is the bookkeeping around them — this still builds its own span + set and computes its own union, which is where every timing defect on this + branch actually lived (see CE063's docstring). Do not "simplify" it into + reading ``tool_union_ms``: that would make the sensor a restatement of the + producer's answer, and the cross-check below is what verifies that field. """ - excluded = _sub_agent_tool_ids(record) - spans: list[tuple[datetime, datetime]] = [] - for command in record.get("commands") or []: - if command.get("tool_id") in excluded: - continue - start = _parse_stamp(command.get("execution_started_at")) - end = _parse_stamp(command.get("execution_completed_at")) - if start is not None and end is not None and end >= start: - spans.append((start, end)) - return union_ms(spans) + turn = TurnRecord.model_validate(record) + return union_ms(main_thread_tool_spans(turn.messages, turn.commands)) -def _parse_stamp(value: Any) -> datetime | None: - if not isinstance(value, str): - return None - try: - return datetime.fromisoformat(value) - except ValueError: - return None +# The stored bucket and the union recomputed here should be the same number; +# a JSON round-trip is the only slack. +_UNION_TOLERANCE_MS = 1e-6 def assert_timing_captured( @@ -286,6 +270,22 @@ def assert_timing_captured( if m.get("role") == "assistant" and m.get("parent_tool_use_id") is None ) tool_ms = _tool_union_ms(record) + # CROSS-CHECK, before the identity assertion: the producer's stored + # bucket must agree with the one just computed independently. This is + # what keeps the sensor a sensor — it verifies the producer's + # BOOKKEEPING (which spans reached the union) rather than reading the + # producer's answer. Skipped when the field is absent, which is a record + # written before it existed rather than a disagreement. + stored_union = record.get("tool_union_ms") + if isinstance(stored_union, (int, float)): + assert abs(stored_union - tool_ms) <= _UNION_TOLERANCE_MS, ( + f"tool_union_ms is {stored_union!r}, but this turn's main-thread command spans " + f"union to {tool_ms:.6f} ms (off by {stored_union - tool_ms:+.6f} ms). The " + "collector writes that field from the same span set it measures the head and the " + "tail against, so a disagreement means a span reached one and not the other — " + "most likely a sub-agent command counted on one side, or a command whose bounds " + "moved after the field was written." + ) bucket_sum = ( generation_ms + tool_ms diff --git a/tests/test_agent_golden_master.py b/tests/test_agent_golden_master.py index c250bb0e6..216bbe80f 100644 --- a/tests/test_agent_golden_master.py +++ b/tests/test_agent_golden_master.py @@ -363,7 +363,14 @@ def _record( identity is trivially satisfied, so these cases constrain only what each is about; the identity has its own cases below. """ + # MODEL-VALID, not merely shaped like a record. `assert_timing_captured` + # validates the dump into a `TurnRecord` so it can call production's own + # span selector instead of re-deriving one, and a fixture missing the + # required fields would fail there rather than on the thing it is about. return { + "iteration": 1, + "user_input": "", + "agent_output": "", "duration_seconds": duration_seconds, "messages": [ { @@ -374,7 +381,7 @@ def _record( } for w in windows ], - "commands": list(commands), + "commands": [{"tool_name": "Bash", "timestamp": "2026-01-01T00:00:00", **command} for command in commands], "harness_startup_ms": overhead[0], "harness_teardown_ms": overhead[1], } diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py index 31dc5dc2d..0b3d56fb8 100644 --- a/tests/test_timing_close_window.py +++ b/tests/test_timing_close_window.py @@ -12,7 +12,7 @@ import pytest -from coder_eval.timing import busy_ms, close_window, decompose_turn, union_ms +from coder_eval.timing import busy_ms, close_window, decompose_turn, main_thread_tool_spans, union_ms MARK = datetime(2026, 9, 11, 12, 0, 0) @@ -36,7 +36,7 @@ class TestCloseWindow: """The RAW window: where it opens, where it ends, and the clamp. The tool subtraction these cases used to cover moved to - `streaming/collector.py::subtract_tool_time`, where it happens once for all + `timing.py::subtract_tool_time`, where it happens once for all five harnesses instead of five times in five reducers — see `tests/test_event_collector.py::TestSubtractToolTime`, which carries the union, grouping, clamping and non-mutation cases. What is left here is the @@ -213,9 +213,9 @@ def test_the_two_recorded_command_readers_agree(self): They read the SAME `task.json` shape — the golden sensor from a dumped record, the gate from the file on disk — and a divergence would let one - pass while the other failed on identical bytes. They keep their own - stamp parsing (the inputs differ in how they are reached); the union - tail is what this pins. + pass while the other failed on identical bytes. Both now validate into a + `TurnRecord` and call the one typed selector, so what this pins is that + neither has quietly grown a second path back. """ from tests._fixtures.golden_streams._scrub import _tool_union_ms @@ -223,17 +223,17 @@ def test_the_two_recorded_command_readers_agree(self): # outside the Makefile's LINT_PATHS), so there is no import to make. _tool_ms = _load_decompose_run()._tool_ms - turn = { - "commands": [ - {"execution_started_at": _at(100).isoformat(), "execution_completed_at": _at(600).isoformat()}, - {"execution_started_at": _at(200).isoformat(), "execution_completed_at": _at(700).isoformat()}, + turn = _turn( + commands=[ + _command("a", _at(100), _at(600)), + _command("b", _at(200), _at(700)), # Never timed: contributes nothing on either side. - {"execution_started_at": None, "execution_completed_at": None}, - # Inverted bounds: both readers drop these while BUILDING their - # span list, which is why `union_ms` does not filter them. - {"execution_started_at": _at(900).isoformat(), "execution_completed_at": _at(800).isoformat()}, + _command("c", None, None), + # Inverted bounds: dropped while BUILDING the span list, which + # is why `union_ms` itself does not filter them. + _command("d", _at(900), _at(800)), ] - } + ) assert _tool_union_ms(turn) == pytest.approx(600.0) assert _tool_ms(turn) == _tool_union_ms(turn) @@ -287,25 +287,55 @@ def test_a_fresh_clock_anchors_on_its_own_pair(self): assert second._wall0 >= first._wall0 +def _command(tool_id: str, started, completed) -> dict: + """A recorded command, MODEL-VALID: both sensors validate before selecting.""" + return { + "tool_id": tool_id, + "tool_name": "Bash", + "timestamp": (started or _at(0)).isoformat(), + "execution_started_at": started.isoformat() if started is not None else None, + "execution_completed_at": completed.isoformat() if completed is not None else None, + } + + +def _turn(*, commands: list[dict], messages: list[dict] | None = None, duration_seconds: float = 3.0) -> dict: + """A `task.json` turn dict that validates as a `TurnRecord`. + + The two sensors no longer parse stamps out of a raw dict; they validate and + call the typed selector, so a fixture below the model's required fields + would fail in validation rather than on the thing the test is about. + """ + return { + "iteration": 1, + "user_input": "", + "agent_output": "", + "duration_seconds": duration_seconds, + "commands": commands, + "messages": messages or [], + } + + class TestTheThreeToolUnionsAgree: - """Three implementations recompute the turn's tool union. They must agree. + """Three readers answer "how long did this turn's tools run". They must agree. - * `EventCollector._main_thread_tool_spans` — what the harness subtracts - from the generation windows and measures the head and tail against. + * `timing.main_thread_tool_spans` + `union_ms` — the TYPED selector the + collector subtracts from its generation windows and measures the head and + tail against, and which now writes `TurnRecord.tool_union_ms`. * `tests/_fixtures/golden_streams/_scrub.py::_tool_union_ms` — the golden - corpus's identity check. + corpus's identity check, which validates the dump and calls that selector. * `scripts/timing/decompose_run.py::_tool_ms` — the LIVE two-sided residual - gate, which `.github/workflows/pr-checks.yml` runs against a real run. - - They agreed by luck once and it cost a defect: the collector filtered its - GENERATIONS to the main thread and then passed EVERY command as a tool - span. A child nests inside the parent Agent call, whose interval the union - already covers, so nothing failed — but Codex's recovered child tools carry - the CHILD's clock, so the nesting is not guaranteed. When the collector - started filtering, the other two did not, and a gate computing a different - tool total than the harness reports a residual that is an artifact of the - disagreement rather than a bucket error. That is the worst possible place - for a divergence, because this is the only live sensor for the identity. + gate, which `.github/workflows/pr-checks.yml` runs against a real run, and + which does the same. + + The three used to be three COPIES of the selection rule, and they agreed by + luck once at a real cost: the collector filtered its GENERATIONS to the main + thread and then passed EVERY command as a tool span. A child nests inside + the parent Agent call, whose interval the union already covers, so nothing + failed — but Codex's recovered child tools carry the CHILD's clock, so the + nesting is not guaranteed. Now there is ONE selector and two callers of it, + and what remains worth pinning is that neither sensor has grown a second + path back, and that each still computes its OWN union rather than reading + the producer's stored answer. """ @staticmethod @@ -315,25 +345,29 @@ def _record() -> dict: Inside, the three agree whatever they filter, so the fixture has to put the child's tool where the parent's interval does not cover it. """ - return { - "duration_seconds": 3.0, - "commands": [ + return _turn( + duration_seconds=3.0, + commands=[ + _command("agent-call", _at(1000), _at(1500)), + _command("child-tool", _at(2000), _at(2400)), + ], + messages=[ { - "tool_id": "agent-call", - "execution_started_at": _at(1000).isoformat(), - "execution_completed_at": _at(1500).isoformat(), + "role": "assistant", + "started_at": _at(0).isoformat(), + "completed_at": _at(1000).isoformat(), + "parent_tool_use_id": None, + "tool_use_ids": ["agent-call"], }, { - "tool_id": "child-tool", - "execution_started_at": _at(2000).isoformat(), - "execution_completed_at": _at(2400).isoformat(), + "role": "assistant", + "started_at": _at(2000).isoformat(), + "completed_at": _at(2400).isoformat(), + "parent_tool_use_id": "agent-call", + "tool_use_ids": ["child-tool"], }, ], - "messages": [ - {"role": "assistant", "parent_tool_use_id": None, "tool_use_ids": ["agent-call"]}, - {"role": "assistant", "parent_tool_use_id": "agent-call", "tool_use_ids": ["child-tool"]}, - ], - } + ) def test_the_two_recomputing_readers_exclude_the_sub_agent_tool(self): from tests._fixtures.golden_streams._scrub import _tool_union_ms @@ -378,5 +412,122 @@ def test_the_collector_excludes_it_too(self): tool_use_ids=["child-tool"], ), ] - spans = collector._main_thread_tool_spans(messages) + spans = main_thread_tool_spans(messages, collector._commands.values()) assert union_ms(spans) == pytest.approx(500.0), "the same 500 ms the other two report" + + +class TestTheSensorsCrossCheckTheStoredUnion: + """Each sensor verifies the producer's BOOKKEEPING, not its answer. + + `TurnRecord.tool_union_ms` is written by the collector from the span set it + measures all four buckets against. A sensor that simply READ it would stop + being a sensor — it would restate the implementation. So each computes its + own union from the commands and asserts the stored value agrees, which + catches exactly the class of defect this branch kept producing: a span that + reached one consumer and not the other. + """ + + @staticmethod + def _scrub_check(turn: dict) -> None: + from tests._fixtures.golden_streams._scrub import assert_timing_captured + + assert_timing_captured(turn, expect_generation_window=False, check_identity=True) + + @staticmethod + def _generating_turn(*, stored: float | None, tool_ms: float = 500.0) -> dict: + turn = _turn( + duration_seconds=10.0, + commands=[_command("t1", _at(1000), _at(1000 + tool_ms))], + messages=[ + { + "role": "assistant", + "started_at": _at(0).isoformat(), + "completed_at": _at(1000).isoformat(), + "generation_duration_ms": 1000.0, + } + ], + ) + turn["harness_startup_ms"] = 0.0 + turn["harness_teardown_ms"] = 5.0 + if stored is not None: + turn["tool_union_ms"] = stored + return turn + + def test_the_golden_sensor_fails_when_the_stored_value_disagrees(self): + with pytest.raises(AssertionError, match="tool_union_ms is"): + self._scrub_check(self._generating_turn(stored=999.0)) + + def test_the_golden_sensor_passes_when_it_agrees(self): + self._scrub_check(self._generating_turn(stored=500.0)) + + def test_the_golden_sensor_skips_a_record_without_the_field(self): + """The legacy case, and the common one for records already on disk.""" + self._scrub_check(self._generating_turn(stored=None)) + + def test_the_live_gate_reports_a_disagreement(self): + breach = _load_decompose_run()._union_breach(self._generating_turn(stored=999.0)) + assert breach is not None + assert "999.000000" in breach and "500.000000" in breach + + def test_the_live_gate_is_silent_when_they_agree_or_the_field_is_absent(self): + union_breach = _load_decompose_run()._union_breach + assert union_breach(self._generating_turn(stored=500.0)) is None + assert union_breach(self._generating_turn(stored=None)) is None + + def test_a_sub_agent_tool_outside_the_parent_call_is_excluded_by_all_three(self): + """One record carrying every shape the selection rule has to decide. + + A sub-agent tool NESTED inside its parent call is covered by the + parent's own interval whatever anyone filters, so the fixture puts one + outside it — which is the only arrangement in which a missing filter + changes the answer. + """ + from tests._fixtures.golden_streams._scrub import _tool_union_ms + + turn = _turn( + duration_seconds=10.0, + commands=[ + _command("agent-call", _at(1000), _at(2000)), + _command("nested-child", _at(1200), _at(1400)), + _command("outside-child", _at(3000), _at(3400)), + _command("unbounded", None, None), + _command("inverted", _at(5000), _at(4000)), + ], + messages=[ + { + "role": "assistant", + "started_at": _at(0).isoformat(), + "completed_at": _at(1000).isoformat(), + "generation_duration_ms": 1000.0, + "tool_use_ids": ["agent-call", "unbounded", "inverted"], + }, + { + "role": "assistant", + "started_at": _at(1200).isoformat(), + "completed_at": _at(3400).isoformat(), + "generation_duration_ms": 400.0, + "parent_tool_use_id": "agent-call", + "tool_use_ids": ["nested-child", "outside-child"], + }, + ], + ) + # Only the parent Agent call's own 1000 ms: both children excluded, the + # unbounded one unplaceable, the inverted one dropped. + expected = 1000.0 + assert _tool_union_ms(turn) == pytest.approx(expected) + assert _load_decompose_run()._tool_ms(turn) == pytest.approx(expected) + + from coder_eval.models import TurnRecord + + record = TurnRecord.model_validate(turn) + assert union_ms(main_thread_tool_spans(record.messages, record.commands)) == pytest.approx(expected) + + def test_a_turn_missing_messages_or_commands_produces_an_empty_span_set(self): + from coder_eval.models import TurnRecord + from tests._fixtures.golden_streams._scrub import _tool_union_ms + + for turn in ({"iteration": 1, "user_input": "", "agent_output": ""},): + record = TurnRecord.model_validate(turn) + assert main_thread_tool_spans(record.messages, record.commands) == [] + assert _tool_union_ms(turn) == 0.0 + assert _load_decompose_run()._tool_ms(turn) == 0.0 From 37211b28172f0d24a354a6031a8af91d8241da19 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:56:06 -0700 Subject: [PATCH 47/54] =?UTF-8?q?feat(reports):=207/8=20=E2=80=94=20the=20?= =?UTF-8?q?buckets=20reach=20every=20surface=20through=20one=20function?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `reports_stats.turn_time_buckets` already said the rule in its own docstring — the evalboard, the markdown report and the HTML report must not each grow their own version — and had exactly one caller. It now has three, and none of the other two sums anything. The `run.json` row projection is where the markdown report's numbers come from. It cannot be otherwise: `task_results[*].iterations` is a deliberate 6-key projection with no `messages`, no `commands` and no `harness_*_ms`, so validating it into a `TurnRecord` there is impossible. The builder already has the `EvaluationResult` in scope, so it calls `turn_time_buckets` once and adds four task-level keys; `reports.py` reads four numbers with `.get()` and renders a dash for each key a `run.json` written before this change does not carry — which is every existing run directory. `build_task_event` emits four optional dimensions, each omitted rather than coalesced when unmeasured, mirroring `Score` verbatim. A dashboard averaging `StartupMs` with no filter would read a laundered zero as a harness that booted instantly, which is indistinguishable from a run predating the capture. `_format_ms` moves into `formatting.py` as `format_ms`, shared by both report renderers. Two renderers formatting the same bucket two ways is how one surface comes to print `0ms` where the other prints a dash — the None-vs-0 distinction thrown away at the last step, after the producer went to the trouble of making it. The Performance section's guard becomes `is not None`. `analysis.py` returns `None` for "nothing timed" and a float otherwise, so a genuine measured 0.0 average — every command resolving faster than the clock's resolution — suppressed the whole section rather than reporting it. On the evalboard, `sumHarnessOverhead` becomes `sumTurnBuckets` and gains the tool total; the task page prefers the stored sum and falls back to computing the union from the message stream for a legacy run. The two agree by construction now that `toolExecutionMs` applies the same bounded-spans-only policy as the Python selector, which is why Phase 3 had to land first. The Generation cell keeps computing from the stream on both sides and has no stored twin by design — the reconciliation entry exists so a consumer sums that stream. The mixed sourcing is deliberate and is commented as such on each side. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- .../__tests__/message-timeline.test.tsx | 24 ++++ .../app/runs/[id]/[...task]/_sections.tsx | 27 ++++- evalboard/app/runs/[id]/[...task]/page.tsx | 1 + evalboard/lib/__tests__/runs.test.ts | 54 +++++---- evalboard/lib/runs.ts | 45 ++++++-- evalboard/lib/timing.ts | 2 +- src/coder_eval/formatting.py | 16 +++ src/coder_eval/orchestrator.py | 17 +++ src/coder_eval/reports.py | 26 ++++- src/coder_eval/reports_experiment.py | 15 ++- src/coder_eval/reports_html.py | 27 ++--- tests/_fixtures/report_snapshots/run_full.md | 8 +- tests/test_orchestrator_telemetry.py | 105 ++++++++++++++++++ tests/test_reports.py | 95 ++++++++++++++++ 14 files changed, 404 insertions(+), 58 deletions(-) diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx index 8e22019f8..cbb939aa7 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx @@ -826,6 +826,7 @@ describe("MessageTimelineSection — Startup and Teardown cells", () => { taskDurationSeconds?: number | null; harnessStartupMs?: number | null; harnessTeardownMs?: number | null; + storedToolMs?: number | null; }) { const m = makeMessage({ generationMs: 4000, @@ -940,6 +941,29 @@ describe("MessageTimelineSection — Startup and Teardown cells", () => { expect(cell("Unaccounted").className).toContain("text-amber-700"); }); + test("the stored tool bucket is preferred over recomputing it", () => { + // The collector wrote `tool_union_ms` from the same span set it + // measured the head and the tail against, so reading it is how this + // cell and the harness are guaranteed to agree. The fixture's own + // messages would compute 1.0s, so a number that is not 2.5s proves the + // stored value was ignored. + renderStrip({ taskDurationSeconds: 10, storedToolMs: 2500 }); + expect(cell("Tool exec").textContent).toBe("2.5s"); + }); + + test("a stored measured zero wins over the fallback", () => { + // `0` is a measurement: spans were recorded and occupied no measurable + // time. Coalescing it away would silently replace it with the 1.0s the + // messages compute. + renderStrip({ taskDurationSeconds: 10, storedToolMs: 0 }); + expect(cell("Tool exec").textContent).toBe("0ms"); + }); + + test("a run predating the field falls back to the message stream", () => { + renderStrip({ taskDurationSeconds: 10 }); + expect(cell("Tool exec").textContent).toBe("1.0s"); + }); + test("each bucket says what it measures and that it is not decomposed", () => { renderStrip({ taskDurationSeconds: 10, diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index 6582a3cca..0b4406c7d 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -320,6 +320,7 @@ export function MessageTimelineSection({ taskDurationSeconds, harnessStartupMs, harnessTeardownMs, + storedToolMs, setupMs, gradingMs, }: { @@ -343,6 +344,15 @@ export function MessageTimelineSection({ // the cells then read "—" while Unaccounted keeps exactly its old meaning. harnessStartupMs?: number | null; harnessTeardownMs?: number | null; + // The tool bucket as the HARNESS recorded it, summed over the task's turns + // (`TurnRecord.tool_union_ms`). Preferred over recomputing it from the + // message stream, because the collector wrote it from the same span set it + // measured the head and the tail against — reading it is how this cell and + // the harness are guaranteed to agree rather than merely observed to. + // Null/absent on a run predating the field, and the cell then computes the + // union itself; the two agree by construction, since `toolExecutionMs` + // applies the same bounded-spans-only policy as the Python selector. + storedToolMs?: number | null; // TASK-scoped phases either side of the turns: provisioning before the // first turn, criteria checking after the last. Named so Unaccounted is a // residual instead of a label for the setup phase — it was ~1.9s of known, @@ -401,7 +411,17 @@ export function MessageTimelineSection({ // occupy the wall clock once. Summing them made Unaccounted negative on // any task that ran tools in parallel, reporting overlap as if the // harness had lost time. - const toolExecMs = toolExecutionMs(mainThread); + // + // STORED first, computed as the fallback. `?? null` and not `?? computed` + // in one expression because `storedToolMs` of 0 is a measurement and must + // win: the harness recorded spans and they occupied no measurable time. + // Only its ABSENCE (a run predating the field) routes here. + // + // The Generation cell below has no stored twin and is deliberately still + // computed from the messages — the reconciliation entry exists so a + // consumer sums that stream rather than reading a separate aggregate. The + // mixed sourcing is intentional; see `sumTurnBuckets` in lib/runs.ts. + const toolExecMs = storedToolMs ?? toolExecutionMs(mainThread); const slowGen = mainThread.filter( (m) => (m.generationMs ?? 0) >= SLOW_GEN_MS, ).length; @@ -496,7 +516,7 @@ export function MessageTimelineSection({ Tool exec
- {fmtMs(toolExecMs)} + {fmtMs(toolExecMs ?? null)}
@@ -840,6 +860,7 @@ export function CostExplorerSection({ taskDurationSeconds, harnessStartupMs, harnessTeardownMs, + storedToolMs, setupMs, gradingMs, }: { @@ -852,6 +873,7 @@ export function CostExplorerSection({ // Forwarded verbatim to the timeline's Startup/Teardown cells. harnessStartupMs?: number | null; harnessTeardownMs?: number | null; + storedToolMs?: number | null; // Forwarded straight through to MessageTimelineSection — this component // renders it and owns no timing of its own. setupMs?: number | null; @@ -896,6 +918,7 @@ export function CostExplorerSection({ taskDurationSeconds={taskDurationSeconds} harnessStartupMs={harnessStartupMs} harnessTeardownMs={harnessTeardownMs} + storedToolMs={storedToolMs} setupMs={setupMs} gradingMs={gradingMs} /> diff --git a/evalboard/app/runs/[id]/[...task]/page.tsx b/evalboard/app/runs/[id]/[...task]/page.tsx index 077a00d70..760dcf53d 100644 --- a/evalboard/app/runs/[id]/[...task]/page.tsx +++ b/evalboard/app/runs/[id]/[...task]/page.tsx @@ -369,6 +369,7 @@ export default async function TaskPage({ setupMs={task.setupMs} gradingMs={task.gradingMs} harnessTeardownMs={task.harnessTeardownMs} + storedToolMs={task.storedToolMs} /> )} diff --git a/evalboard/lib/__tests__/runs.test.ts b/evalboard/lib/__tests__/runs.test.ts index 2a0cced0c..d7736f8de 100644 --- a/evalboard/lib/__tests__/runs.test.ts +++ b/evalboard/lib/__tests__/runs.test.ts @@ -24,7 +24,7 @@ import { parseCriterionResults, type RawTaskResult, sortArtifacts, - sumHarnessOverhead, + sumTurnBuckets, toTaskRow, visibleTurnsFromRaw, walkArtifacts, @@ -247,59 +247,71 @@ describe("aggregateSubAgentUsage", () => { }); }); -describe("sumHarnessOverhead", () => { - test("sums both buckets across iterations", () => { +describe("sumTurnBuckets", () => { + test("sums all three buckets across iterations", () => { expect( - sumHarnessOverhead([ - { harness_startup_ms: 3000, harness_teardown_ms: 800 }, - { harness_startup_ms: 120, harness_teardown_ms: 40 }, + sumTurnBuckets([ + { harness_startup_ms: 3000, harness_teardown_ms: 800, tool_union_ms: 200 }, + { harness_startup_ms: 120, harness_teardown_ms: 40, tool_union_ms: 50 }, ]), - ).toEqual({ startupMs: 3120, teardownMs: 840 }); + ).toEqual({ startupMs: 3120, teardownMs: 840, toolMs: 250 }); }); test("a measured zero is a measurement and still sums", () => { // A harness that reached its first model output with nothing // measurable in front of it legitimately reports 0.0 — a clamped - // inversion where both ends were still observed. That is a number, - // not a gap, and the assertion holds however the head is produced. + // inversion where both ends were still observed. The same holds for a + // tool union: spans were recorded and occupied no measurable time. expect( - sumHarnessOverhead([{ harness_startup_ms: 0, harness_teardown_ms: 3.5 }]), - ).toEqual({ startupMs: 0, teardownMs: 3.5 }); + sumTurnBuckets([ + { harness_startup_ms: 0, harness_teardown_ms: 3.5, tool_union_ms: 0 }, + ]), + ).toEqual({ startupMs: 0, teardownMs: 3.5, toolMs: 0 }); }); test("is null when EVERY iteration is null — never 0", () => { // 0 would claim the harness started instantly; null says nobody looked. expect( - sumHarnessOverhead([ - { harness_startup_ms: null, harness_teardown_ms: null }, + sumTurnBuckets([ + { harness_startup_ms: null, harness_teardown_ms: null, tool_union_ms: null }, {}, ]), - ).toEqual({ startupMs: null, teardownMs: null }); + ).toEqual({ startupMs: null, teardownMs: null, toolMs: null }); + }); + + test("a run predating tool_union_ms reports null for it and real numbers beside it", () => { + // The legacy shape, and the one that routes the task page to computing + // the union from the message stream instead. + expect( + sumTurnBuckets([{ harness_startup_ms: 500, harness_teardown_ms: 90 }]), + ).toEqual({ startupMs: 500, teardownMs: 90, toolMs: null }); }); test("sums the measured iterations and ignores the unmeasured ones", () => { expect( - sumHarnessOverhead([ + sumTurnBuckets([ { harness_startup_ms: 500 }, { harness_teardown_ms: 90 }, + { tool_union_ms: 12 }, ]), - ).toEqual({ startupMs: 500, teardownMs: 90 }); + ).toEqual({ startupMs: 500, teardownMs: 90, toolMs: 12 }); }); test("is null on an empty turn list", () => { - expect(sumHarnessOverhead([])).toEqual({ + expect(sumTurnBuckets([])).toEqual({ startupMs: null, teardownMs: null, + toolMs: null, }); }); test("a non-finite value is dropped rather than poisoning the sum", () => { expect( - sumHarnessOverhead([ - { harness_startup_ms: NaN, harness_teardown_ms: 10 }, - { harness_startup_ms: 25 }, + sumTurnBuckets([ + { harness_startup_ms: NaN, harness_teardown_ms: 10, tool_union_ms: Infinity }, + { harness_startup_ms: 25, tool_union_ms: 7 }, ]), - ).toEqual({ startupMs: 25, teardownMs: 10 }); + ).toEqual({ startupMs: 25, teardownMs: 10, toolMs: 7 }); }); }); diff --git a/evalboard/lib/runs.ts b/evalboard/lib/runs.ts index 04288b119..f19ab684d 100644 --- a/evalboard/lib/runs.ts +++ b/evalboard/lib/runs.ts @@ -370,6 +370,12 @@ export interface TaskDetail extends TaskResultSummary { // that residual is what is left after every named bucket. harnessStartupMs: number | null; harnessTeardownMs: number | null; + // The task's tool bucket as the HARNESS recorded it: the sum of its turns' + // `tool_union_ms`. `null` on a run predating the field, which is what makes + // the task page fall back to computing the union from the messages — the + // two agree by construction, since `toolExecutionMs` applies the same + // bounded-spans-only policy as `timing.main_thread_tool_spans`. + storedToolMs: number | null; // TASK-scoped phases either side of the agent's turns, so the timeline's // Unaccounted cell is a residual rather than a name for the setup phase. // `setupMs` is sandbox provisioning + agent start() + pre_run; `gradingMs` @@ -422,18 +428,31 @@ function sumMeasured(values: (number | null | undefined)[]): number | null { return total; } -// The task's harness head and tail, summed over its turns. The per-turn values -// are measured by `coder_eval/timing.py::decompose_turn`; the summation is -// evalboard-only, and the arithmetic that consumes it — the Unaccounted -// residual in `_sections.tsx` — is the deliberate second implementation that -// helper's docstring names (as `pricing.ts` mirrors `pricing.py`). -export function sumHarnessOverhead(turns: TurnEntry[]): { +// Three of the task's four wall-clock buckets, summed over its turns. The +// per-turn values are measured by `coder_eval/timing.py` — head and tail by +// `decompose_turn`, the tool union at the collector seam — and the summation is +// evalboard-only, mirroring `reports_stats.turn_time_buckets` the way +// `pricing.ts` mirrors `pricing.py`. The arithmetic that consumes it, the +// Unaccounted residual in `_sections.tsx`, is the deliberate second +// implementation `decompose_turn`'s docstring names. +// +// GENERATION is deliberately not here and has no stored twin on either side: +// it is a one-line sum over the message stream, and the reconciliation entry +// exists precisely so a consumer sums that stream rather than reading a +// separate aggregate. `toolMs` is the opposite case — union arithmetic plus a +// sub-agent filter — which is why it earns storage. +// +// `toolMs` is null when NO turn carries the field, which is every run recorded +// before it existed; the caller then computes it from the messages instead. +export function sumTurnBuckets(turns: TurnEntry[]): { startupMs: number | null; teardownMs: number | null; + toolMs: number | null; } { return { startupMs: sumMeasured(turns.map((t) => t.harness_startup_ms)), teardownMs: sumMeasured(turns.map((t) => t.harness_teardown_ms)), + toolMs: sumMeasured(turns.map((t) => t.tool_union_ms)), }; } @@ -1580,6 +1599,12 @@ export interface TurnEntry { // cases nobody measured, which is a different fact from a measured 0. harness_startup_ms?: number | null; harness_teardown_ms?: number | null; + // The turn's tool bucket: the UNION of its main-thread bounded command + // intervals, written by the collector from the same span set the two above + // are measured against. Optional for the same reason they are — a run + // predating the field carries none, which is what routes the task page to + // computing it from the messages instead. + tool_union_ms?: number | null; // Per-call actual cost + cache audit rows (LiteLLM/open-weight backend); // empty/absent on Claude/Bedrock. Surfaced as a standalone per-call table. provider_call_costs?: ProviderCallEntryRaw[]; @@ -2575,8 +2600,11 @@ export async function readTaskDetail( const tokens = selectTokenTotals(messages, task?.iterations ?? []); const subAgentUsageByToolId = aggregateSubAgentUsage(messages); - const { startupMs: harnessStartupMs, teardownMs: harnessTeardownMs } = - sumHarnessOverhead(task?.iterations ?? []); + const { + startupMs: harnessStartupMs, + teardownMs: harnessTeardownMs, + toolMs: storedToolMs, + } = sumTurnBuckets(task?.iterations ?? []); // Through `sumMeasured` for the single-value case too, so the None-vs-0 // and non-finite rules have ONE implementation: a `?? null` here would // pass a NaN straight into the Unaccounted subtraction. @@ -2624,6 +2652,7 @@ export async function readTaskDetail( subAgentUsageByToolId, harnessStartupMs, harnessTeardownMs, + storedToolMs, setupMs, gradingMs, providerCalls, diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts index f26fba3d4..05b33a4c3 100644 --- a/evalboard/lib/timing.ts +++ b/evalboard/lib/timing.ts @@ -138,7 +138,7 @@ export function epochMs(value: string | null | undefined): number | null { // // The TypeScript twin of `coder_eval.timing.busy_ms`, deliberately the // same algorithm — the harness subtracts tool time from its generation windows -// with it (once, in streaming/collector.py::subtract_tool_time), and this file +// with it (once, in coder_eval/timing.py::subtract_tool_time), and this file // subtracts tool time from a task's wall clock, so the two must agree about // what "tool execution took N ms" means. Held in step by // tests/_fixtures/timing_union_cases.json, which both suites replay. diff --git a/src/coder_eval/formatting.py b/src/coder_eval/formatting.py index 15685eeb4..a5cd052ee 100644 --- a/src/coder_eval/formatting.py +++ b/src/coder_eval/formatting.py @@ -18,6 +18,22 @@ logger = logging.getLogger(__name__) +def format_ms(ms: float | None) -> str: + """A duration in ms, or an em dash when it was never measured. + + SHARED by the HTML report and the markdown one. They render the same four + wall-clock buckets from the same `reports_stats.turn_time_buckets` call, so + formatting them twice is how one surface comes to print `0ms` where the + other prints a dash — the `None`-vs-`0.0` distinction CE058 enforces on the + producing side, thrown away at the last step. + """ + if ms is None: + return "—" + if ms < 1000: + return f"{ms:.0f}ms" + return f"{ms / 1000:.2f}s" + + def format_messages( messages: list[Message], *, diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index a7b04e471..51f8fb7ec 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -319,6 +319,23 @@ def build_task_event(result: EvaluationResult, *, driver: str, variant_id: str) # An absent dimension drops out of the average instead. if result.weighted_score is not None: props["Score"] = float(result.weighted_score) + # The four wall-clock buckets, from the ONE canonical summation — this + # function does not add anything up itself. Each is OMITTED rather than + # coalesced to 0, for the same reason as `Score` above: a dashboard + # averaging `StartupMs` with no filter would read a laundered zero as a + # harness that booted instantly, which is indistinguishable from a run that + # predates the capture. An absent dimension drops out of the average. + from .reports_stats import turn_time_buckets + + buckets = turn_time_buckets(result) + for name, value in ( + ("StartupMs", buckets.startup_ms), + ("GenerationMs", buckets.generation_ms), + ("ToolExecMs", buckets.tool_ms), + ("TeardownMs", buckets.teardown_ms), + ): + if value is not None: + props[name] = float(value) return "CoderEval.Task.End", props diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index 2475d1b43..4981b9793 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -8,6 +8,7 @@ from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Literal, assert_never +from .formatting import format_ms from .models import ( CriterionAggregate, CriterionStats, @@ -295,7 +296,12 @@ def _generate_command_statistics_section(stats: CommandStatistics) -> list[str]: pct = count / total * 100 if total > 0 else 0 lines.append(f"| {tool} | {count} | {pct:.1f}% |") - if stats.avg_command_time_ms and stats.avg_command_time_ms > 0: + # `is not None`, not truthiness. `analysis.py` returns `None` when + # nothing was timed and a float otherwise, so a genuine measured `0.0` + # average — every command resolving faster than the clock's resolution — + # used to suppress the whole section. The distinction the producer makes + # has to survive to the surface that renders it. + if stats.avg_command_time_ms is not None: lines.extend( [ "", @@ -340,8 +346,10 @@ def _generate_generation_metrics_section(task_results: list[dict[str, Any]]) -> lines = [ "## Generation Metrics", "", - "| Task ID | Total Latency | Turns | Asst Turns | Avg Turn Latency |", - "|---------|---------------|-------|------------|------------------|", + "| Task ID | Total Latency | Turns | Asst Turns | Avg Turn Latency " + + "| Startup | Generation | Tool exec | Teardown |", + "|---------|---------------|-------|------------|------------------" + + "|---------|------------|-----------|----------|", ] for task in task_results: @@ -358,7 +366,17 @@ def _generate_generation_metrics_section(task_results: list[dict[str, Any]]) -> else: avg_turn_str = "N/A" - lines.append(f"| {task_id} | {total_latency} | {num_turns} | {asst_turns} | {avg_turn_str} |") + # READ, never summed here. The four values are computed once by + # `reports_stats.turn_time_buckets` and carried on the row by + # `reports_experiment.eval_result_to_task_dict`; `iterations` above + # is a 6-key projection that cannot support the arithmetic anyway. + # `.get()` because a `run.json` written before this phase has none + # of the four — which then renders as a dash, not as `0ms`. + buckets = " | ".join( + format_ms(task.get(key)) for key in ("startup_ms", "generation_ms", "tool_ms", "teardown_ms") + ) + + lines.append(f"| {task_id} | {total_latency} | {num_turns} | {asst_turns} | {avg_turn_str} | {buckets} |") return lines diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index dc3b5eb68..acbd2a72c 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -102,7 +102,7 @@ def eval_result_to_task_dict( downstream consumers (evalboard) collapse them to one. ``None`` when the caller doesn't track replicates (repeats disabled / legacy). """ - from coder_eval.reports_stats import expected_turns_overage, visible_turn_count + from coder_eval.reports_stats import expected_turns_overage, turn_time_buckets, visible_turn_count from coder_eval.reports_stats import has_final_reply as _has_final_reply ref_similarity: float | None = None @@ -134,6 +134,8 @@ def eval_result_to_task_dict( if isinstance(raw, int) and raw >= 1: expected_turns_value = raw + _buckets = turn_time_buckets(result) + d: dict[str, Any] = { "task_id": result.task_id, "replicate_index": replicate_index, @@ -154,6 +156,17 @@ def eval_result_to_task_dict( } for t in result.iterations ], + # The four wall-clock buckets, computed ONCE here through the canonical + # `turn_time_buckets` and carried as TASK-level keys. `iterations` below + # is a deliberate 6-key projection with no `messages`, no `commands` and + # no `harness_*_ms`, so the markdown report cannot re-derive them from + # it — and a second implementation of the summation is exactly what that + # function exists to prevent. Each stays `float | None`: an unmeasured + # bucket renders as a dash, never as `0ms` (CE049). + "startup_ms": _buckets.startup_ms, + "generation_ms": _buckets.generation_ms, + "tool_ms": _buckets.tool_ms, + "teardown_ms": _buckets.teardown_ms, "model_used": result.model_used, "reference_similarity": ref_similarity, "input_tokens": (result.total_token_usage.uncached_input_tokens if result.total_token_usage else None), diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index c0c1c8ebf..e41ac64a7 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any +from coder_eval.formatting import format_ms from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs from .reports import early_stop_gate_note @@ -303,14 +304,6 @@ def _format_duration(seconds: float | None) -> str: return f"{minutes}m {secs:.0f}s" -def _format_ms(ms: float | None) -> str: - if ms is None: - return "—" - if ms < 1000: - return f"{ms:.0f}ms" - return f"{ms / 1000:.2f}s" - - def _format_params(params: dict[str, Any]) -> str: """Pretty-print tool parameters as JSON, truncating long values.""" try: @@ -675,7 +668,7 @@ def _render_command(cmd: CommandTelemetry) -> str: #{cmd.sequence_number} {_esc(cmd.tool_name)} {_esc(status_label)} - {_esc(_format_ms(cmd.duration_ms))} + {_esc(format_ms(cmd.duration_ms))}
@@ -818,13 +811,13 @@ def _render_command_stats(stats: Any | None) -> str: if len(params_full) > SLOW_PARAMS_PREVIEW_CHARS: params_preview += "..." slow_rows_list.append( - f"{_esc(c.tool)}{_esc(_format_ms(c.duration_ms))}" + f"{_esc(c.tool)}{_esc(format_ms(c.duration_ms))}" + f"{_esc(params_preview)}" ) slow_rows = "".join(slow_rows_list) or "—" success_pct = (stats.successful_commands / stats.total_commands * 100) if stats.total_commands else 0.0 successful_str = f"{stats.successful_commands} ({success_pct:.0f}%)" - avg_str = _esc(_format_ms(stats.avg_command_time_ms)) + avg_str = _esc(format_ms(stats.avg_command_time_ms)) extras: list[str] = [] if stats.most_common_sequence: @@ -933,7 +926,7 @@ def _render_token_usage(result: EvaluationResult) -> str: def _format_signed_ms(ms: float | None) -> str: - """Like `_format_ms`, but keeps a NEGATIVE residual visible and signed. + """Like `format_ms`, but keeps a NEGATIVE residual visible and signed. A negative Unaccounted is real and means generation and tool execution overlapped, so it is rendered rather than clamped — the evalboard does the @@ -941,7 +934,7 @@ def _format_signed_ms(ms: float | None) -> str: """ if ms is None: return "—" - return f"-{_format_ms(-ms)}" if ms < 0 else _format_ms(ms) + return f"-{format_ms(-ms)}" if ms < 0 else format_ms(ms) def _render_generation_metrics(result: EvaluationResult) -> str: @@ -968,10 +961,10 @@ def _render_generation_metrics(result: EvaluationResult) -> str: # a run predating the head/tail capture measured nothing, and a zero would # claim it measured instantly (CE058). buckets = turn_time_buckets(result) - startup = _format_ms(buckets.startup_ms) - generation = _format_ms(buckets.generation_ms) - tool_exec = _format_ms(buckets.tool_ms) - teardown = _format_ms(buckets.teardown_ms) + startup = format_ms(buckets.startup_ms) + generation = format_ms(buckets.generation_ms) + tool_exec = format_ms(buckets.tool_ms) + teardown = format_ms(buckets.teardown_ms) unaccounted = _format_signed_ms(buckets.unaccounted_ms) unaccounted_title = _esc( "the task's wall clock minus the four buckets. Measured against the whole task, so it " diff --git a/tests/_fixtures/report_snapshots/run_full.md b/tests/_fixtures/report_snapshots/run_full.md index 3e14b5ec7..e576cf416 100644 --- a/tests/_fixtures/report_snapshots/run_full.md +++ b/tests/_fixtures/report_snapshots/run_full.md @@ -33,10 +33,10 @@ ## Generation Metrics -| Task ID | Total Latency | Turns | Asst Turns | Avg Turn Latency | -|---------|---------------|-------|------------|------------------| -| alpha | 12.5s | 1 | 3 | 4.2s | -| beta | 8.0s | 1 | 1 | 2.0s | +| Task ID | Total Latency | Turns | Asst Turns | Avg Turn Latency | Startup | Generation | Tool exec | Teardown | +|---------|---------------|-------|------------|------------------|---------|------------|-----------|----------| +| alpha | 12.5s | 1 | 3 | 4.2s | — | — | — | — | +| beta | 8.0s | 1 | 1 | 2.0s | — | — | — | — | ## Token Usage diff --git a/tests/test_orchestrator_telemetry.py b/tests/test_orchestrator_telemetry.py index c5de17132..9d6699d4a 100644 --- a/tests/test_orchestrator_telemetry.py +++ b/tests/test_orchestrator_telemetry.py @@ -236,3 +236,108 @@ async def test_docker_path_emits_task_end_host_side(tmp_path): assert name == "CoderEval.Task.End" assert props["Driver"] == "docker" assert props["TaskId"] == hash_identifier("dock-task") + + +class TestTheFourBucketDimensions: + """`CoderEval.Task.End` carries the wall-clock buckets, each independently optional. + + Each is OMITTED rather than coalesced to 0 when unmeasured, for the same + reason `Score` is: a dashboard averaging `StartupMs` with no filter reads a + laundered zero as a harness that booted instantly, which is + indistinguishable from a run predating the capture. Four separate + assertions, not one combined — a single `all four present` check passes + even if one dimension were wired to another's value. + """ + + _NAMES = ("StartupMs", "GenerationMs", "ToolExecMs", "TeardownMs") + + @staticmethod + def _result(turns): + from coder_eval.models import TurnRecord + + return EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime.now(), + final_status=FinalStatus.SUCCESS, + iteration_count=len(turns), + environment_info={}, + duration_seconds=10.0, + iterations=[TurnRecord.model_validate(t) for t in turns], + ) + + @staticmethod + def _turn(**overrides): + from datetime import timedelta + + base = datetime(2026, 9, 11, 9, 0, 0) + turn = { + "iteration": 1, + "user_input": "go", + "agent_output": "done", + "duration_seconds": 5.0, + "messages": [ + { + "role": "assistant", + "started_at": base.isoformat(), + "completed_at": (base + timedelta(milliseconds=800)).isoformat(), + "generation_duration_ms": 800.0, + } + ], + "harness_startup_ms": 500.0, + "harness_teardown_ms": 100.0, + "tool_union_ms": 200.0, + } + turn.update(overrides) + return turn + + def _props(self, **overrides): + from coder_eval.orchestrator import build_task_event + + _, props = build_task_event(self._result([self._turn(**overrides)]), driver="tempdir", variant_id="v1") + return props + + def test_every_dimension_is_present_and_carries_its_own_value(self): + props = self._props() + assert props["StartupMs"] == pytest.approx(500.0) + assert props["GenerationMs"] == pytest.approx(800.0) + assert props["ToolExecMs"] == pytest.approx(200.0) + assert props["TeardownMs"] == pytest.approx(100.0) + + def test_an_unmeasured_startup_is_omitted(self): + props = self._props(harness_startup_ms=None) + assert "StartupMs" not in props + assert "GenerationMs" in props and "ToolExecMs" in props and "TeardownMs" in props + + def test_an_unmeasured_teardown_is_omitted(self): + props = self._props(harness_teardown_ms=None) + assert "TeardownMs" not in props + assert "StartupMs" in props + + def test_an_unmeasured_tool_bucket_is_omitted(self): + # No stored value AND no bounded command to derive one from. + props = self._props(tool_union_ms=None, commands=[]) + assert "ToolExecMs" not in props + assert "GenerationMs" in props + + def test_an_unmeasured_generation_is_omitted(self): + props = self._props(messages=[]) + assert "GenerationMs" not in props + assert "StartupMs" in props + + def test_a_measured_zero_is_emitted_rather_than_omitted(self): + """The control: 0.0 is a measurement and must reach the dashboard.""" + props = self._props(harness_startup_ms=0.0) + assert props["StartupMs"] == 0.0 + + def test_it_does_not_sum_anything_itself(self): + """One producer for the buckets, and `build_task_event` is not it.""" + import inspect + + from coder_eval.orchestrator import build_task_event + + source = inspect.getsource(build_task_event) + assert "turn_time_buckets(result)" in source + assert "harness_startup_ms" not in source, "the summation belongs to reports_stats" diff --git a/tests/test_reports.py b/tests/test_reports.py index 7462fbd75..0b8df6347 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1381,3 +1381,98 @@ def test_an_ordinary_graded_run_is_untouched(self): assert summary.pass_rate == 0.5 assert summary.error_share == 0.25 + + +class TestTheGenerationMetricsBuckets: + """The markdown table's four bucket columns, READ off the row projection. + + `reports.py` neither sums nor validates anything here: the numbers are + computed once by `reports_stats.turn_time_buckets` and carried as + task-level keys by `reports_experiment.eval_result_to_task_dict`. The rows + below are that projection's shape, not a `TurnRecord`. + """ + + @staticmethod + def _row(**overrides) -> dict: + row = { + "task_id": "alpha", + "duration": 12.5, + "iterations": [{"duration_seconds": 12.5, "assistant_turn_count": 3}], + } + row.update(overrides) + return row + + @staticmethod + def _cells(row: dict) -> list[str]: + from coder_eval.reports import ReportGenerator + + lines = ReportGenerator._generate_generation_metrics_section([row]) + return [cell.strip() for cell in lines[-1].strip("|").split("|")] + + def test_a_measured_run_renders_every_bucket(self): + cells = self._cells(self._row(startup_ms=500.0, generation_ms=1800.0, tool_ms=200.0, teardown_ms=100.0)) + assert cells[-4:] == ["500ms", "1.80s", "200ms", "100ms"] + + def test_a_run_json_predating_the_keys_renders_dashes_not_zeros(self): + """The common case for every existing run directory. + + `0ms` would claim a measurement nobody took — the same distinction + CE058 enforces on the producing side, and the reason the keys are read + with `.get()` rather than indexed. + """ + assert self._cells(self._row())[-4:] == ["—", "—", "—", "—"] + + def test_a_measured_zero_still_renders_as_zero(self): + assert self._cells(self._row(startup_ms=0.0))[-4] == "0ms" + + def test_each_column_carries_its_own_value(self): + """Pins the key-to-column WIRING: four distinct numbers, so a swap shows.""" + cells = self._cells(self._row(startup_ms=1.0, generation_ms=2.0, tool_ms=3.0, teardown_ms=4.0)) + assert cells[-4:] == ["1ms", "2ms", "3ms", "4ms"] + + def test_the_report_neither_sums_nor_validates(self): + """Asserted on the compiled NAMES, not on the source text. + + The function's comment legitimately names `turn_time_buckets` to say + where the numbers came from; a substring check over the source would + read that as a call. `co_names` sees what the code actually references. + """ + from coder_eval.reports import ReportGenerator + + names = set(ReportGenerator._generate_generation_metrics_section.__code__.co_names) + assert not names & {"turn_time_buckets", "TurnRecord", "harness_startup_ms", "model_validate"} + + +class TestThePerformanceSectionRendersAMeasuredZero: + """`analysis.py` returns `None` for "nothing timed" and a float otherwise. + + The guard was `if stats.avg_command_time_ms and ... > 0`, so a genuine + measured 0.0 average — every command resolving faster than the clock's + resolution — suppressed the whole section. The distinction the producer + makes has to survive to the surface that renders it. + """ + + @staticmethod + def _section(avg: float | None, total: float = 0.0) -> list[str]: + from coder_eval.models import CommandStatistics + from coder_eval.reports import ReportGenerator + + stats = CommandStatistics( + total_commands=3, + successful_commands=3, + avg_command_time_ms=avg, + total_command_time_ms=total, + ) + return ReportGenerator._generate_command_statistics_section(stats) + + def test_a_measured_zero_average_still_renders(self): + lines = self._section(0.0) + assert any("### Performance" in line for line in lines) + assert any("**Average Command Time**: 0.0ms" in line for line in lines) + + def test_an_unmeasured_average_is_omitted(self): + assert not any("### Performance" in line for line in self._section(None)) + + def test_an_ordinary_average_is_unchanged(self): + lines = self._section(150.0, total=450.0) + assert any("**Average Command Time**: 150.0ms" in line for line in lines) From 8b4d4a83e29f3500f50824565ffaeb6dc7c1cf78 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 10:56:19 -0700 Subject: [PATCH 48/54] =?UTF-8?q?refactor(opencode):=208/8=20=E2=80=94=20o?= =?UTF-8?q?ne=20duplicate=20parse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `on_tool_use` read `state["time"]` and re-wrapped it as a dict twice, identically — once at the top and again immediately before closing the tool. `state` is bound once at the start of the function and nothing between the two rebinds or mutates it, so the second read produced the same value from the same source. The test that comes with it is the point, since the deletion is behaviour-preserving by construction and would pass either way: the two events it feeds carry DIFFERENT `time` payloads, so a close that took its `end` stamp from the wrong read would fail it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- src/coder_eval/agents/opencode_agent.py | 10 ++++--- tests/test_opencode_agent.py | 36 +++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/src/coder_eval/agents/opencode_agent.py b/src/coder_eval/agents/opencode_agent.py index c7bd6d561..42fca0d4d 100644 --- a/src/coder_eval/agents/opencode_agent.py +++ b/src/coder_eval/agents/opencode_agent.py @@ -363,7 +363,7 @@ def on_step_start(self, part: dict[str, Any]) -> None: self.step_text_parts = [] self.step_tool_ids = [] # There is no per-step span list to reset here any more, and that whole - # class of defect is gone with it: `EventCollector.subtract_tool_time` + # class of defect is gone with it: `timing.subtract_tool_time` # sees every span at once and clips each to the window it overlaps, so # a call closing in the gap before this `step_start` needs nobody to # remember it. The reset rule that used to live here was wrong once @@ -452,8 +452,10 @@ def on_tool_use(self, part: dict[str, Any]) -> None: message = None status = ToolEndStatus.OK - time_val = state.get("time") - times = time_val if isinstance(time_val, dict) else {} + # `times` is the SAME dict read at the top of this function: `state` is + # bound once at the start and nothing between here and there rebinds or + # mutates it, so re-reading `state["time"]` produced an identical value + # from an identical source. One read, one name. self._close_tool( call_id, status=status, @@ -703,7 +705,7 @@ def on_step_finish(self, part: dict[str, Any]) -> None: blocks.append(ContentBlock(block_type="tool_use", sequence=i, tool_use_id=tool_id)) # Tile from the previous step's finish. The RAW window only — - # `EventCollector.subtract_tool_time` takes the tool union back out of + # `timing.subtract_tool_time` takes the tool union back out of # it, once, for every harness. started, generation_ms = close_window( mark=self.gen_mark if self.gen_mark is not None else step_start, diff --git a/tests/test_opencode_agent.py b/tests/test_opencode_agent.py index 605563fab..7b7b679ea 100644 --- a/tests/test_opencode_agent.py +++ b/tests/test_opencode_agent.py @@ -514,6 +514,38 @@ async def test_one_tool_start_end_pair_is_emitted(self, patch_exec, tmp_path): assert len([e for e in recorder.events if isinstance(e, ToolStartEvent)]) == 1 assert len([e for e in recorder.events if isinstance(e, ToolEndEvent)]) == 1 + async def test_the_completion_time_end_becomes_execution_completed_at(self, patch_exec, tmp_path): + """The path the deleted second `state["time"]` read served. + + `on_tool_use` parsed `state.time` twice, identically, once at the top + and again just before closing the tool. The second read is gone; this + asserts the close still gets its `end` stamp from the same dict — and + gets the RIGHT one, since the two events carry different times and only + the completion's may be published. + """ + patch_exec( + _FakeProcess( + [ + _evt("step_start", {"id": "prt_1", "messageID": "msg_1", "type": "step-start"}), + self._event("running", {"time": {"start": 1786663018214}}), + self._event( + "completed", + { + "input": {"command": "ls"}, + "output": "ok", + "time": {"start": 1786663018214, "end": 1786663018231}, + }, + ), + ] + ) + ) + record = await _run(_agent(), tmp_path) + + cmd = record.commands[0] + assert cmd.execution_completed_at == datetime.fromtimestamp(1786663018231 / 1000.0) + assert cmd.execution_started_at == datetime.fromtimestamp(1786663018214 / 1000.0) + assert cmd.duration_ms == pytest.approx(17.0) + async def test_a_later_event_without_input_never_clears_what_we_have(self, patch_exec, tmp_path): """Absent evidence is not evidence of absence — the first event's args stay.""" patch_exec( @@ -1788,7 +1820,7 @@ class TestGenerationWindowExcludesToolExecution: """A tool running inside a step is not model time — asserted where it is now DECIDED. The reducer no longer subtracts anything. It publishes the RAW window, and - `EventCollector.subtract_tool_time` takes the tool union back out of it + `timing.subtract_tool_time` takes the tool union back out of it once, for all five harnesses. So these cases drive the reducer and then a real collector, and assert the PUBLISHED number — the one that reaches `task.json` — rather than an intermediate the reducer used to own. @@ -2046,7 +2078,7 @@ class TestToolSpansSurviveTheStepBoundary: `step_start` — after the window it feeds had already opened at `gen_mark` — so a call closing in the gap had its span wiped before the next `step_finish` could subtract it. That list is gone. - `EventCollector.subtract_tool_time` sees every span at once and clips each + `timing.subtract_tool_time` sees every span at once and clips each to the windows it overlaps, so the property now holds by construction rather than by a reset rule. Kept, and re-pointed at the collector, because the property is what matters: a future reducer change could still break it From 0cf7101c57eadbe03692568e94f2759972f70983 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 14:03:22 -0700 Subject: [PATCH 49/54] fix: code review fixes for turn-timing-consolidation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from the final multi-model review, all in the arithmetic the change moved rather than in what it added. The seam assertion's tolerance was one NANOSECOND expressed in milliseconds. The exposure that check is actually for is a third-party agent registered through the `coder_eval.plugins` SPI — and that is precisely the producer most likely to record microsecond-precision bounds while publishing a duration rounded to whole milliseconds, which the check would have rejected by 0.4 ms, crashing every one of its turns. A guard that kills legitimate producers is relocating a defect, not removing one. One millisecond is the coarsest unit a field named `_ms` can honestly be published in, and it still catches the defect class by three to six orders of magnitude: a reducer that narrowed a window by subtracting its own tool time is off by tens to thousands of milliseconds. The apportioning loop rounded every share but the last to six places while the last took the remainder. Rounding each earlier share UP can push the running total past the net, and the last member then receives a NEGATIVE duration. It needs a net well under a microsecond — a window almost entirely covered by tool execution — so it had never been seen, but a negative generation is an invariant break rather than a rounding artifact, and the remainder already prevents the drift the rounding was there for. `toolExecutionMs` derived its extent with `Math.min(...spans.map(…))`, which passes one ARGUMENT per span; a long enough trace throws RangeError and the whole task page fails to render. Folded instead, which is how the Python twin's generator `min`/`max` already behaves. Also states, where the code is rather than in a commit message, why the `raw_total <= 0` skip has to run BEFORE the assertion: `close_window` clamps a measured inversion to `0.0` while its bounds still say `completed_at < started_at`, so checking first would kill the turn on exactly the shape `decompose_turn` deliberately tolerates. A reviewer read that ordering as a bypass, which is a fair reading of code that did not say so. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- .claude/harness-candidates.md | 16 +++++++++++++ evalboard/lib/timing.ts | 12 ++++++++-- src/coder_eval/timing.py | 40 +++++++++++++++++++++++++++++++-- tests/test_event_collector.py | 42 +++++++++++++++++++++++++++-------- 4 files changed, 97 insertions(+), 13 deletions(-) diff --git a/.claude/harness-candidates.md b/.claude/harness-candidates.md index 7069c7c33..fcfb02b43 100644 --- a/.claude/harness-candidates.md +++ b/.claude/harness-candidates.md @@ -857,3 +857,19 @@ re-derive from scratch. or enclosing timing-constructor keyword — matches `_TIMING_NAME`. Deferred because the target-name resolution is new machinery rather than a variant of an existing form. Caught in: the turn-timing consolidation, Phase 5 review. + +- [ ] **Pre-existing, surfaced by the turn-timing final review: + `reports_stats.regularized_incomplete_beta` clamps an out-of-domain `x` + instead of raising.** Its docstring says "Raises ValueError outside that + domain — returning NaN would let a bad input render as a real-looking + statistic downstream", and it does raise for a non-finite `a`/`b`/`x` and for + a non-positive `a`/`b`. But the boundary branches are `if x <= 0.0: return + 0.0` / `if x >= 1.0: return 1.0`, so a NEGATIVE `x` or one above 1 silently + becomes a valid-looking probability — exactly the outcome the docstring says + it prevents. `x == 0.0` and `x == 1.0` are legitimately in the domain, so the + fix is to split the equality from the inequality, not to tighten the branch. + The internal Student-t callers construct an in-range `x`, so nothing ships + wrong today; the exposure is a future or external caller. NOT touched by the + timing work (the function is zero lines of its diff) and not a guardrail + candidate — a small real bug needing its own change. Caught in: the + turn-timing consolidation final review (gpt-5.6-sol). diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts index 05b33a4c3..58bbc00ab 100644 --- a/evalboard/lib/timing.ts +++ b/evalboard/lib/timing.ts @@ -201,7 +201,15 @@ export function toolExecutionMs(messages: MessageEvent[]): number { } } if (spans.length === 0) return 0; - const lo = Math.min(...spans.map(([s]) => s)); - const hi = Math.max(...spans.map(([, e]) => e)); + // Folded rather than `Math.min(...spans.map(…))`: the spread passes one + // ARGUMENT per span, so a long enough trace throws RangeError and the whole + // task page fails to render. The Python twin uses generator `min`/`max` and + // has no such ceiling; this keeps the two bounded the same way. + let lo = spans[0][0]; + let hi = spans[0][1]; + for (const [start, end] of spans) { + if (start < lo) lo = start; + if (end > hi) hi = end; + } return busyMs(spans, lo, hi); } diff --git a/src/coder_eval/timing.py b/src/coder_eval/timing.py index 788a77327..4ce7d219a 100644 --- a/src/coder_eval/timing.py +++ b/src/coder_eval/timing.py @@ -364,6 +364,23 @@ def main_thread_tool_spans( ] +#: How far a published window may sit from the span its own bounds describe. +#: +#: ONE MILLISECOND, which is the coarsest unit a field named ``_ms`` can +#: honestly be published in: a producer that records microsecond-precision +#: bounds and rounds its duration to whole milliseconds is within its rights, +#: and crashing its turns over 0.001 ms would be the guard relocating a defect +#: rather than removing one. The exposure this check is actually for is a +#: third-party agent registered through the ``coder_eval.plugins`` SPI, which is +#: exactly the producer most likely to round — so the tolerance has to admit it. +#: +#: It still catches everything it is for. The defect class is a reducer that +#: NARROWED or WIDENED a window without moving its bounds — subtracting its own +#: tool time, most plausibly — which is tens to thousands of milliseconds, three +#: to six orders of magnitude above this. +_WINDOW_TOLERANCE_MS = 1.0 + + def subtract_tool_time( messages: list[TranscriptMessage], spans: list[tuple[datetime, datetime]], @@ -459,10 +476,22 @@ def subtract_tool_time( raw_total = sum(raw for _, raw in members) # Nothing to apportion, and dividing by it is a ZeroDivisionError. A # group already at zero stays at zero. + # + # THE SKIP RUNS BEFORE THE CHECK BELOW, and that order is load-bearing + # rather than incidental. `close_window` clamps an inverted window — + # `now` before `mark`, two clocks disagreeing — to `0.0` while the + # bounds it writes still say `completed_at < started_at`, so `bounds_ms` + # is NEGATIVE and the equality fails. That is a measured inversion, the + # case `decompose_turn` deliberately clamps because both ends were + # observed; raising on it would kill turns on exactly the shape the + # clamp exists to tolerate. The cost is that a `0.0` published beside a + # POSITIVE window slips through — a shape no in-tree reducer produces, + # and one that reads downstream as "measured, and instant" rather than + # as a crashed turn. if raw_total <= 0: continue bounds_ms = (completed - started).total_seconds() * 1000.0 - if not math.isclose(raw_total, bounds_ms, rel_tol=1e-9, abs_tol=1e-6): + if not math.isclose(raw_total, bounds_ms, rel_tol=1e-9, abs_tol=_WINDOW_TOLERANCE_MS): raise ValueError( f"generation_duration_ms: a group of {len(members)} message(s) bounded " + f"{started} -> {completed} ({bounds_ms:.6f} ms) publishes {raw_total:.6f} ms of " @@ -479,7 +508,14 @@ def subtract_tool_time( for n, (index, raw) in enumerate(members): # The last member takes the remainder so the parts reconstruct the # group's net exactly, rather than drifting by the rounding. - share = net - assigned if n == len(members) - 1 else round(net * (raw / raw_total), 6) + # NOT rounded. The last member already takes the remainder, so the + # parts reconstruct the group's net exactly without it — while + # rounding each earlier share UP could push `assigned` past `net` + # and hand the last member a NEGATIVE duration. That needs a net of + # well under a microsecond (a window almost entirely covered by + # tool execution) and so had never been seen, but a negative + # generation is an invariant break, not a rounding artifact. + share = net - assigned if n == len(members) - 1 else net * (raw / raw_total) out[index] = out[index].model_copy(update={"generation_duration_ms": share}) assigned += share return out diff --git a/tests/test_event_collector.py b/tests/test_event_collector.py index ba2ced513..9755120a5 100644 --- a/tests/test_event_collector.py +++ b/tests/test_event_collector.py @@ -932,18 +932,42 @@ def test_a_split_whose_parts_sum_to_the_wrong_total_raises(self): with pytest.raises(ValueError): subtract_tool_time([self._msg(0, 1000, 400.0, message_id="m"), self._msg(0, 1000, 400.0)], []) - def test_a_zero_group_is_skipped_before_the_check_runs(self): - """The `raw_total <= 0` skip runs FIRST, and must keep running first. - - A window measured at zero between IDENTICAL bounds would satisfy the - equality anyway; the case that needs the order is a `0.0` published - beside bounds that are not identical, which is a shape the tree - tolerates today. Raising on it would turn a tolerated record into a - killed turn, so the bounds here are deliberately 500 ms apart. + def test_a_clamped_inversion_is_skipped_before_the_check_runs(self): + """The `raw_total <= 0` skip runs FIRST, and that order is load-bearing. + + `close_window` clamps an inverted window — `now` before `mark`, two + clocks disagreeing — to `0.0` while the bounds it writes still say + `completed_at < started_at`. `bounds_ms` is then NEGATIVE and the + equality fails, so checking first would kill the turn on exactly the + measured inversion `decompose_turn` deliberately clamps because both + ends were observed. """ - out = subtract_tool_time([self._msg(0, 500, 0.0)], []) + out = subtract_tool_time([self._msg(500, 0, 0.0)], []) assert out[0].generation_duration_ms == 0.0 + def test_a_duration_rounded_to_whole_milliseconds_is_admitted(self): + """The tolerance has to fit the producer it exists for. + + A third-party agent registered through the `coder_eval.plugins` SPI is + the exposure this check is actually for, and is the producer most + likely to record microsecond bounds while publishing a duration rounded + to whole milliseconds. Crashing its turns over 0.4 ms would relocate a + defect rather than remove one. + """ + started = self._at(0) + completed = started + timedelta(microseconds=1000 * 1000 + 400) # 1000.4 ms + message = AssistantMessage(started_at=started, completed_at=completed, generation_duration_ms=1000.0) + assert subtract_tool_time([message], [])[0].generation_duration_ms == pytest.approx(1000.0) + + def test_a_narrowing_larger_than_the_tolerance_still_raises(self): + """The control: widening for rounding must not admit the defect class. + + A reducer subtracting its own tool time narrows a window by tens to + thousands of milliseconds, orders of magnitude past the tolerance. + """ + with pytest.raises(ValueError): + subtract_tool_time([self._msg(0, 1000, 998.0)], []) + def test_an_unmeasured_window_never_reaches_the_check(self): out = subtract_tool_time([self._msg(0, 5000, None)], []) assert out[0].generation_duration_ms is None From 2c3cc10cc65b4b3038e5fcbb047556add5da0d06 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 14:04:21 -0700 Subject: [PATCH 50/54] test(harness): no test may read the pinned timing corpus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `scripts/timing/corpus/` is stale by design: two of its five records preserve defects the live code no longer has — claude-code reconciling at -481 ms, and a `0.0` head on two harnesses. Its README says so, and says that re-recording it after a change would destroy the only thing it is good for. The danger is that a test pointed at it looks entirely reasonable: a green assertion over real recorded numbers, quietly pinning a fixed defect as expected behaviour. Moving it out of `tests/_fixtures/` removed the invitation; this removes the possibility. Until now the rule lived only in prose, which is the shape this repo converts to a check. Three arms, because a guard on a directory that has moved guards nothing: the corpus still exists where the rule says, no test module references it, and the README still states the rule so the prose and the check cannot drift apart. Mutation-verified — a probe module naming the path fails it by name. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- tests/test_custom_lint.py | 51 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 4da38090b..735d5bcf3 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -45,6 +45,57 @@ def test_no_violations(rule_class: type) -> None: ) +@pytest.mark.lint +class TestNoTestReadsThePinnedTimingCorpus: + """`scripts/timing/corpus/` is stale BY DESIGN, so no test may read it. + + Two of its five records deliberately preserve defects the live code no + longer has — claude-code reconciling at -481 ms (the pre-subtraction + defect) and a `0.0` head on two harnesses (the clamped inversion). Its + README says outright that it exists to carry wall-clock MAGNITUDES for + `scripts/timing/decompose_run.py`, and that re-recording it after a change + would destroy the only thing it is good for. + + A test pointed at it would pin a fixed defect as expected behaviour, and it + would look entirely reasonable while doing so — a green assertion over real + recorded numbers. Moving it out of `tests/_fixtures/` removes the + invitation; this removes the possibility. The rule lived only in the + README, which is the shape this repo converts to a check. + """ + + #: The rule is about `tests/` reading the corpus. `decompose_run.py` is its + #: intended reader and the README documents that invocation, so `scripts/` + #: and the README itself are out of scope by construction — this only walks + #: the test tree. + _CORPUS = "scripts/timing/corpus" + + def test_the_corpus_exists_where_the_rule_says_it_does(self): + """A rule guarding a directory that has moved guards nothing.""" + corpus = Path(__file__).parents[1] / self._CORPUS + assert corpus.is_dir(), f"{self._CORPUS} is gone; move this rule with it or retire it" + assert list(corpus.glob("*.json")), f"{self._CORPUS} holds no records" + + def test_no_test_module_references_it(self): + tests_root = Path(__file__).parent + offenders = [ + str(path.relative_to(tests_root)) + for path in tests_root.rglob("*.py") + if path != Path(__file__) and self._CORPUS in path.read_text(encoding="utf-8") + ] + assert not offenders, ( + f"{offenders} reads {self._CORPUS}, which is stale by design: two of its records " + "preserve defects the live code no longer has, so an assertion over them pins a " + "fixed defect as expected behaviour. Point the test at a fixture that moves with " + "the code — tests/_fixtures/golden_streams/ for a replay, or " + "tests/test_timing_identity_contract.py for a magnitude." + ) + + def test_the_readme_still_states_the_rule(self): + """The prose and the check have to agree, or one of them is wrong.""" + readme = (Path(__file__).parents[1] / self._CORPUS / "README.md").read_text(encoding="utf-8") + assert "NO TEST MAY READ THIS DIRECTORY" in readme + + @pytest.mark.lint class TestCE016NoComputedTokenUsageKwargs: """CE016 fires on TokenUsage(input_tokens=/total_tokens=) but not elsewhere.""" From 159047ac9c02173ce182726930a9c920cecac715 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 14:19:48 -0700 Subject: [PATCH 51/54] =?UTF-8?q?fix:=20review=20pass=20B=20=E2=80=94=20th?= =?UTF-8?q?e=20tool=20bucket's=20dash=20survives=20the=20language=20bounda?= =?UTF-8?q?ry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The None-vs-0.0 contract this branch exists to enforce broke at exactly the seam it introduced. `toolExecutionMs` returns `number`, never null, so the new `storedToolMs ?? toolExecutionMs(mainThread)` fallback turned "no bounded span was recorded" into `0ms` — a measurement claim — while every Python surface rendered a dash for the same run. Three comments asserted the two sides "agree by construction", which was true of the bounded-spans FILTER and not of this. It is not a legacy-only case, which is what makes it worth a fix rather than a note. A turn that simply ran no tools produces the same empty span list, and so does one whose calls were all TIMED BUT UNBOUNDED — the historical-codex and out-of-tree `delegate-sdk` population this branch's own comments quantify at ~8 h. Claiming those took no time is the one thing certainly false about them. `measuredToolExecutionMs` is the missing layer, and it mirrors the Python split rather than inventing one: `union_ms` returns `0.0` for an empty span list because that is what a union of nothing is, and the shared corpus pins both sides on exactly that; the None decision sits one layer up, where `main_thread_tool_spans` returns a list and its caller turns an empty one into `None`. `toolExecutionMs` is therefore untouched and the corpus still pins it. The per-row EXEC cell moves to the same helper, replacing a `durationMs != null` guard that asked whether the harness TIMED anything rather than whether it BOUNDED anything. Separately, `decompose_run.py` checked its two new breach classes AFTER the no-gateable-turns arm, so a corpus whose turns were all under `--min-turn-ms` printed a real validation failure to stderr and exited 0 — the "measured nothing, reported success" shape that arm exists to refuse. Neither class is a residual question, so neither may depend on a turn being long enough to gate on. CI always passes `--max-residual-pct`, so it was not reachable there; the standalone invocation the script's own header documents is where it bit. `main()`'s exit code now has end-to-end tests, including the arm-order case directly. It had none: every other property of that script was asserted on its helpers, and the exit code is the only thing CI actually reads. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01LkF1Up5DfjWR7HsyFpVAZr --- .../__tests__/message-timeline.test.tsx | 24 ++++++- .../app/runs/[id]/[...task]/_sections.tsx | 16 +++-- .../lib/__tests__/no-zero-coalesce.test.ts | 4 ++ evalboard/lib/timing.ts | 21 ++++++ scripts/timing/decompose_run.py | 12 ++-- tests/test_timing_close_window.py | 66 +++++++++++++++++++ 6 files changed, 131 insertions(+), 12 deletions(-) diff --git a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx index cbb939aa7..e8a3b7095 100644 --- a/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx +++ b/evalboard/app/runs/[id]/[...task]/__tests__/message-timeline.test.tsx @@ -692,8 +692,15 @@ describe("MessageTimelineSection — Unaccounted cell", () => { }, ], }); - expect(cell("Tool exec").textContent).toBe("0ms"); - // 10s − 4s generation − 0s tool exec: the second is the point. + // A DASH, not "0ms". No bounded span was recorded, so nothing measured + // the tool time — and `0ms` would claim it was measured and instant, + // which is the one thing certainly false about a call the harness DID + // time. Every Python surface renders a dash for the same run; this cell + // used to disagree with them, which is the defect class this branch + // exists to remove, relocated across the language boundary. + expect(cell("Tool exec").textContent).toBe("—"); + // The time is not lost: an unmeasured bucket is subtracted as 0, so it + // stays IN the residual instead of vanishing. expect(cell("Unaccounted").textContent).toBe("6.0s (60%)"); }); @@ -964,6 +971,19 @@ describe("MessageTimelineSection — Startup and Teardown cells", () => { expect(cell("Tool exec").textContent).toBe("1.0s"); }); + test("the fallback renders a dash, not 0ms, when nothing was bounded", () => { + // Both halves of the None-vs-0.0 contract have to survive the fallback, + // or a modern run that simply ran no tools reads as "measured, and + // instant" on this surface and as a dash on every Python one. + render( + , + ); + expect(cell("Tool exec").textContent).toBe("—"); + }); + test("each bucket says what it measures and that it is not decomposed", () => { renderStrip({ taskDurationSeconds: 10, diff --git a/evalboard/app/runs/[id]/[...task]/_sections.tsx b/evalboard/app/runs/[id]/[...task]/_sections.tsx index 0b4406c7d..2cced0936 100644 --- a/evalboard/app/runs/[id]/[...task]/_sections.tsx +++ b/evalboard/app/runs/[id]/[...task]/_sections.tsx @@ -13,7 +13,7 @@ import type { TokenTotals, ToolCall, } from "@/lib/runs"; -import { toolExecutionMs } from "@/lib/timing"; +import { measuredToolExecutionMs, toolExecutionMs } from "@/lib/timing"; import { type PerMessageImpact, buildThinkingModel, @@ -421,7 +421,7 @@ export function MessageTimelineSection({ // computed from the messages — the reconciliation entry exists so a // consumer sums that stream rather than reading a separate aggregate. The // mixed sourcing is intentional; see `sumTurnBuckets` in lib/runs.ts. - const toolExecMs = storedToolMs ?? toolExecutionMs(mainThread); + const toolExecMs = storedToolMs ?? measuredToolExecutionMs(mainThread); const slowGen = mainThread.filter( (m) => (m.generationMs ?? 0) >= SLOW_GEN_MS, ).length; @@ -451,7 +451,7 @@ export function MessageTimelineSection({ taskMs != null ? taskMs - totalGenMs - - toolExecMs - + (toolExecMs ?? 0) - (harnessStartupMs ?? 0) - (harnessTeardownMs ?? 0) - (setupMs ?? 0) - @@ -1560,8 +1560,12 @@ function MessageRow({ // `toolExecutionMs` was changed to union and this line was not. Expand the // row to see each call's own wall clock: sequential calls still add up to // this number, concurrent ones deliberately do not. - const execMs = toolExecutionMs([m]); - const hasExec = m.toolUses.some((t) => t.durationMs != null); + // `measured…`, so a row whose calls were TIMED BUT UNBOUNDED reads "—" + // rather than "0ms". Under the union policy such a call contributes to no + // bucket, and claiming it took no time is the one thing that is certainly + // false. This replaces a `durationMs != null` guard, which asked whether + // the harness timed anything rather than whether it bounded anything. + const execMs = measuredToolExecutionMs([m]); // Render full body only when something more than the summary exists. const hasBody = m.toolUses.length > 0 || @@ -1606,7 +1610,7 @@ function MessageRow({ : "text-gray-600") } > - {hasExec ? fmtMs(execMs) : "—"} + {fmtMs(execMs)} ([ "s + m.toolUses.filter((t) => (t.durationMs ?? 0) >= SLOW_TOOL_MS).length,", "A threshold comparison: an untimed call is not a slow call, so 0 answers the question asked.", ], + [ + "(toolExecMs ?? 0) -", + "The residual's tool half, and the reason the cell itself now renders a dash: a turn with no BOUNDED span measured no tool time, so its time belongs IN the residual rather than being subtracted as a zero.", + ], [ "(harnessStartupMs ?? 0) -", "The residual. Subtracting only what was measured is the whole point; an unmeasured head leaves its time IN the residual rather than silently claiming it.", diff --git a/evalboard/lib/timing.ts b/evalboard/lib/timing.ts index 58bbc00ab..db23a1c1d 100644 --- a/evalboard/lib/timing.ts +++ b/evalboard/lib/timing.ts @@ -191,6 +191,27 @@ export function busyMs( // Tool exec and into Unaccounted. Going forward the only harness reporting a // bare duration is the out-of-tree `delegate-sdk`; see // docs/agents/HARNESS_PARITY.md. +// The same union, but `null` when NOTHING bounded was recorded — the direct +// twin of `reports_stats._turn_tool_union_ms`, and the one a display cell wants. +// +// `toolExecutionMs` above returns `0` for an empty span list because that is +// what a UNION of nothing is, and what `coder_eval.timing.union_ms` returns; +// the shared corpus pins both sides on exactly that. The None-vs-0.0 decision +// sits one layer up on the Python side too (`main_thread_tool_spans` returns a +// list, and its caller turns an empty one into `None`), and this is that layer. +// +// The distinction is not academic. A turn that ran no tools, and a turn whose +// tools were all TIMED BUT UNBOUNDED — the historical-codex and out-of-tree +// `delegate-sdk` population — both produce an empty span list. Rendering `0ms` +// there claims the tools were measured and took no time, while every Python +// surface renders a dash for the same run. +export function measuredToolExecutionMs(messages: MessageEvent[]): number | null { + const anyBounded = messages.some((m) => + m.toolUses.some((t) => t.execStartMs != null && t.execEndMs != null), + ); + return anyBounded ? toolExecutionMs(messages) : null; +} + export function toolExecutionMs(messages: MessageEvent[]): number { const spans: [number, number][] = []; for (const m of messages) { diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index bc2b11e94..f60f98c6b 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -309,16 +309,20 @@ def main(argv: list[str]) -> int: for harness, path, index, detail in union_breaches: print(f" {harness:<14} {path} turn {index}: {detail}", file=sys.stderr) + # BEFORE the no-gateable-turns arm below, not after. Neither of these is a + # residual question, so neither depends on a turn being long enough to gate + # on — and a corpus whose turns are all under --min-turn-ms would otherwise + # print a real validation failure to stderr and exit 0, which is the + # "measured nothing, reported success" shape the arm below exists to refuse. + if union_breaches or invalid: + return 1 + if not gateable_total: # A gate that passes because it measured nothing is the exact failure # this script exists to remove, so it only passes when none was asked for. print("no gateable turns", file=sys.stderr) return 1 if args.max_residual_pct is not None else 0 - if union_breaches or invalid: - # Independent of --max-residual-pct: neither is a residual question, and - # a disagreement about a stored bucket is exactly what a gate is for. - return 1 if args.max_residual_pct is None: return 0 if not breaches: diff --git a/tests/test_timing_close_window.py b/tests/test_timing_close_window.py index 0b3d56fb8..e69c11ba8 100644 --- a/tests/test_timing_close_window.py +++ b/tests/test_timing_close_window.py @@ -7,6 +7,7 @@ """ import importlib.util +import json from datetime import UTC, datetime, timedelta from pathlib import Path @@ -531,3 +532,68 @@ def test_a_turn_missing_messages_or_commands_produces_an_empty_span_set(self): assert main_thread_tool_spans(record.messages, record.commands) == [] assert _tool_union_ms(turn) == 0.0 assert _load_decompose_run()._tool_ms(turn) == 0.0 + + +class TestTheLiveGatesExitCode: + """`main()`'s return value, driven end to end. + + Everything else about this script is asserted on its helpers. The exit code + is what CI actually reads, and the ORDER of its arms is load-bearing: a + validation failure and a stored-union disagreement are not residual + questions, so neither may depend on a turn being long enough to gate on. + They were once checked after the no-gateable-turns arm, where a corpus of + only-short turns printed a real breach to stderr and exited 0. + """ + + @staticmethod + def _record(tmp_path, *, stored: float | None = None, valid: bool = True) -> str: + turn = _turn( + duration_seconds=10.0, + commands=[_command("t1", _at(1000), _at(1500))], + messages=[ + { + "role": "assistant", + "started_at": _at(0).isoformat(), + "completed_at": _at(1000).isoformat(), + "generation_duration_ms": 1000.0, + } + ], + ) + turn["harness_startup_ms"] = 0.0 + turn["harness_teardown_ms"] = 8500.0 + if stored is not None: + turn["tool_union_ms"] = stored + if not valid: + # Below what `TurnRecord` requires — the shape the whole run history + # measured zero of, so a non-zero count is news rather than noise. + del turn["user_input"] + path = tmp_path / "task.json" + path.write_text(json.dumps({"agent_type": "pi", "iterations": [turn]}), encoding="utf-8") + return str(path) + + def test_a_clean_record_exits_zero(self, tmp_path): + main = _load_decompose_run().main + assert main([self._record(tmp_path, stored=500.0), "--min-turn-ms", "0"]) == 0 + + def test_a_union_disagreement_exits_one_even_with_no_gate_asked_for(self, tmp_path): + main = _load_decompose_run().main + assert main([self._record(tmp_path, stored=999.0), "--min-turn-ms", "0"]) == 1 + + def test_a_union_disagreement_exits_one_even_when_every_turn_is_too_short_to_gate(self, tmp_path): + """The arm-order regression, asserted directly. + + The turn is excluded from the share columns and the gate, so the + no-gateable-turns arm fires — and used to return before the breach was + ever consulted. + """ + main = _load_decompose_run().main + assert main([self._record(tmp_path, stored=999.0), "--min-turn-ms", "999999999"]) == 1 + + def test_an_invalid_record_exits_one(self, tmp_path): + main = _load_decompose_run().main + assert main([self._record(tmp_path, valid=False), "--min-turn-ms", "0"]) == 1 + + def test_a_legacy_record_without_the_field_exits_zero(self, tmp_path): + """The cross-check SKIPS rather than failing — the common case on disk.""" + main = _load_decompose_run().main + assert main([self._record(tmp_path), "--min-turn-ms", "0"]) == 0 From 94ac71b56e64f51892349c3c3a8fc6972c06eced Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sat, 12 Sep 2026 21:34:16 -0700 Subject: [PATCH 52/54] fix(timing): claude-code's windows tile across a tool result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `on_user_message` reset the generation mark to `self.clock.now()`, so the next window opened when the tool RESULT arrived instead of tiling from the previous emission's close. Everything in between — SDK transport, CLI processing, next-request dispatch — fell into no bucket at all, and the four-bucket identity stopped closing. The live stream delivers TWO user messages per tool call, and the mark was reset on each, so the window opened at the LAST one. Traced on `tasks/dataset_example.yaml`: msg2 (issues the Write) closes 27.773889 -> 28.067824 user message #1 28.080804 <- mark reset here user message #2 30.009645 <- and again, 1.93s later msg3 window opened at 30.009645 (should be 28.067824) 1.94 s lost from an 11.7 s turn. CI's residual gate has been failing on exactly these two rows (16.668% and 16.325% on the runner, 21-28% locally). Same task after the fix: 0.02%, worst turn 4.5 ms — the known `turn_start_time`-vs-`AgentStartEvent` baseline and nothing else. The tool's own interval is not double-counted: it is a separate bucket and `subtract_tool_time` clips the tool union out of every window it overlaps, once, for all five harnesses. That central subtraction is precisely what lets the reducer leave its mark alone — the same rule pi follows with `gen_mark`, and the one pi was explicitly fixed for. WHY THIS SURVIVED, which is worth more than the one-line fix. Three ways to write a test for it cannot fail, and I wrote two of them before getting one that does: 1. A tool-heavy shape. Three concurrent `sleep 3` calls make the tool union absorb the interval; every live probe I ran read 0.05% and I concluded the harness was healthy. The defect needs a FAST tool. 2. A single tool result. claude-code reconstructs `execution_started_at` by subtracting the measured duration from the resolve instant, so with one message the discarded interval and the tool's own span are the SAME milliseconds — `subtract_tool_time` removes them either way and the identity closes with or without the bug. This is why the existing `_claude_turn` case passed throughout. 3. A duplicate tool RESULT as the second message. That re-resolves the call and stretches the tool span over the very interval being probed. `test_a_slow_tool_result_round_trip_is_not_lost` scripts the shape that discriminates: a 20 ms tool, then a second user message carrying no tool result 2 s later. Mutation-checked — restoring the reset fails that case by exactly -2000 ms and leaves the other seven green, which is the production situation reproduced in the suite. No golden regeneration: `_scrub.py::SCRUB_KEYS` masks every timing value, so the corpus cannot see this class of change. That is a known property of the goldens, not an oversight, and it is the reason the ms-exact identity contract exists alongside them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01FpDo37ypvLjLiWXFsEkg6k --- docs/agents/HARNESS_PARITY.md | 23 ++++ src/coder_eval/agents/claude_code_agent.py | 23 +++- tests/test_timing_identity_contract.py | 118 ++++++++++++++++++++- 3 files changed, 158 insertions(+), 6 deletions(-) diff --git a/docs/agents/HARNESS_PARITY.md b/docs/agents/HARNESS_PARITY.md index 4b1cabd1d..b3501e081 100644 --- a/docs/agents/HARNESS_PARITY.md +++ b/docs/agents/HARNESS_PARITY.md @@ -262,6 +262,29 @@ figures in the table above are means of six live `tasks/hello_date` turns per harness and move with CLI cache warmth, so read their ORDER OF MAGNITUDE, not the digits. +**Windows tile ACROSS a tool result, on every harness.** claude-code used to +reset its generation mark when the tool-result `UserMessage` arrived, so the +next window opened at the result rather than tiling from the previous +emission. Everything in between — SDK transport, CLI processing, next-request +dispatch — fell into no bucket. The live stream delivers TWO user messages per +tool call, ~2 s apart, and the mark was reset on each, so the window opened at +the LAST one: measured on `tasks/dataset_example.yaml` at 1.94 s lost from an +11.7 s turn, 16-28% of wall clock, and it is what failed CI's residual gate. +The mark is now left where `on_assistant_message` put it and the tool's own +interval is removed centrally by `subtract_tool_time`, exactly as pi does with +`gen_mark`. Same task after the fix: 0.02%. + +Why it survived so long is the more useful half. A tool-heavy shape cannot see +it — three concurrent `sleep 3` calls make the tool union absorb the interval +and the residual reads 0.05%. Neither can a single-tool-result fixture: +claude-code reconstructs `execution_started_at` by subtracting the measured +duration from the resolve instant, so with one message the discarded interval +and the tool's own span are the SAME milliseconds and the identity closes +either way. It takes a FAST tool plus a SECOND user message carrying no tool +result to separate them, which is what +`test_a_slow_tool_result_round_trip_is_not_lost` scripts. Two earlier drafts of +that test could not fail. + **Four turn buckets, two task buckets — and they are different scopes.** The four above tile ONE TURN and their identity (`head + Σgeneration + UNION(tool) + tail == the turn's span`) is asserted to the millisecond by diff --git a/src/coder_eval/agents/claude_code_agent.py b/src/coder_eval/agents/claude_code_agent.py index 4e792b0a8..cfcd5843d 100644 --- a/src/coder_eval/agents/claude_code_agent.py +++ b/src/coder_eval/agents/claude_code_agent.py @@ -597,7 +597,28 @@ def on_user_message(self, message: Message) -> None: """Process tool results (and a sub-agent's terminal generation) from a tool-result UserMessage. The sub-agent message is appended BEFORE the tool-result loop — its position in ``sdk_messages`` is observable.""" - self.last_event_wall = self.clock.now() + # The generation mark is DELIBERATELY NOT advanced here. It used to be + # reset to `self.clock.now()`, which opened the next window at the + # instant the tool RESULT arrived rather than tiling it from the + # previous window's close — so everything between the tool finishing + # and its result reaching this handler (SDK transport, CLI processing, + # next-request dispatch) fell into no bucket at all. Measured on + # `tasks/dataset_example.yaml`: a 21.5 ms `Write` followed by a 2511.7 ms + # round trip, which is 21% of an 11.7 s turn accounted to nothing and + # the reason CI's residual gate failed on that task while a + # `sleep`-heavy probe read 0.05%. A tool-heavy shape cannot see this: + # the tool union absorbs the interval. A fast tool leaves it exposed. + # + # Leaving the mark where `on_assistant_message` put it makes the next + # window run from the previous emission's arrival, so the windows tile + # the turn contiguously — the same rule pi follows with `gen_mark`, and + # the one pi was explicitly fixed for. + # + # The tool's OWN interval is not double-counted by this: it is a + # separate bucket, and `streaming/collector.py::subtract_tool_time` + # clips the tool union out of every window it overlaps, once, for all + # five harnesses. That is exactly why the mark can be left alone here — + # the reducer no longer has to carve the tool out of its own windows. sub_msg = self._agent._synthesize_subagent_terminal_message(message, self.sdk_model_used) if sub_msg is not None: diff --git a/tests/test_timing_identity_contract.py b/tests/test_timing_identity_contract.py index 2dd07ad90..382157324 100644 --- a/tests/test_timing_identity_contract.py +++ b/tests/test_timing_identity_contract.py @@ -449,11 +449,17 @@ def _claude_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: `tests/test_agent_telemetry.py`; here it shows up as the windows still tiling. - Note where its windows do NOT tile: the tool result resets both marks, so - the interval between the emission that ISSUED the call and the result is - left outside every window. That gap is the tool's own execution, which is - exactly what the tool bucket claims — which is why the identity still - closes to the millisecond. + Its windows TILE across the tool result, and this case only proved that by + accident until the reducer was fixed. The mark used to be reset when the + result arrived, so the interval between the emission that ISSUED the call + and the result landed in no bucket. Here that interval IS the tool's + execution exactly — the case scripts the result at the instant the tool + ends — so the tool bucket happened to claim the same milliseconds and the + identity closed anyway. On a real turn the two differ: a 21.5 ms `Write` + can be followed by a 2.5 s round trip, and 21% of the turn goes missing. + `test_a_slow_tool_result_round_trip_is_not_lost` is the case that + discriminates; this one deliberately keeps the coincident shape so the two + read as a pair. """ from coder_eval.agents import claude_code_agent as claude_module from coder_eval.agents.claude_code_agent import ClaudeCodeAgent, _ClaudeTurnState @@ -517,6 +523,98 @@ def _monotonic() -> float: return Turn(started_ms=0.0, ended_ms=3000.0, messages=list(state.sdk_messages), commands=commands) +def _claude_slow_result_turn(monkeypatch: pytest.MonkeyPatch) -> Turn: + """A FAST tool followed by a SLOW result round trip — the shape that hid a defect. + + ``_claude_turn`` above scripts the tool result at the instant the tool + finishes, so the un-tiled interval and the tool's own span were the same + milliseconds and the identity closed even while the mark was being reset. + Every live probe had the same blind spot from the other direction: three + concurrent ``sleep 3`` calls make the tool union so large that the round + trip rounds away (measured: 0.05% residual). + + Here the tool runs for 20 ms and its result takes 2000 ms to come back, + which is `tasks/dataset_example.yaml` — the task CI actually runs, where a + 21.5 ms ``Write`` met a 2511.7 ms round trip and 21% of the turn was + accounted to nothing. The identity closing here is the whole point: the + window after the result must tile from the previous emission, not open + when the result lands. + """ + from coder_eval.agents import claude_code_agent as claude_module + from coder_eval.agents.claude_code_agent import ClaudeCodeAgent, _ClaudeTurnState + from coder_eval.streaming.events import AgentEndStatus as _AgentEndStatus + from tests._fixtures.golden_streams.claude_fixtures import AssistantMessage as SdkAssistantMessage + from tests._fixtures.golden_streams.claude_fixtures import ToolUseBlock, UserMessage, message_start + + clock = _InjectedClock() + + def _monotonic() -> float: + return clock.at_ms / 1000.0 + + monkeypatch.setattr(claude_module, "time", SimpleNamespace(monotonic=_monotonic)) + + agent = ClaudeCodeAgent(parse_agent_config(type=AgentKind.CLAUDE_CODE, permission_mode="acceptEdits")) + collector = EventCollector() + commands: list[CommandTelemetry] = [] + + clock.at_ms = 200 + state = _ClaudeTurnState( + agent, + emit=CompositeStreamCallback( + [ + collector, + SimpleNamespace(on_event=lambda e: commands.append(e.tool) if isinstance(e, ToolEndEvent) else None), + ] + ), + collector=collector, + task_id="t", + user_input="go", + iteration=1, + max_turns=None, + log=agent._log, + turn_start_time=_monotonic(), + deadline=None, + clock=clock, + ) + + clock.at_ms = 500 + state.on_stream_event(message_start("m1")) + clock.at_ms = 980 + state.on_assistant_message( + SdkAssistantMessage( + [ToolUseBlock("c1", "Write", {"file_path": "out.txt"})], + usage={"input_tokens": 10, "output_tokens": 5}, + message_id="m1", + ) + ) + # The tool itself is 20 ms. What follows is the shape a live turn actually + # has, traced off `tasks/dataset_example.yaml`: the SDK delivers TWO user + # messages, the second ~2 s after the first. The old code reset the mark on + # each, so the next window opened at the LAST one and that 2 s vanished. + # + # One user message is not enough to catch it, and that is exactly why this + # shipped: claude-code reconstructs `execution_started_at` by subtracting + # the measured duration from the resolve instant, so with a single message + # the discarded interval and the tool's own span are the SAME milliseconds + # — `subtract_tool_time` removes them either way and the identity closes + # with or without the bug. The second message is what separates them. + clock.at_ms = 1000 + state.on_user_message(UserMessage("c1", False, "written")) + clock.at_ms = 3000 + # The second one carries NO tool-result block, which is what the live + # stream does — the traced turn kept its 21.5 ms Write span across it. A + # duplicate RESULT would instead re-resolve the call and stretch the tool + # span over the very interval this case exists to expose, which is a third + # way to write a test that cannot fail. + state.on_user_message(SimpleNamespace(content=[], tool_use_result=None)) + state.on_stream_event(message_start("m2")) + clock.at_ms = 3400 + state.on_assistant_message(SdkAssistantMessage([], usage={"input_tokens": 10, "output_tokens": 5}, message_id="m2")) + state.finalize(_AgentEndStatus.COMPLETED) + + return Turn(started_ms=0.0, ended_ms=3800.0, messages=list(state.sdk_messages), commands=commands) + + # -------------------------------------------------------------------------- # The contract # -------------------------------------------------------------------------- @@ -542,6 +640,16 @@ def test_claude_code_buckets_tile_the_turn(monkeypatch: pytest.MonkeyPatch): assert_identity_closes(_claude_turn(monkeypatch)) +def test_a_slow_tool_result_round_trip_is_not_lost(monkeypatch: pytest.MonkeyPatch): + """The discriminating case: a fast tool whose result takes 2 s to come back. + + Reverting the fix (re-adding `last_event_wall = self.clock.now()` to + `on_user_message`) fails THIS and leaves every other case in the file + green, which is exactly what happened in production. + """ + assert_identity_closes(_claude_slow_result_turn(monkeypatch)) + + def test_every_built_in_harness_has_a_case(): """A sensor that silently covers four of five is worse than one naming the gap. From b84153792db4650e00326c8cdc79020a0abcba6f Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sun, 13 Sep 2026 08:48:37 -0700 Subject: [PATCH 53/54] chore(timing): decompose_run's dead datetime import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unused since it landed in 5237eb4d. `scripts/` sits outside the Makefile's LINT_PATHS — which this file's own docstring already notes — so ruff never looked at it and CodeQL was the first thing to say so, as a `py/unused-import` alert that turned the PR's CodeQL check red. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Mgy33fwSvD5dPx8qD2bXJe --- scripts/timing/decompose_run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/timing/decompose_run.py b/scripts/timing/decompose_run.py index f60f98c6b..633914c6a 100644 --- a/scripts/timing/decompose_run.py +++ b/scripts/timing/decompose_run.py @@ -31,7 +31,6 @@ import statistics import sys from collections import defaultdict -from datetime import datetime from pathlib import Path from pydantic import ValidationError From bfec0dd00c50b549a62f97ad967e2f42d10c1cf2 Mon Sep 17 00:00:00 2001 From: uipreliga Date: Sun, 13 Sep 2026 08:48:53 -0700 Subject: [PATCH 54/54] =?UTF-8?q?test(lint):=20CE058=20form=206=20?= =?UTF-8?q?=E2=80=94=20the=20zero=20its=20guard=20does=20not=20vouch=20for?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CE058 form 4 fires on `if x.duration_ms is None: x.duration_ms = 0.0`: it keys on the `if` test naming a TIMING attribute. The live `_finalize_commands` defect matched only because it happened to spell the inner guard that way. The assignment sat inside an enclosing `if cmd.result_status is None:` block, and writing the literal under THAT guard instead — which reads just as naturally and books the identical lie — was invisible to all five forms. Verified by mutation: `cmd.duration_ms = 0.0` in that block passed all 626 lint tests. So the rule was one plausible refactor away from silent on the exact defect it was written for. A guard is evidence about a value only when the guard names the value; with no guard there is no evidence at all, which is strictly worse and must not be the case the rule misses. Form 6 is a plain assignment of a ZERO literal to a timing-named target, and the narrowing is the point. Under `is None` the guard PROVES the value was never measured, so form 4 rejects any invented number. A bare assignment proves nothing — `cmd.duration_ms = elapsed_ms` is how a real one is written and a literal 1234.0 is a plausible factory or replay — so only the placeholder zero is the tell. Same narrowing form 1 already makes on constructor keywords. Forms 4 and 6 overlap on the zero case, so form 4 registers what it flags and form 6 skips it. Ordering-safe rather than lucky: `visit_If` runs its check before `generic_visit` descends into the body. One defect, one violation — otherwise a `# noqa` silences half of it. `test_allows_a_guard_on_a_different_receiver` changed rather than being deleted. Its property — form 4 keys on the RECEIVER, so a guard about `a` must not vouch for `b` — is still real and now uses a non-zero literal to stay a form-4 test. The zero spelling gets its own case asserting form 6 claims it, because `b.duration_ms = 0.0` under a guard naming `a` is an ungrounded zero. Clean on the tree today: zero-literal assignments to a timing-named target across all of src/coder_eval/ = 0. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Mgy33fwSvD5dPx8qD2bXJe --- tests/lint/rules/ce058_no_timing_literal.py | 62 +++++++++++++++++-- tests/test_custom_lint.py | 66 ++++++++++++++++++++- 2 files changed, 121 insertions(+), 7 deletions(-) diff --git a/tests/lint/rules/ce058_no_timing_literal.py b/tests/lint/rules/ce058_no_timing_literal.py index 5647811d6..d1364c5ed 100644 --- a/tests/lint/rules/ce058_no_timing_literal.py +++ b/tests/lint/rules/ce058_no_timing_literal.py @@ -25,7 +25,7 @@ down to nothing by the tool execution inside it, or a clamped inversion where both ends really were observed — so the two values must stay distinguishable. -Five syntactic forms, one invariant, one id — the shapes the codebase actually +Six syntactic forms, one invariant, one id — the shapes the codebase actually produced: 1. a ``0`` / ``0.0`` constructor keyword on one of the telemetry constructors @@ -39,12 +39,43 @@ 5. ``model_copy(update={"duration_ms": 0.0})`` — a keyword rule is blind to a dict, and the dict is how ``CommandTelemetry.duration_ms`` is actually written on the Antigravity DONE path, so forms 1-4 alone would have left - the next author's ``"duration_ms": 0.0`` in that idiom unguarded. - -BLIND SPOT worth knowing: form 1 keys on the callee's spelling, so + the next author's ``"duration_ms": 0.0`` in that idiom unguarded; +6. ``cmd.duration_ms = 0.0`` as a PLAIN assignment — form 4 without the + ``is None`` guard, or under a guard that tests something else. + +Form 6 exists because form 4 was passing the live defect by coincidence. Form 4 +keys on the ``if`` test naming a timing attribute, and the shipped +``_finalize_commands`` bug happened to spell it that way +(``if cmd.duration_ms is None:``) — but the assignment sat inside an outer +``if cmd.result_status is None:`` block, and rewriting it to set the literal +under THAT guard instead, which reads just as naturally and books the identical +lie, was invisible to all five earlier forms. The rule was one plausible +refactor away from silent. A guard is only evidence about the value when the +guard names the value; without one there is no evidence at all, which is +strictly worse and must not be the case the rule misses. + +It uses ``_zero_literal``, not form 4's broader ``_numeric_literal``, and the +asymmetry is the point. Under ``if x is None`` the guard PROVES the value was +never measured, so any invented number is a defect. A bare assignment proves +nothing: ``cmd.duration_ms = elapsed_ms`` is how a measured value is written, +and a literal ``1234.0`` is a legitimate test factory or replay. Only the +placeholder zero is the tell — the same narrowing form 1 already makes, and for +the same reason. + +Forms 4 and 6 overlap on the zero case, so form 4 records the statements it +flags and form 6 skips them. The visit order makes that sound rather than +lucky: ``visit_If`` runs its own check BEFORE ``generic_visit`` descends into +the body, so the assignment is always registered before ``visit_Assign`` sees +it. Form 4 keeps its wider literal set, so a guarded ``= 1234.0`` still fires +exactly once, from form 4. + +BLIND SPOTS worth knowing. Form 1 keys on the callee's spelling, so ``AssistantMessageTelemetry`` (an import alias for ``AssistantMessage`` in -``claude_code_agent``) is matched by name only. Renaming that alias silently -disarms form 1 for that module. +``claude_code_agent``) is matched by name only; renaming that alias silently +disarms form 1 for that module. And form 6 keys on the TARGET's spelling, so it +sees ``cmd.duration_ms = 0.0`` but not a write through a rebound local or +``setattr(cmd, field, 0.0)`` — the same limit every AST rule here has without +type inference. ``# noqa: CE058`` for a genuinely aggregate-internal use where a missing value really is a zero, with a comment saying so. @@ -160,6 +191,10 @@ class NoTimingLiteral(BaseRule): def __init__(self, filepath: str) -> None: super().__init__(filepath) self._in_scope = bool(_SRC_ROOT.search(filepath)) + #: Assignments form 4 has already flagged, so form 6 does not report + #: the same statement a second time. Populated in `visit_If` before + #: `generic_visit` reaches the body — see the module docstring. + self._flagged_assigns: set[ast.Assign] = set() def visit_Call(self, node: ast.Call) -> None: # Form 1: `AssistantMessage(generation_duration_ms=0.0, ...)` @@ -223,5 +258,20 @@ def visit_If(self, node: ast.If) -> None: and _numeric_literal(stmt.value) and _same_target(stmt.targets[0], operand) ): + self._flagged_assigns.add(stmt) self.violation(stmt, _MESSAGE) self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> None: + # Form 6: `cmd.duration_ms = 0.0` with no `is None` guard on the value + # itself — see the module docstring for why this is narrower than + # form 4 and why the dedupe below is ordering-safe rather than lucky. + if ( + self._in_scope + and node not in self._flagged_assigns + and len(node.targets) == 1 + and _timing_name(node.targets[0]) is not None + and _zero_literal(node.value) + ): + self.violation(node, _MESSAGE) + self.generic_visit(node) diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 735d5bcf3..d158a1ad0 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -4507,9 +4507,22 @@ def test_allows_a_guard_that_assigns_a_different_field(self): assert not self._run(src) def test_allows_a_guard_on_a_different_receiver(self): - src = "if a.duration_ms is None:\n b.duration_ms = 0.0\n" + # Form 4 keys on the RECEIVER, not just the field name: a guard about + # `a` says nothing about `b`, so form 4 must not claim it did. A + # non-zero literal keeps this a form-4 test — the zero spelling is now + # form 6's, which is the case below. + src = "if a.duration_ms is None:\n b.duration_ms = 1234.0\n" assert not self._run(src) + def test_a_zero_on_a_different_receiver_is_form_sixs(self): + # The same shape with a ZERO does fire, and from form 6 rather than + # form 4. That is the correct reading: a guard naming `a` is no + # evidence at all about `b`, so `b.duration_ms = 0.0` is an ungrounded + # zero — exactly what form 6 is for. Pinned so the hand-off between + # the two forms is a stated property and not an accident of ordering. + src = "if a.duration_ms is None:\n b.duration_ms = 0.0\n" + assert len(self._run(src)) == 1 + def test_allows_a_guard_that_assigns_a_measured_value(self): src = "if cmd.duration_ms is None:\n cmd.duration_ms = measured\n" assert not self._run(src) @@ -4574,6 +4587,54 @@ def test_ignores_a_bare_union_ms(self): assert not self._run("x = union_ms(spans) or 0") assert not self._run("cfg = TurnRecord(tool_union_ms_limit=0)") + # Form 6 — the PLAIN assignment. Form 4 without the `is None` guard, or + # under a guard that tests something else. The live `_finalize_commands` + # defect was caught by form 4 only because it happened to spell its guard + # `if cmd.duration_ms is None:`; written under the enclosing + # `if cmd.result_status is None:` instead — which reads just as naturally + # and books the identical lie — it was invisible to forms 1-5. + def test_flags_a_bare_zero_assignment(self): + assert self._run("cmd.duration_ms = 0.0") + + def test_flags_a_zero_assignment_under_a_non_timing_guard(self): + # The shape that made form 4's coverage a coincidence. + src = "if cmd.result_status is None:\n cmd.duration_ms = 0.0\n" + assert self._run(src) + + def test_flags_a_bare_zero_assignment_to_a_head_or_tail(self): + assert self._run("rec.harness_startup_ms = 0") + assert self._run("rec.harness_teardown_ms = 0.0") + assert self._run("rec.tool_union_ms = 0.0") + + def test_form_six_flags_only_a_zero(self): + # A bare assignment proves NOTHING about whether the value was + # measured — `cmd.duration_ms = elapsed_ms` is how a real one is + # written, and a literal 1234.0 is a plausible factory or replay. Only + # the placeholder zero is the tell, so form 6 narrows the way form 1 + # does rather than the way form 4 does. + assert not self._run("cmd.duration_ms = 1234.0") + assert not self._run("cmd.duration_ms = measured") + assert not self._run("cmd.duration_ms = None") + + def test_form_six_ignores_a_non_timing_target(self): + assert not self._run("cmd.result_status = 0") + assert not self._run("cfg.duration_ms_limit = 0") + + def test_a_guarded_zero_is_reported_exactly_once(self): + # Forms 4 and 6 overlap on the zero case. `visit_If` registers the + # statement before `generic_visit` descends into the body, so the + # dedupe is ordering-safe rather than lucky — and one defect must + # produce one violation, or a noqa silences half of it. + src = "if cmd.duration_ms is None:\n cmd.duration_ms = 0.0\n" + assert len(self._run(src)) == 1 + + def test_form_four_still_owns_the_guarded_non_zero(self): + # Form 6 narrowed to zero, so a guarded `= 1234.0` must still fire — + # from form 4, which keeps the wider literal set because the guard + # PROVES the value was never measured. + src = "if cmd.duration_ms is None:\n cmd.duration_ms = 1234.0\n" + assert len(self._run(src)) == 1 + # Scope + suppression. def test_is_out_of_scope_outside_src(self): assert not self._run( @@ -4581,6 +4642,9 @@ def test_is_out_of_scope_outside_src(self): filepath="tests/test_codex_agent.py", ) + def test_form_six_is_out_of_scope_outside_src(self): + assert not self._run("cmd.duration_ms = 0.0", filepath="tests/test_codex_agent.py") + def test_noqa_suppresses(self): from tests.lint.runner import check_file