From 6736fce00048655b913d06a976eb7e016db6ab8b Mon Sep 17 00:00:00 2001 From: Josh Joseph Date: Fri, 16 Oct 2026 22:41:20 +0000 Subject: [PATCH 1/3] Make smart-routing reasons easier to read --- README.md | 6 + src/ucode/smart_routing/rationale.py | 64 +++++++++++ src/ucode/smart_routing/routing.py | 14 ++- tests/README.md | 4 + tests/integration/README.md | 4 + tests/test_routing_rationale.py | 162 +++++++++++++++++++++++++++ 6 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 src/ucode/smart_routing/rationale.py create mode 100644 tests/test_routing_rationale.py diff --git a/README.md b/README.md index 5853a7568..e16868b8a 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,12 @@ candidate models, their harnesses, and the router name sent with each recorded decision. Feedback investigations can compare the selection with that original menu without querying the router again. +Claude and Codex routing reasons summarize the task profile in plain language, such as +"This task has a clear target and limited scope." A separate sentence explains when +model availability requires a higher tier. Custom gateway notices are retained, and +older or unrecognized responses keep their original wording. Diagnostic decision records +keep the gateway's full explanation. + On Windows, Claude smart routing uses subagent hooks only. If first-prompt routing is enabled, ug warns and falls back to subagent routing because the first-prompt wrapper requires a Unix terminal. diff --git a/src/ucode/smart_routing/rationale.py b/src/ucode/smart_routing/rationale.py new file mode 100644 index 000000000..78cafebad --- /dev/null +++ b/src/ucode/smart_routing/rationale.py @@ -0,0 +1,64 @@ +"""Plain-language explanations of structured task-router decisions.""" + +_PROFILE_REASONS = ( + ( + "localization_burden", + "open_search", + "This task needs investigation to find what needs attention.", + ), + ("scope", "cross_cutting", "This task spans several components."), + ("environment_burden", "live_external", "This task depends on live systems or data."), + ("reasoning_depth", "open_ended", "Solving this task requires exploring possible approaches."), + ("verification_strength", "weak", "There is no clear way to check the result of this task."), + ("scope", "connected", "This task involves several related components."), + ("reasoning_depth", "multi_step", "This task involves several dependent steps."), + ("localization_burden", "bounded_search", "This task needs a focused search."), +) + + +def format_rationale(selection: dict, rationale: str) -> str: + """Summarize task_v3 profiles, preserving unfamiliar responses and gateway notices.""" + params = selection.get("params") + if not isinstance(params, dict) or params.get("router_version") != "task_v3": + return rationale + task_tier, served_tier = params.get("task_tier"), params.get("served_tier") + tiers = ("Low", "Medium", "High") + if task_tier not in tiers or served_tier not in tiers: + return rationale + if tiers.index(served_tier) < tiers.index(task_tier): + return rationale + + evidence = params.get("evidence_spans") + if not isinstance(evidence, list) or len(evidence) != 1 or not isinstance(evidence[0], str): + return rationale + ending = f" Request: “{evidence[0]}”." + if task_tier != served_tier: + ending += f" Your configured model availability requires service at {served_tier}." + # Only replace the known generated portion; custom gateway notices follow it. + _, separator, notice = rationale.partition(ending) + if not separator: + return rationale + + profile = params.get("task_profile") + if params.get("profiler_status") == "unavailable": + if task_tier != "High" or served_tier != "High": + return rationale + reason = "The task could not be assessed, so a high-capability model was selected." + elif params.get("profiler_status") == "ok" and isinstance(profile, dict): + reason = next( + (text for field, value, text in _PROFILE_REASONS if profile.get(field) == value), + "", + ) + if ( + not reason + and profile.get("localization_burden") == "explicit_target" + and profile.get("scope") == "localized" + ): + reason = "This task has a clear target and limited scope." + if not reason: + return rationale + else: + return rationale + if task_tier != served_tier: + reason += " A higher-tier model was selected because the requested tier isn't available." + return " ".join(part for part in (reason, notice.strip()) if part) diff --git a/src/ucode/smart_routing/routing.py b/src/ucode/smart_routing/routing.py index efc352c4d..35fe22d74 100644 --- a/src/ucode/smart_routing/routing.py +++ b/src/ucode/smart_routing/routing.py @@ -23,6 +23,8 @@ from pathlib import Path from typing import Any +from ucode.smart_routing.rationale import format_rationale + ROUTER_NAME = "task_v3" ROUTER_NAME_ENV_VAR = "SMART_ROUTER_NAME" ROUTING_PATH = "/ai-gateway/routing/v1/routes:select" @@ -94,6 +96,7 @@ class RoutingDecision: rationale: str = "" route_options: tuple[tuple[str, str | None], ...] = () router_name: str = "" + raw_rationale: str | None = None def display_message( self, @@ -262,14 +265,17 @@ def select_route( model = resolve(raw_model) if model is None: return None, f"router selected unsupported model {raw_model!r}" - rationale = payload.get("rationale") if isinstance(payload, dict) else None + raw_rationale = payload.get("rationale") + raw_rationale = raw_rationale if isinstance(raw_rationale, str) else "" + rationale = format_rationale(payload["route_selection"][0], raw_rationale) return ( RoutingDecision( model=model, raw_model=raw_model, - rationale=rationale if isinstance(rationale, str) else "", + rationale=rationale, route_options=offered_routes, router_name=router_name, + raw_rationale=raw_rationale if rationale != raw_rationale else None, ), None, ) @@ -423,7 +429,9 @@ def write_decision_record( "router_name": decision.router_name, # Persisted for diagnosis: an empty value means the gateway returned # no rationale (vs. a placement bug in how we surface it). - "rationale": decision.rationale, + "rationale": decision.raw_rationale + if decision.raw_rationale is not None + else decision.rationale, "at": time.time(), }, ) diff --git a/tests/README.md b/tests/README.md index 17c67b403..b1e7a7129 100644 --- a/tests/README.md +++ b/tests/README.md @@ -377,6 +377,10 @@ they do not establish live agent coverage. ## CUJ coverage matrix +`test_routing_rationale.py` covers plain-language reasons for Claude and Codex, model +availability and profiling fallbacks, preserved gateway notices, and unchanged model +selection and diagnostic reasons. It replaces the HTTP response; it does not run live agents. + These are **implemented assertions**, not a claim that every version passes. Consult the run's JUnit report and artifacts for results. Each function states its **Scenario** and **Expected** outcome and shows its configure and launch diff --git a/tests/integration/README.md b/tests/integration/README.md index e9f5f2f75..69d200eb4 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,5 +1,9 @@ # Integration tests +`../test_routing_rationale.py` covers routing-reason wording, availability and profiling +fallbacks, preserved gateway notices, and unchanged decisions as component checks with +controlled HTTP responses. It does not establish live agent rendering coverage. + The separate [dedicated-workspace CUJ](../e2e_cuj/AGENTS.md) lives in `tests/e2e_cuj/` and runs directly with pytest. It does not use this suite's runner or config fixtures. It reuses the session, terminal, file-task, and transcript helpers in `utils/`. diff --git a/tests/test_routing_rationale.py b/tests/test_routing_rationale.py new file mode 100644 index 000000000..8db8971e3 --- /dev/null +++ b/tests/test_routing_rationale.py @@ -0,0 +1,162 @@ +"""Component coverage for displayed routing reasons and unchanged router decisions.""" + +import io +import json + +import pytest + +from ucode.smart_routing import claude_routing, codex_routing, routing +from ucode.smart_routing.rationale import format_rationale + + +@pytest.fixture +def router_response(): + return { + "route_selection": [ + { + "route_option": {"model": "gpt-5-6-luna", "harness": "codex"}, + "params": { + "router_version": "task_v3", + "task_tier": "Low", + "served_tier": "Low", + "profiler_status": "ok", + "evidence_spans": ["Inspect the parser."], + "task_profile": { + "task_type": "investigate", + "localization_burden": "explicit_target", + "scope": "localized", + "reasoning_depth": "direct", + "environment_burden": "workspace", + "verification_strength": "explicit", + "difficulty": "low", + }, + }, + } + ], + "rationale": ( + "Routed to Low because its task-only profile is low: " + "an explicit target, localized scope, direct reasoning. " + "Request: “Inspect the parser.”." + ), + } + + +@pytest.mark.parametrize( + ("profile", "expected"), + [ + ({}, "This task has a clear target and limited scope."), + ({"scope": "connected"}, "This task involves several related components."), + ({"scope": "cross_cutting"}, "This task spans several components."), + ( + {"localization_burden": "open_search"}, + "This task needs investigation to find what needs attention.", + ), + ({"environment_burden": "live_external"}, "This task depends on live systems or data."), + ({"reasoning_depth": "multi_step"}, "This task involves several dependent steps."), + ], +) +def test_reasons_describe_the_task_profile(router_response, profile, expected): + selection = router_response["route_selection"][0] + selection["params"]["task_profile"].update(profile) + + assert format_rationale(selection, router_response["rationale"]) == expected + + +def test_availability_explanation_and_custom_gateway_notice(router_response): + selection = router_response["route_selection"][0] + selection["params"]["served_tier"] = "High" + rationale = ( + router_response["rationale"] + + " Your configured model availability requires service at High." + + " Contact the platform team with feedback." + ) + + assert format_rationale(selection, rationale) == ( + "This task has a clear target and limited scope. " + "A higher-tier model was selected because the requested tier isn't available. " + "Contact the platform team with feedback." + ) + + +def test_unavailable_profile_explains_the_fallback(router_response): + selection = router_response["route_selection"][0] + selection["params"].update( + task_tier="High", served_tier="High", profiler_status="unavailable", task_profile=None + ) + + assert format_rationale(selection, router_response["rationale"]) == ( + "The task could not be assessed, so a high-capability model was selected." + ) + + +@pytest.mark.parametrize( + "params", + [ + None, + [], + {"router_version": "task_v4"}, + {"task_profile": None}, + {"task_profile": {"scope": "unknown"}}, + {"evidence_spans": []}, + {"served_tier": "unknown"}, + ], +) +def test_unrecognized_metadata_keeps_the_gateway_reason(router_response, params): + selection = router_response["route_selection"][0] + if isinstance(params, dict): + selection["params"].update(params) + else: + selection["params"] = params + + assert format_rationale(selection, router_response["rationale"]) == router_response["rationale"] + + +@pytest.mark.parametrize("rationale", ["", "A custom routing explanation."]) +def test_unrecognized_text_is_preserved(router_response, rationale): + assert format_rationale(router_response["route_selection"][0], rationale) == rationale + + +@pytest.mark.parametrize( + ("client", "model", "models"), + [ + ( + claude_routing, + "claude-sonnet-5", + ["system.ai.claude-sonnet-5", "system.ai.claude-opus-4-8"], + ), + (codex_routing, "gpt-5-6-luna", ["system.ai.gpt-5-6-luna", "system.ai.gpt-5-6-sol"]), + ], +) +def test_agent_notices_preserve_selection_and_diagnostics( + router_response, monkeypatch, tmp_path, client, model, models +): + router_response["route_selection"][0]["route_option"]["model"] = model + requests = [] + + # Replace only the remote response; exercise parsing, notices, and diagnostic writes. + def urlopen(request, timeout): + requests.append(json.loads(request.data)) + return io.BytesIO(json.dumps(router_response).encode()) + + monkeypatch.setattr(routing.urllib.request, "urlopen", urlopen) + decision, error = client.request_routing_decision( + "https://example.databricks.com", "token", "Inspect the parser.", models + ) + + assert error is None + assert decision.model == f"system.ai.{model}" + assert decision.raw_model == model + assert decision.rationale == "This task has a clear target and limited scope." + assert len(requests) == 1 + assert requests[0]["task"] == {"prompt": "Inspect the parser."} + assert requests[0]["route_selector"] == {"router_name": "task_v3"} + for is_subagent in (False, True): + assert f"Reason : {decision.rationale}" in decision.display_message(is_subagent=is_subagent) + + decisions_path = tmp_path / "decisions.jsonl" + routing.write_decision_record( + decisions_path, {}, "Inspect the parser.", decision, decision.model + ) + record = json.loads(decisions_path.read_text()) + assert record["rationale"] == router_response["rationale"] + assert record["requested_model"] == decision.model From ee981f7035fea8c09ca81424c8afa8f20789ff99 Mon Sep 17 00:00:00 2001 From: Josh Joseph Date: Fri, 16 Oct 2026 22:48:12 +0000 Subject: [PATCH 2/3] Trim routing rationale tests to two direct checks --- tests/README.md | 4 - tests/integration/README.md | 4 - tests/test_routing_rationale.py | 170 ++++---------------------------- 3 files changed, 17 insertions(+), 161 deletions(-) diff --git a/tests/README.md b/tests/README.md index b1e7a7129..17c67b403 100644 --- a/tests/README.md +++ b/tests/README.md @@ -377,10 +377,6 @@ they do not establish live agent coverage. ## CUJ coverage matrix -`test_routing_rationale.py` covers plain-language reasons for Claude and Codex, model -availability and profiling fallbacks, preserved gateway notices, and unchanged model -selection and diagnostic reasons. It replaces the HTTP response; it does not run live agents. - These are **implemented assertions**, not a claim that every version passes. Consult the run's JUnit report and artifacts for results. Each function states its **Scenario** and **Expected** outcome and shows its configure and launch diff --git a/tests/integration/README.md b/tests/integration/README.md index 69d200eb4..e9f5f2f75 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -1,9 +1,5 @@ # Integration tests -`../test_routing_rationale.py` covers routing-reason wording, availability and profiling -fallbacks, preserved gateway notices, and unchanged decisions as component checks with -controlled HTTP responses. It does not establish live agent rendering coverage. - The separate [dedicated-workspace CUJ](../e2e_cuj/AGENTS.md) lives in `tests/e2e_cuj/` and runs directly with pytest. It does not use this suite's runner or config fixtures. It reuses the session, terminal, file-task, and transcript helpers in `utils/`. diff --git a/tests/test_routing_rationale.py b/tests/test_routing_rationale.py index 8db8971e3..03b4add21 100644 --- a/tests/test_routing_rationale.py +++ b/tests/test_routing_rationale.py @@ -1,162 +1,26 @@ -"""Component coverage for displayed routing reasons and unchanged router decisions.""" - -import io -import json - -import pytest - -from ucode.smart_routing import claude_routing, codex_routing, routing from ucode.smart_routing.rationale import format_rationale -@pytest.fixture -def router_response(): - return { - "route_selection": [ - { - "route_option": {"model": "gpt-5-6-luna", "harness": "codex"}, - "params": { - "router_version": "task_v3", - "task_tier": "Low", - "served_tier": "Low", - "profiler_status": "ok", - "evidence_spans": ["Inspect the parser."], - "task_profile": { - "task_type": "investigate", - "localization_burden": "explicit_target", - "scope": "localized", - "reasoning_depth": "direct", - "environment_burden": "workspace", - "verification_strength": "explicit", - "difficulty": "low", - }, - }, - } - ], - "rationale": ( - "Routed to Low because its task-only profile is low: " - "an explicit target, localized scope, direct reasoning. " - "Request: “Inspect the parser.”." - ), +def test_plain_language_reason(): + selection = { + "params": { + "router_version": "task_v3", + "task_tier": "Low", + "served_tier": "Low", + "profiler_status": "ok", + "evidence_spans": ["Inspect the parser."], + "task_profile": {"localization_burden": "explicit_target", "scope": "localized"}, + } } - - -@pytest.mark.parametrize( - ("profile", "expected"), - [ - ({}, "This task has a clear target and limited scope."), - ({"scope": "connected"}, "This task involves several related components."), - ({"scope": "cross_cutting"}, "This task spans several components."), - ( - {"localization_burden": "open_search"}, - "This task needs investigation to find what needs attention.", - ), - ({"environment_burden": "live_external"}, "This task depends on live systems or data."), - ({"reasoning_depth": "multi_step"}, "This task involves several dependent steps."), - ], -) -def test_reasons_describe_the_task_profile(router_response, profile, expected): - selection = router_response["route_selection"][0] - selection["params"]["task_profile"].update(profile) - - assert format_rationale(selection, router_response["rationale"]) == expected - - -def test_availability_explanation_and_custom_gateway_notice(router_response): - selection = router_response["route_selection"][0] - selection["params"]["served_tier"] = "High" rationale = ( - router_response["rationale"] - + " Your configured model availability requires service at High." - + " Contact the platform team with feedback." - ) - - assert format_rationale(selection, rationale) == ( - "This task has a clear target and limited scope. " - "A higher-tier model was selected because the requested tier isn't available. " - "Contact the platform team with feedback." + "Routed to Low because its task-only profile is low: " + "an explicit target, localized scope, direct reasoning. Request: “Inspect the parser.”." ) - - -def test_unavailable_profile_explains_the_fallback(router_response): - selection = router_response["route_selection"][0] - selection["params"].update( - task_tier="High", served_tier="High", profiler_status="unavailable", task_profile=None + assert ( + format_rationale(selection, rationale) == "This task has a clear target and limited scope." ) - assert format_rationale(selection, router_response["rationale"]) == ( - "The task could not be assessed, so a high-capability model was selected." - ) - -@pytest.mark.parametrize( - "params", - [ - None, - [], - {"router_version": "task_v4"}, - {"task_profile": None}, - {"task_profile": {"scope": "unknown"}}, - {"evidence_spans": []}, - {"served_tier": "unknown"}, - ], -) -def test_unrecognized_metadata_keeps_the_gateway_reason(router_response, params): - selection = router_response["route_selection"][0] - if isinstance(params, dict): - selection["params"].update(params) - else: - selection["params"] = params - - assert format_rationale(selection, router_response["rationale"]) == router_response["rationale"] - - -@pytest.mark.parametrize("rationale", ["", "A custom routing explanation."]) -def test_unrecognized_text_is_preserved(router_response, rationale): - assert format_rationale(router_response["route_selection"][0], rationale) == rationale - - -@pytest.mark.parametrize( - ("client", "model", "models"), - [ - ( - claude_routing, - "claude-sonnet-5", - ["system.ai.claude-sonnet-5", "system.ai.claude-opus-4-8"], - ), - (codex_routing, "gpt-5-6-luna", ["system.ai.gpt-5-6-luna", "system.ai.gpt-5-6-sol"]), - ], -) -def test_agent_notices_preserve_selection_and_diagnostics( - router_response, monkeypatch, tmp_path, client, model, models -): - router_response["route_selection"][0]["route_option"]["model"] = model - requests = [] - - # Replace only the remote response; exercise parsing, notices, and diagnostic writes. - def urlopen(request, timeout): - requests.append(json.loads(request.data)) - return io.BytesIO(json.dumps(router_response).encode()) - - monkeypatch.setattr(routing.urllib.request, "urlopen", urlopen) - decision, error = client.request_routing_decision( - "https://example.databricks.com", "token", "Inspect the parser.", models - ) - - assert error is None - assert decision.model == f"system.ai.{model}" - assert decision.raw_model == model - assert decision.rationale == "This task has a clear target and limited scope." - assert len(requests) == 1 - assert requests[0]["task"] == {"prompt": "Inspect the parser."} - assert requests[0]["route_selector"] == {"router_name": "task_v3"} - for is_subagent in (False, True): - assert f"Reason : {decision.rationale}" in decision.display_message(is_subagent=is_subagent) - - decisions_path = tmp_path / "decisions.jsonl" - routing.write_decision_record( - decisions_path, {}, "Inspect the parser.", decision, decision.model - ) - record = json.loads(decisions_path.read_text()) - assert record["rationale"] == router_response["rationale"] - assert record["requested_model"] == decision.model +def test_unknown_response_is_preserved(): + rationale = "A custom routing explanation." + assert format_rationale({}, rationale) == rationale From 71641689294b63fa00682ce4ca1f07a649360a5e Mon Sep 17 00:00:00 2001 From: Josh Joseph Date: Fri, 16 Oct 2026 22:59:03 +0000 Subject: [PATCH 3/3] Keep logged routing reasons aligned with displayed text --- README.md | 4 ++-- src/ucode/smart_routing/routing.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e16868b8a..e21fb4983 100644 --- a/README.md +++ b/README.md @@ -284,8 +284,8 @@ menu without querying the router again. Claude and Codex routing reasons summarize the task profile in plain language, such as "This task has a clear target and limited scope." A separate sentence explains when model availability requires a higher tier. Custom gateway notices are retained, and -older or unrecognized responses keep their original wording. Diagnostic decision records -keep the gateway's full explanation. +older or unrecognized responses keep their original wording. Diagnostic records keep +the displayed reason in `rationale` and the gateway's full explanation in `raw_rationale`. On Windows, Claude smart routing uses subagent hooks only. If first-prompt routing is enabled, ug warns and falls back to subagent routing because the first-prompt diff --git a/src/ucode/smart_routing/routing.py b/src/ucode/smart_routing/routing.py index 35fe22d74..6286c767d 100644 --- a/src/ucode/smart_routing/routing.py +++ b/src/ucode/smart_routing/routing.py @@ -429,7 +429,8 @@ def write_decision_record( "router_name": decision.router_name, # Persisted for diagnosis: an empty value means the gateway returned # no rationale (vs. a placement bug in how we surface it). - "rationale": decision.raw_rationale + "rationale": decision.rationale, + "raw_rationale": decision.raw_rationale if decision.raw_rationale is not None else decision.rationale, "at": time.time(),