From cb21c3430aea484c2e21fa78ab0eb757d60b9776 Mon Sep 17 00:00:00 2001 From: hywznn Date: Wed, 19 Aug 2026 16:18:10 +0900 Subject: [PATCH 1/2] =?UTF-8?q?fix(language):=20=EA=B5=AC=EC=A1=B0?= =?UTF-8?q?=ED=99=94=20=EB=B2=88=EC=97=AD=20=EC=8B=A4=ED=8C=A8=20=EC=A7=84?= =?UTF-8?q?=EB=8B=A8=20=EB=B3=B4=EC=99=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/agents/language/easy_korean.py | 5 +- app/agents/language/generation/__init__.py | 2 + app/agents/language/generation/ollama.py | 12 +- .../language/generation/openai_compatible.py | 150 ++++++++++++++++-- app/agents/language/translation.py | 5 +- app/agents/language/validation.py | 27 +++- tests/agents/language/test_generation_port.py | 55 ++++++- tests/agents/language/test_graph.py | 7 + tests/agents/language/test_validation.py | 12 +- 9 files changed, 244 insertions(+), 31 deletions(-) diff --git a/app/agents/language/easy_korean.py b/app/agents/language/easy_korean.py index 14d489f..1b29cb3 100644 --- a/app/agents/language/easy_korean.py +++ b/app/agents/language/easy_korean.py @@ -185,7 +185,10 @@ def generate_fn(is_correction: bool, payload: dict[str, object]) -> EasyKoreanDr WarningItem( component="easy_korean", code=WarningCode.EASY_KOREAN_GENERATION_FAILED, - message="Easy Korean generation failed", + message=( + "Easy Korean generation failed: " + f"{correction_result.generation_error_code or 'GENERATION_FAILED'}" + ), ), WarningItem( component="easy_korean", diff --git a/app/agents/language/generation/__init__.py b/app/agents/language/generation/__init__.py index e5f4ed6..196342f 100644 --- a/app/agents/language/generation/__init__.py +++ b/app/agents/language/generation/__init__.py @@ -9,6 +9,7 @@ from .openai_compatible import ( GenerationError, GenerationHTTPError, + GenerationRefusalError, GenerationResponseTooLargeError, GenerationSchemaError, GenerationTransportError, @@ -20,6 +21,7 @@ "EasyKoreanDraft", "GenerationError", "GenerationHTTPError", + "GenerationRefusalError", "GenerationResponseTooLargeError", "GenerationSchemaError", "GenerationTransportError", diff --git a/app/agents/language/generation/ollama.py b/app/agents/language/generation/ollama.py index edc2cfe..f791ff3 100644 --- a/app/agents/language/generation/ollama.py +++ b/app/agents/language/generation/ollama.py @@ -91,18 +91,14 @@ def generate( response = client.post(url, headers=headers, json=request_body) if len(response.content) > 1_048_576: - raise GenerationResponseTooLargeError( - "Response content exceeds 1 MiB limit" - ) + raise GenerationResponseTooLargeError("Response content exceeds 1 MiB limit") if response.status_code == 200: try: response_json = response.json() content = response_json.get("message", {}).get("content") if not isinstance(content, str): - raise GenerationSchemaError( - "Content is missing or not a string" - ) + raise GenerationSchemaError("Content is missing or not a string") except (json.JSONDecodeError, AttributeError) as err: raise GenerationSchemaError( f"Invalid Ollama response wrapper: {err}" @@ -124,9 +120,7 @@ def generate( continue raise last_error - raise GenerationHTTPError( - f"HTTP generation request failed with status {response.status_code}" - ) + raise GenerationHTTPError(status_code=response.status_code) except (httpx.TimeoutException, httpx.NetworkError, httpx.TransportError) as err: last_error = GenerationTransportError( f"Network transport error: {type(err).__name__}" diff --git a/app/agents/language/generation/openai_compatible.py b/app/agents/language/generation/openai_compatible.py index 64d9a0a..6c22c67 100644 --- a/app/agents/language/generation/openai_compatible.py +++ b/app/agents/language/generation/openai_compatible.py @@ -1,4 +1,6 @@ import json +import logging +import re from collections.abc import Mapping from typing import TypeVar @@ -11,6 +13,9 @@ from app.agents.language.resources.prompts import load_prompt T = TypeVar("T", bound=BaseModel) +logger = logging.getLogger(__name__) + +_SAFE_PROVIDER_CODE = re.compile(r"^[A-Za-z0-9_.-]{1,100}$") def _sanitize_payload(payload: Mapping[str, object]) -> dict[str, object]: @@ -24,34 +29,114 @@ def _sanitize_payload(payload: Mapping[str, object]) -> dict[str, object]: result[key] = sanitize_user_input(value) elif isinstance(value, list): result[key] = [ - sanitize_user_input(item) if isinstance(item, str) else item - for item in value + sanitize_user_input(item) if isinstance(item, str) else item for item in value ] else: result[key] = value return result - class GenerationError(Exception): """Base exception for generation errors.""" + code = "GENERATION_FAILED" + + def __init__(self, message: str, *, request_id: str | None = None) -> None: + super().__init__(message) + self.request_id = request_id + class GenerationHTTPError(GenerationError): """Raised for non-retryable HTTP error statuses (e.g. 400, 401, 403, 404).""" + def __init__( + self, + *, + status_code: int, + provider_error_code: str | None = None, + request_id: str | None = None, + ) -> None: + if status_code == 400: + self.code = "PROVIDER_REQUEST_INVALID" + elif status_code in (401, 403): + self.code = "PROVIDER_AUTH_FAILED" + else: + self.code = "PROVIDER_HTTP_ERROR" + self.status_code = status_code + self.provider_error_code = provider_error_code + provider_suffix = f" provider_code={provider_error_code}" if provider_error_code else "" + super().__init__( + f"HTTP generation request failed with status {status_code}{provider_suffix}", + request_id=request_id, + ) + class GenerationTransportError(GenerationError): """Raised for retryable transport errors (429, 5xx, timeouts).""" + code = "PROVIDER_UNAVAILABLE" + class GenerationSchemaError(GenerationError): """Raised when LLM output violates JSON schema or contract.""" + code = "STRUCTURED_OUTPUT_INVALID" + + +class GenerationRefusalError(GenerationError): + """Raised when the provider explicitly refuses a structured-output request.""" + + code = "PROVIDER_REFUSED" + class GenerationResponseTooLargeError(GenerationError): """Raised when response body exceeds 1 MiB size cap.""" + code = "PROVIDER_RESPONSE_TOO_LARGE" + + +def _request_id(response: httpx.Response) -> str | None: + value = response.headers.get("x-request-id") + if value and _SAFE_PROVIDER_CODE.fullmatch(value): + return value + return None + + +def _provider_error_code(response: httpx.Response) -> str | None: + """Extract only a bounded provider error identifier, never the raw error body.""" + try: + payload = response.json() + except (json.JSONDecodeError, ValueError): + return None + if not isinstance(payload, dict): + return None + error = payload.get("error") + if not isinstance(error, dict): + return None + for key in ("code", "type"): + value = error.get(key) + if isinstance(value, str) and _SAFE_PROVIDER_CODE.fullmatch(value): + return value + return None + + +def _log_generation_failure( + *, + operation: GenerationOperation, + model: str, + error: GenerationError, +) -> None: + """Emit metadata only; prompts, response bodies and credentials stay excluded.""" + logger.warning( + "structured_generation_failed operation=%s model=%s error_code=%s " + "error_type=%s provider_request_id=%s", + operation, + model, + error.code, + type(error).__name__, + error.request_id or "unavailable", + ) + class OpenAICompatibleGenerationPort(StructuredGenerationPort): def __init__( @@ -95,7 +180,6 @@ def generate( {"role": "system", "content": system_prompt}, {"role": "user", "content": json.dumps(safe_payload, ensure_ascii=False)}, ], - "temperature": 0, "response_format": { "type": "json_schema", "json_schema": { @@ -122,37 +206,73 @@ def generate( raise GenerationResponseTooLargeError("Response content exceeds 1 MiB limit") if response.status_code == 200: + request_id = _request_id(response) try: resp_json = response.json() choices = resp_json.get("choices", []) if not choices or not isinstance(choices, list): - raise GenerationSchemaError("Missing or invalid choices in response") - content = choices[0].get("message", {}).get("content") + raise GenerationSchemaError( + "Missing or invalid choices in response", + request_id=request_id, + ) + message = choices[0].get("message", {}) + if not isinstance(message, dict): + raise GenerationSchemaError( + "Message is missing or not an object", + request_id=request_id, + ) + refusal = message.get("refusal") + if isinstance(refusal, str) and refusal.strip(): + raise GenerationRefusalError( + "Provider refused the structured-output request", + request_id=request_id, + ) + content = message.get("content") if not isinstance(content, str): - raise GenerationSchemaError("Content is missing or not a string") + raise GenerationSchemaError( + "Content is missing or not a string", + request_id=request_id, + ) except (json.JSONDecodeError, AttributeError) as err: raise GenerationSchemaError( - f"Invalid completion JSON wrapper: {err}" + f"Invalid completion JSON wrapper: {type(err).__name__}", + request_id=request_id, ) from err try: return response_model.model_validate_json(content) except Exception as err: raise GenerationSchemaError( - f"Model validation error for {response_model.__name__}: {err}" + f"Model validation error for {response_model.__name__}: " + f"{type(err).__name__}", + request_id=request_id, ) from err if response.status_code in (429, 500, 502, 503, 504): last_error = GenerationTransportError( - f"HTTP transport status {response.status_code}" + f"HTTP transport status {response.status_code}", + request_id=_request_id(response), ) if attempt < max_attempts: continue + _log_generation_failure( + operation=operation, + model=self.model, + error=last_error, + ) raise last_error - raise GenerationHTTPError( - f"HTTP generation request failed with status {response.status_code}" + http_error = GenerationHTTPError( + status_code=response.status_code, + provider_error_code=_provider_error_code(response), + request_id=_request_id(response), ) + _log_generation_failure( + operation=operation, + model=self.model, + error=http_error, + ) + raise http_error except (httpx.TimeoutException, httpx.NetworkError, httpx.TransportError) as err: last_error = GenerationTransportError( @@ -160,6 +280,11 @@ def generate( ) if attempt < max_attempts: continue + _log_generation_failure( + operation=operation, + model=self.model, + error=last_error, + ) raise last_error from None if last_error: @@ -170,6 +295,7 @@ def generate( __all__ = [ "GenerationError", "GenerationHTTPError", + "GenerationRefusalError", "GenerationResponseTooLargeError", "GenerationSchemaError", "GenerationTransportError", diff --git a/app/agents/language/translation.py b/app/agents/language/translation.py index 18b294d..a0b2c36 100644 --- a/app/agents/language/translation.py +++ b/app/agents/language/translation.py @@ -199,7 +199,10 @@ def generate_fn(is_correction: bool, payload: dict[str, object]) -> TranslationD WarningItem( component="translation", code=WarningCode.TRANSLATION_GENERATION_FAILED, - message="Translation generation failed completely", + message=( + "Translation generation failed: " + f"{correction_result.generation_error_code or 'GENERATION_FAILED'}" + ), ) ) translation_result = TranslationResult( diff --git a/app/agents/language/validation.py b/app/agents/language/validation.py index adce505..bd5d52c 100644 --- a/app/agents/language/validation.py +++ b/app/agents/language/validation.py @@ -1,3 +1,4 @@ +import logging import re from collections import Counter from collections.abc import Sequence @@ -25,6 +26,8 @@ SemanticValidationPort, ) +logger = logging.getLogger(__name__) + def normalize_date_string(value: str) -> date | None: """Normalize surface date expressions into a canonical datetime.date object.""" @@ -240,6 +243,7 @@ class CorrectionResult: warnings: tuple[WarningCode, ...] requires_human_review: bool time_budget_exceeded: bool + generation_error_code: str | None = None class BoundedCorrectionController: @@ -268,6 +272,7 @@ def run( last_inconclusive_checks: tuple[ValidationCheckId, ...] = () warnings: list[WarningCode] = [] time_budget_exceeded = False + generation_error_code: str | None = None while True: # Check remaining time budget before scheduling generation/correction call @@ -301,18 +306,35 @@ def run( else: break last_draft = current_draft - except Exception: + except Exception as exc: + error_code = getattr(exc, "code", "GENERATION_FAILED") + if not isinstance(error_code, str): + error_code = "GENERATION_FAILED" + request_id = getattr(exc, "request_id", None) + if not isinstance(request_id, str): + request_id = "unavailable" + logger.warning( + "language_generation_failed component=%s error_code=%s " + "error_type=%s provider_request_id=%s correction=%s", + component, + error_code, + type(exc).__name__, + request_id, + is_correction, + ) + generation_error_code = error_code if attempts == 0: # Hard generation failure on initial attempt return CorrectionResult( draft=None, status="failed", retry_count=0, - failed_checks=("request_reason.present",), + failed_checks=(), inconclusive_checks=(), warnings=(), requires_human_review=True, time_budget_exceeded=False, + generation_error_code=generation_error_code, ) break @@ -401,6 +423,7 @@ def run( warnings=tuple(warnings), requires_human_review=True, time_budget_exceeded=time_budget_exceeded, + generation_error_code=generation_error_code, ) diff --git a/tests/agents/language/test_generation_port.py b/tests/agents/language/test_generation_port.py index 1e768db..25839ab 100644 --- a/tests/agents/language/test_generation_port.py +++ b/tests/agents/language/test_generation_port.py @@ -10,6 +10,7 @@ from app.agents.language.generation.openai_compatible import ( GenerationError, GenerationHTTPError, + GenerationRefusalError, GenerationResponseTooLargeError, GenerationSchemaError, GenerationTransportError, @@ -175,6 +176,7 @@ def handle_request(request: httpx.Request) -> httpx.Response: assert req_json["response_format"]["type"] == "json_schema" assert req_json["response_format"]["json_schema"]["strict"] is True assert req_json["response_format"]["json_schema"]["name"] == "EasyKoreanDraft" + assert "temperature" not in req_json def test_adapter_parses_valid_json() -> None: @@ -271,7 +273,17 @@ def handle_request_429(request: httpx.Request) -> httpx.Response: def test_adapter_maps_400_to_http_error() -> None: def handle_request_400(request: httpx.Request) -> httpx.Response: - return httpx.Response(400, content=b'{"error": "bad request"}') + return httpx.Response( + 400, + headers={"x-request-id": "req_test_400"}, + json={ + "error": { + "message": "must never leak", + "type": "invalid_request_error", + "code": "unsupported_parameter", + } + }, + ) transport = httpx.MockTransport(handle_request_400) port = OpenAICompatibleGenerationPort( @@ -281,12 +293,51 @@ def handle_request_400(request: httpx.Request) -> httpx.Response: transport=transport, ) - with pytest.raises(GenerationHTTPError): + with pytest.raises(GenerationHTTPError) as exc_info: port.generate( operation="easy_korean", payload={}, response_model=EasyKoreanDraft, ) + assert exc_info.value.code == "PROVIDER_REQUEST_INVALID" + assert exc_info.value.provider_error_code == "unsupported_parameter" + assert exc_info.value.request_id == "req_test_400" + assert "must never leak" not in str(exc_info.value) + + +def test_adapter_maps_provider_refusal_to_typed_error() -> None: + def handle_request(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + headers={"x-request-id": "req_refusal"}, + json={ + "choices": [ + { + "message": { + "content": None, + "refusal": "provider refusal text must not leak", + } + } + ] + }, + ) + + port = OpenAICompatibleGenerationPort( + base_url="https://api.fake-llm.com/v1", + api_key="secret", + model="gpt-5.6-luna", + transport=httpx.MockTransport(handle_request), + ) + + with pytest.raises(GenerationRefusalError) as exc_info: + port.generate( + operation="translation", + payload={}, + response_model=TranslationDraft, + ) + assert exc_info.value.code == "PROVIDER_REFUSED" + assert exc_info.value.request_id == "req_refusal" + assert "provider refusal text" not in str(exc_info.value) def test_adapter_retries_transport_once_only() -> None: diff --git a/tests/agents/language/test_graph.py b/tests/agents/language/test_graph.py index 495d4a7..d645dcc 100644 --- a/tests/agents/language/test_graph.py +++ b/tests/agents/language/test_graph.py @@ -7,6 +7,7 @@ LanguageAssistantOutput, LanguageExecutionPolicy, RequestContext, + WarningCode, ) from app.agents.language.generation.models import ( EasyKoreanDraft, @@ -171,6 +172,12 @@ def test_translation_failure_preserves_easy_and_standard( assert output.requires_human_review is True assert output.standard_korean_text != "" assert output.easy_korean_text != "" + translation_failure = next( + warning + for warning in output.warnings + if warning.code == WarningCode.TRANSLATION_GENERATION_FAILED + ) + assert translation_failure.message.endswith("GENERATION_FAILED") def test_parallel_execution_without_inter_branch_dependency( diff --git a/tests/agents/language/test_validation.py b/tests/agents/language/test_validation.py index 8ec097d..e1dc8c4 100644 --- a/tests/agents/language/test_validation.py +++ b/tests/agents/language/test_validation.py @@ -108,9 +108,7 @@ def test_extra_requested_item_fails(sample_context: RequestContext) -> None: requested_items=("여권 사본 1부", "근로계약서 사본 1부", "추가 서류 1부"), submission_method="방문 제출", ) - checks = validate_deterministic( - request_context=sample_context, candidate=draft_extra_item - ) + checks = validate_deterministic(request_context=sample_context, candidate=draft_extra_item) assert "requested_items.cardinality" in checks @@ -445,8 +443,12 @@ def generate_fn(is_correction: bool, payload: dict) -> EasyKoreanDraft: def test_hard_generation_failure_has_no_candidate(sample_context: RequestContext) -> None: + class ProviderRequestError(RuntimeError): + code = "PROVIDER_REQUEST_INVALID" + request_id = "req_safe_identifier" + def failing_generate_fn(is_correction: bool, payload: dict) -> EasyKoreanDraft: - raise RuntimeError("Generation failed completely") + raise ProviderRequestError("Generation failed completely") class PassValidator: def validate(self, **kwargs: object) -> SemanticValidationDecision: @@ -463,6 +465,8 @@ def validate(self, **kwargs: object) -> SemanticValidationDecision: ) assert result.draft is None assert result.status == "failed" + assert result.failed_checks == () + assert result.generation_error_code == "PROVIDER_REQUEST_INVALID" def test_branch_budget_uses_monotonic_clock(sample_context: RequestContext) -> None: From c5ea7db0b2b55788128e0175865e925330b61e4e Mon Sep 17 00:00:00 2001 From: hywznn Date: Wed, 19 Aug 2026 16:37:29 +0900 Subject: [PATCH 2/2] =?UTF-8?q?fix(language):=20strict=20=EC=9D=91?= =?UTF-8?q?=EB=8B=B5=20=EC=8A=A4=ED=82=A4=EB=A7=88=20=EC=A0=95=EA=B7=9C?= =?UTF-8?q?=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../language/generation/openai_compatible.py | 22 ++++++++- tests/agents/language/test_generation_port.py | 46 +++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/app/agents/language/generation/openai_compatible.py b/app/agents/language/generation/openai_compatible.py index 6c22c67..6ba992c 100644 --- a/app/agents/language/generation/openai_compatible.py +++ b/app/agents/language/generation/openai_compatible.py @@ -36,6 +36,26 @@ def _sanitize_payload(payload: Mapping[str, object]) -> dict[str, object]: return result +def _strict_json_schema(response_model: type[BaseModel]) -> dict[str, object]: + """Normalize Pydantic JSON Schema to the OpenAI strict-output subset.""" + schema = response_model.model_json_schema(mode="validation") + + def normalize(node: object) -> None: + if isinstance(node, dict): + node.pop("default", None) + properties = node.get("properties") + if isinstance(properties, dict): + node["required"] = list(properties) + for value in node.values(): + normalize(value) + elif isinstance(node, list): + for value in node: + normalize(value) + + normalize(schema) + return schema + + class GenerationError(Exception): """Base exception for generation errors.""" @@ -173,7 +193,7 @@ def generate( if self.api_key: headers["Authorization"] = f"Bearer {self.api_key}" - json_schema = response_model.model_json_schema(mode="validation") + json_schema = _strict_json_schema(response_model) request_body = { "model": self.model, "messages": [ diff --git a/tests/agents/language/test_generation_port.py b/tests/agents/language/test_generation_port.py index 25839ab..d40db3e 100644 --- a/tests/agents/language/test_generation_port.py +++ b/tests/agents/language/test_generation_port.py @@ -1,3 +1,5 @@ +import json + import httpx import pytest from pydantic import ValidationError @@ -179,6 +181,50 @@ def handle_request(request: httpx.Request) -> httpx.Response: assert "temperature" not in req_json +def test_adapter_normalizes_defaulted_fields_for_strict_schema() -> None: + captured: list[httpx.Request] = [] + + def handle_request(request: httpx.Request) -> httpx.Response: + captured.append(request) + return httpx.Response( + 200, + json={ + "choices": [ + { + "message": { + "content": json.dumps( + { + "status": "passed", + "failed_checks": [], + "inconclusive_checks": [], + } + ) + } + } + ] + }, + ) + + port = OpenAICompatibleGenerationPort( + base_url="https://api.fake-llm.com/v1", + api_key="secret", + model="gpt-5.6-luna", + transport=httpx.MockTransport(handle_request), + ) + + result = port.generate( + operation="semantic_validation", + payload={}, + response_model=SemanticValidationDraft, + ) + + schema = json.loads(captured[0].content)["response_format"]["json_schema"]["schema"] + assert schema["required"] == ["status", "failed_checks", "inconclusive_checks"] + assert "default" not in schema["properties"]["failed_checks"] + assert "default" not in schema["properties"]["inconclusive_checks"] + assert result.status == "passed" + + def test_adapter_parses_valid_json() -> None: def handle_request(request: httpx.Request) -> httpx.Response: body = (