From 88fee28b0e825a1d635f4a3c2ddd5611f95b00ad Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 13 Oct 2026 11:03:35 -0400 Subject: [PATCH 1/2] fix(mistral): capture new request params and streamed logprobs --- ..._setup_instruments_beta_conversations.yaml | 76 ----- ..._setup_instruments_beta_conversations.yaml | 66 ---- ..._setup_instruments_beta_conversations.yaml | 66 ---- .../integrations/mistral/test_mistral.py | 290 ++++++++---------- .../integrations/mistral/tracing.py | 18 ++ 5 files changed, 144 insertions(+), 372 deletions(-) delete mode 100644 py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_mistral_integration_setup_instruments_beta_conversations.yaml delete mode 100644 py/src/braintrust/integrations/mistral/cassettes/2.10.1/test_mistral_integration_setup_instruments_beta_conversations.yaml delete mode 100644 py/src/braintrust/integrations/mistral/cassettes/latest/test_mistral_integration_setup_instruments_beta_conversations.yaml diff --git a/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_mistral_integration_setup_instruments_beta_conversations.yaml b/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_mistral_integration_setup_instruments_beta_conversations.yaml deleted file mode 100644 index 5bb9dbdba..000000000 --- a/py/src/braintrust/integrations/mistral/cassettes/1.12.4/test_mistral_integration_setup_instruments_beta_conversations.yaml +++ /dev/null @@ -1,76 +0,0 @@ -interactions: -- request: - body: '{"inputs":"What is 4+4? Reply with just the number.","stream":false,"model":"mistral-small-latest"}' - headers: - Accept: - - application/json - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '99' - Host: - - api.mistral.ai - content-type: - - application/json - user-agent: - - mistral-client-python/1.12.4 - method: POST - uri: https://api.mistral.ai/v1/conversations - response: - body: - string: '{"object":"conversation.response","conversation_id":"conv_019d9d49ecb373428ac739585b3543ad","outputs":[{"object":"entry","type":"message.output","created_at":"2026-04-17T21:12:41.070743Z","completed_at":"2026-04-17T21:12:41.070743Z","model":"mistral-small-latest","id":"msg_019d9d49ed6e7757b61d28512b25d4b0","role":"assistant","content":"8"}],"usage":{"prompt_tokens":16,"completion_tokens":2,"total_tokens":18},"guardrails":null}' - headers: - CF-RAY: - - 9ede6de6e9e8ac4e-YYZ - Connection: - - keep-alive - Content-Type: - - application/json - Date: - - Fri, 17 Apr 2026 21:12:41 GMT - Server: - - cloudflare - Strict-Transport-Security: - - max-age=15552000; includeSubDomains; preload - Transfer-Encoding: - - chunked - X-Content-Type-Options: - - nosniff - access-control-allow-origin: - - '*' - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - content-length: - - '430' - mistral-correlation-id: - - 019d9d49-ec9b-73df-b016-f400c9f6c02a - set-cookie: - - __cf_bm=urgOUix4LFR3_573igkAsuF8r2VaU7n7jxOBA__cu4s-1776460360.7896147-1.0.1.1-CE2.sDKjSCwKJN7MApEk7pmY7uV7enPMwblU4x9ehi97bTEQurOHWqM5_4E8.g2M537HlLRGnu4cxULE2lO6LyNeb3OfohZ5pYE4zETGnmoanX2nHNvs0dAmIZw08_kX; - HttpOnly; Secure; Path=/; Domain=mistral.ai; Expires=Fri, 17 Apr 2026 21:42:41 - GMT - - _cfuvid=TyRm2V_n_kS.71yr5zIKq5G_MyOOgbYu9LOWoAXF.JM-1776460360.7896147-1.0.1.1-bPoyNr1swGONJROJsF66C6V8ZERwjEZZZVRjbIf7ctM; - HttpOnly; SameSite=None; Secure; Path=/; Domain=mistral.ai - x-envoy-upstream-service-time: - - '227' - x-kong-proxy-latency: - - '16' - x-kong-request-id: - - 019d9d49-ec9b-73df-b016-f400c9f6c02a - x-kong-upstream-latency: - - '230' - x-ratelimit-limit-tokens-minute: - - '500000' - x-ratelimit-limit-tokens-month: - - '1000000000' - x-ratelimit-remaining-tokens-minute: - - '499674' - x-ratelimit-remaining-tokens-month: - - '999999534' - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/2.10.1/test_mistral_integration_setup_instruments_beta_conversations.yaml b/py/src/braintrust/integrations/mistral/cassettes/2.10.1/test_mistral_integration_setup_instruments_beta_conversations.yaml deleted file mode 100644 index fbffd4c8f..000000000 --- a/py/src/braintrust/integrations/mistral/cassettes/2.10.1/test_mistral_integration_setup_instruments_beta_conversations.yaml +++ /dev/null @@ -1,66 +0,0 @@ -interactions: -- request: - body: '{"inputs":"What is 4+4? Reply with just the number.","stream":false,"model":"mistral-small-latest"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate - connection: - - keep-alive - content-length: - - '99' - content-type: - - application/json - host: - - api.mistral.ai - user-agent: - - mistral-client-python/2.10.0 - method: POST - uri: https://api.mistral.ai/v1/conversations - response: - body: - string: '{"object":"conversation.response","conversation_id":"conv_01a0a0238d4272b281ed0707ac888c17","outputs":[{"object":"entry","type":"message.output","created_at":"2026-09-14T13:37:59.785502Z","completed_at":"2026-09-14T13:37:59.785502Z","model":"mistral-small-latest","id":"msg_01a0a0238de97006a1b4484b2dd1657d","role":"assistant","content":"8"}],"usage":{"prompt_tokens":16,"completion_tokens":2,"total_tokens":18},"guardrails":null}' - headers: - access-control-allow-origin: - - '*' - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - cf-ray: - - a3afca1aecf1ab1e-YYZ - connection: - - keep-alive - content-length: - - '430' - content-type: - - application/json - date: - - Mon, 14 Sep 2026 13:37:59 GMT - mistral-correlation-id: - - 01a0a023-8d22-7097-b045-65e2b21ccd3d - server: - - cloudflare - strict-transport-security: - - max-age=15552000; includeSubDomains; preload - transfer-encoding: - - chunked - x-content-type-options: - - nosniff - x-envoy-upstream-service-time: - - '246' - x-kong-proxy-latency: - - '13' - x-kong-request-id: - - 01a0a023-8d22-7097-b045-65e2b21ccd3d - x-kong-upstream-latency: - - '248' - x-ratelimit-limit-tokens-minute: - - '20000' - x-ratelimit-remaining-tokens-minute: - - '19277' - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_mistral_integration_setup_instruments_beta_conversations.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_mistral_integration_setup_instruments_beta_conversations.yaml deleted file mode 100644 index 4e5838687..000000000 --- a/py/src/braintrust/integrations/mistral/cassettes/latest/test_mistral_integration_setup_instruments_beta_conversations.yaml +++ /dev/null @@ -1,66 +0,0 @@ -interactions: -- request: - body: '{"inputs":"What is 4+4? Reply with just the number.","stream":false,"model":"mistral-small-latest"}' - headers: - accept: - - application/json - accept-encoding: - - gzip, deflate, zstd - connection: - - keep-alive - content-length: - - '99' - content-type: - - application/json - host: - - api.mistral.ai - user-agent: - - mistral-client-python/3.0.0 - method: POST - uri: https://api.mistral.ai/v1/conversations - response: - body: - string: '{"object":"conversation.response","conversation_id":"conv_01a10c87644376d2a099cbbcb82762ff","outputs":[{"object":"entry","type":"message.output","created_at":"2026-10-05T14:46:02.252917Z","completed_at":"2026-10-05T14:46:02.252917Z","model":"mistral-small-latest","id":"msg_01a10c87650c7018bf7e20dfdafcd779","role":"assistant","content":"8"}],"usage":{"prompt_tokens":16,"completion_tokens":2,"total_tokens":18},"guardrails":null}' - headers: - access-control-allow-origin: - - '*' - alt-svc: - - h3=":443"; ma=86400 - cf-cache-status: - - DYNAMIC - cf-ray: - - a45d36a60f0b1185-YYZ - connection: - - keep-alive - content-length: - - '430' - content-type: - - application/json - date: - - Mon, 05 Oct 2026 14:46:02 GMT - mistral-correlation-id: - - 01a10c87-63fd-741a-afad-79a14615259b - server: - - cloudflare - strict-transport-security: - - max-age=15552000; includeSubDomains; preload - transfer-encoding: - - chunked - x-content-type-options: - - nosniff - x-envoy-upstream-service-time: - - '291' - x-kong-proxy-latency: - - '27' - x-kong-request-id: - - 01a10c87-63fd-741a-afad-79a14615259b - x-kong-upstream-latency: - - '293' - x-ratelimit-limit-tokens-minute: - - '20000' - x-ratelimit-remaining-tokens-minute: - - '19320' - status: - code: 200 - message: OK -version: 1 diff --git a/py/src/braintrust/integrations/mistral/test_mistral.py b/py/src/braintrust/integrations/mistral/test_mistral.py index bc9ac2cf2..5edeb352d 100644 --- a/py/src/braintrust/integrations/mistral/test_mistral.py +++ b/py/src/braintrust/integrations/mistral/test_mistral.py @@ -30,12 +30,8 @@ from mistralai import Mistral try: - Conversations = importlib.import_module("mistralai.client.conversations").Conversations - Transcriptions = importlib.import_module("mistralai.client.transcriptions").Transcriptions models = importlib.import_module("mistralai.client.models") except ImportError: - Conversations = importlib.import_module("mistralai.conversations").Conversations - Transcriptions = importlib.import_module("mistralai.transcriptions").Transcriptions models = importlib.import_module("mistralai.models") try: @@ -83,11 +79,10 @@ def _assert_speech_output_attachment(span): assert attachment.reference["content_type"] == "audio/mpeg" -def _assert_transcription_complete_span(span, response_text, start, end, *, check_content_type=False): +def _assert_transcription_complete_span(span, response_text, start, end): assert isinstance(span["input"]["file"], Attachment) assert span["input"]["file"].reference["filename"] == TEST_AUDIO_FILE.name - if check_content_type: - assert span["input"]["file"].reference["content_type"] == "audio/wav" + assert span["input"]["file"].reference["content_type"] == "audio/wav" assert span["metadata"]["provider"] == "mistral" assert span["metadata"]["model"] in AUDIO_TRANSCRIPTION_RESPONSE_MODELS assert span["output"] == response_text @@ -128,50 +123,6 @@ def _assert_conversation_span(span, expected_input, start, end, *, expected_cont assert_metrics_are_valid(span["metrics"], start, end) -def _method_refs(*targets): - """Snapshot ``{(cls, method): static attr}`` for ``(cls, methods)`` pairs, skipping missing classes.""" - return { - (cls, method): inspect.getattr_static(cls, method) - for cls, methods in targets - if cls is not None - for method in methods - } - - -def _audio_method_refs(): - return _method_refs( - (Transcriptions, ("complete", "complete_async", "stream", "stream_async")), - (Speech, ("complete", "complete_async")), - ) - - -def _conversation_method_refs(): - return _method_refs( - ( - Conversations, - ( - "start", - "start_async", - "start_stream", - "start_stream_async", - "append", - "append_async", - "append_stream", - "append_stream_async", - "restart", - "restart_async", - "restart_stream", - "restart_stream_async", - ), - ), - ) - - -def _restore_method_refs(monkeypatch, refs): - for (cls, method), original in refs.items(): - monkeypatch.setattr(cls, method, original) - - @pytest.fixture def memory_logger(): init_test_logger(PROJECT_NAME) @@ -183,6 +134,55 @@ def _get_client(): return Mistral(api_key=os.environ.get("MISTRAL_API_KEY")) +@pytest.fixture +def mistral_client(request, monkeypatch): + if request.param == "setup": + for composite in MistralIntegration.patchers: + for patcher in composite.sub_patchers: + if not patcher.applies(None, None): + continue + module = patcher.resolve_root(None, None) + class_name, method_name = patcher.target_path.split(".") + cls = getattr(module, class_name) + method = inspect.getattr_static(cls, method_name) + monkeypatch.setattr(cls, method_name, method) + # wrapt forwards patch markers to the original callable; restore + # those too so another setup case can instrument it again. + marker = patcher.patch_marker_attr() + monkeypatch.setattr(method, marker, getattr(method, marker, False), raising=False) + assert MistralIntegration.setup() + return _get_client() + return wrap_mistral(_get_client()) + + +def _completion_params(method): + if "logprobs" not in inspect.signature(method).parameters: + return {} + return { + "logprobs": True, + "top_logprobs": 2, + "prompt_logprobs": True, + "top_prompt_logprobs": 2, + "min_tokens": 1, + "repetition_penalty": 1.1, + "top_k": 20, + } + + +def _assert_completion_params(span, params): + for key, value in params.items(): + assert span["metadata"][key] == value + + +def _assert_completion_logprobs(span, response): + for key in ("logprobs", "prompt_logprobs"): + expected = getattr(response.choices[0], key) + assert hasattr(expected, "content") and expected.content, ( + f"Expected populated {key}; re-record this cassette with Mistral 3.2.0." + ) + assert span["output"][0][key] == expected.model_dump(mode="python", by_alias=True) + + def _weather_tool(): return { "type": "function", @@ -229,11 +229,13 @@ def test_wrap_mistral_chat_complete_sync(memory_logger): assert not memory_logger.pop() client = wrap_mistral(_get_client()) + params = _completion_params(client.chat.complete) start = time.time() response = client.chat.complete( model=CHAT_MODEL, messages=[{"role": "user", "content": "What is 2+2? Reply with just the number."}], max_tokens=10, + **params, ) end = time.time() @@ -247,6 +249,9 @@ def test_wrap_mistral_chat_complete_sync(memory_logger): assert span["input"] == [{"role": "user", "content": "What is 2+2? Reply with just the number."}] assert span["metadata"]["provider"] == "mistral" assert span["metadata"]["model"] == CHAT_MODEL + _assert_completion_params(span, params) + if params: + _assert_completion_logprobs(span, response) assert "4" in str(span["output"]) assert span["metrics"]["prompt_cached_tokens"] == 0 assert_metrics_are_valid(span["metrics"], start, end) @@ -305,11 +310,13 @@ def test_wrap_mistral_chat_stream_sync(memory_logger): assert not memory_logger.pop() client = wrap_mistral(_get_client()) + params = _completion_params(client.chat.stream) start = time.time() with client.chat.stream( model=CHAT_MODEL, messages=[{"role": "user", "content": "What is 5+5? Reply with just the number."}], max_tokens=10, + **params, ) as stream: chunks = list(stream) end = time.time() @@ -329,6 +336,18 @@ def test_wrap_mistral_chat_stream_sync(memory_logger): assert span["metadata"]["provider"] == "mistral" assert span["metadata"]["model"] == CHAT_MODEL assert span["metadata"]["stream"] == True + _assert_completion_params(span, params) + if params: + for key in ("logprobs", "prompt_logprobs"): + expected_content = [ + token.model_dump(mode="python", by_alias=True) + for chunk in chunks + for choice in chunk.data.choices + if hasattr(logprobs := getattr(choice, key), "content") + for token in logprobs.content + ] + assert expected_content, f"Expected populated {key}; re-record this cassette with Mistral 3.2.0." + assert span["output"][0][key]["content"] == expected_content assert span["metrics"]["time_to_first_token"] >= 0 assert "10" in str(span["output"]) assert_metrics_are_valid(span["metrics"], start, end) @@ -359,13 +378,13 @@ async def test_wrap_mistral_chat_complete_async(memory_logger): assert_metrics_are_valid(span["metrics"], start, end) -@pytest.mark.vcr -def test_wrap_mistral_beta_conversations_start_sync(memory_logger): +@pytest.mark.vcr(cassette_name="test_wrap_mistral_beta_conversations_start_sync") +@pytest.mark.parametrize("mistral_client", ["wrap", "setup"], indirect=True) +def test_mistral_beta_conversations_start_sync(memory_logger, mistral_client): assert not memory_logger.pop() - client = wrap_mistral(_get_client()) start = time.time() - response = client.beta.conversations.start( + response = mistral_client.beta.conversations.start( model=CHAT_MODEL, inputs="What is 2+2? Reply with just the number.", ) @@ -557,12 +576,14 @@ def test_wrap_mistral_agents_complete_sync(memory_logger): assert not memory_logger.pop() client = wrap_mistral(_get_client()) + params = _completion_params(client.agents.complete) with _temporary_agent(client) as agent_id: start = time.time() response = client.agents.complete( agent_id=agent_id, messages=[{"role": "user", "content": "What is 8+1? Reply with just the number."}], max_tokens=10, + **params, ) end = time.time() @@ -574,6 +595,9 @@ def test_wrap_mistral_agents_complete_sync(memory_logger): assert span["input"] == [{"role": "user", "content": "What is 8+1? Reply with just the number."}] assert span["metadata"]["provider"] == "mistral" assert span["metadata"]["agent_id"] == agent_id + _assert_completion_params(span, params) + if params: + _assert_completion_logprobs(span, response) assert "9" in str(span["output"]) assert_metrics_are_valid(span["metrics"], start, end) @@ -792,7 +816,6 @@ def test_wrap_mistral_fim_complete_sync(memory_logger): assert span["input"]["suffix"] == "\n\nprint(add(2, 3))" assert span["metadata"]["provider"] == "mistral" assert span["metadata"]["model"] == FIM_MODEL - assert "return" in str(span["input"]) assert span["output"] assert_metrics_are_valid(span["metrics"], start, end) @@ -890,36 +913,42 @@ async def test_wrap_mistral_fim_stream_async(memory_logger): assert_metrics_are_valid(span["metrics"], start, end) -@pytest.mark.vcr -def test_wrap_mistral_audio_transcriptions_complete_sync(memory_logger): - assert not memory_logger.pop() - - client = wrap_mistral(_get_client()) - with TEST_AUDIO_FILE.open("rb") as file_obj: - start = time.time() - response = client.audio.transcriptions.complete( - model=AUDIO_TRANSCRIPTION_MODEL, - file=_transcription_file_payload(file_obj), - ) - end = time.time() - - spans = memory_logger.pop() - assert len(spans) == 1 - _assert_transcription_complete_span(spans[0], response.text, start, end, check_content_type=True) - - -@pytest.mark.vcr @pytest.mark.asyncio -async def test_wrap_mistral_audio_transcriptions_complete_async(memory_logger): +@pytest.mark.parametrize( + ("mistral_client", "method_name"), + [ + pytest.param( + "wrap", + "complete", + id="sync-wrap", + marks=pytest.mark.vcr(cassette_name="test_wrap_mistral_audio_transcriptions_complete_sync"), + ), + pytest.param( + "setup", + "complete", + id="sync-setup", + marks=pytest.mark.vcr(cassette_name="test_wrap_mistral_audio_transcriptions_complete_sync"), + ), + pytest.param( + "wrap", + "complete_async", + id="async-wrap", + marks=pytest.mark.vcr(cassette_name="test_wrap_mistral_audio_transcriptions_complete_async"), + ), + ], + indirect=["mistral_client"], +) +async def test_mistral_audio_transcriptions_complete(memory_logger, mistral_client, method_name): assert not memory_logger.pop() - client = wrap_mistral(_get_client()) with TEST_AUDIO_FILE.open("rb") as file_obj: start = time.time() - response = await client.audio.transcriptions.complete_async( + response = getattr(mistral_client.audio.transcriptions, method_name)( model=AUDIO_TRANSCRIPTION_MODEL, file=_transcription_file_payload(file_obj), ) + if inspect.isawaitable(response): + response = await response end = time.time() spans = memory_logger.pop() @@ -968,33 +997,21 @@ def test_wrap_mistral_audio_transcriptions_stream_sync(memory_logger): assert_metrics_are_valid(span["metrics"], start, end) -@pytest.mark.vcr -def test_wrap_mistral_audio_speech_complete_sync(memory_logger): - if Speech is None: - pytest.skip("Mistral speech API is unavailable in this SDK version") - - assert not memory_logger.pop() - - client = wrap_mistral(_get_client()) - start = time.time() - response = client.audio.speech.complete( - input="Hello from Braintrust.", - model=SPEECH_MODEL, - response_format="mp3", - voice_id=SPEECH_VOICE_ID, - ) - end = time.time() - - assert response.audio_data - - spans = memory_logger.pop() - assert len(spans) == 1 - _assert_speech_complete_span(spans[0], start, end) - - -@pytest.mark.vcr @pytest.mark.asyncio -async def test_wrap_mistral_audio_speech_complete_async(memory_logger): +@pytest.mark.parametrize( + "method_name", + [ + pytest.param( + "complete", id="sync", marks=pytest.mark.vcr(cassette_name="test_wrap_mistral_audio_speech_complete_sync") + ), + pytest.param( + "complete_async", + id="async", + marks=pytest.mark.vcr(cassette_name="test_wrap_mistral_audio_speech_complete_async"), + ), + ], +) +async def test_wrap_mistral_audio_speech_complete(memory_logger, method_name): if Speech is None: pytest.skip("Mistral speech API is unavailable in this SDK version") @@ -1002,12 +1019,14 @@ async def test_wrap_mistral_audio_speech_complete_async(memory_logger): client = wrap_mistral(_get_client()) start = time.time() - response = await client.audio.speech.complete_async( + response = getattr(client.audio.speech, method_name)( input="Hello from Braintrust.", model=SPEECH_MODEL, response_format="mp3", voice_id=SPEECH_VOICE_ID, ) + if inspect.isawaitable(response): + response = await response end = time.time() assert response.audio_data @@ -1017,62 +1036,6 @@ async def test_wrap_mistral_audio_speech_complete_async(memory_logger): _assert_speech_complete_span(spans[0], start, end) -@pytest.mark.vcr(cassette_name="test_wrap_mistral_audio_transcriptions_complete_sync") -def test_mistral_integration_setup_instruments_audio_transcriptions(memory_logger, monkeypatch): - assert not memory_logger.pop() - - original_audio_methods = _audio_method_refs() - - assert MistralIntegration.setup() - client = _get_client() - try: - with TEST_AUDIO_FILE.open("rb") as file_obj: - start = time.time() - response = client.audio.transcriptions.complete( - model=AUDIO_TRANSCRIPTION_MODEL, - file=_transcription_file_payload(file_obj), - ) - end = time.time() - finally: - _restore_method_refs(monkeypatch, original_audio_methods) - - spans = memory_logger.pop() - assert len(spans) == 1 - _assert_transcription_complete_span(spans[0], response.text, start, end) - - -@pytest.mark.vcr -def test_mistral_integration_setup_instruments_beta_conversations(memory_logger, monkeypatch): - assert not memory_logger.pop() - - original_conversation_methods = _conversation_method_refs() - - assert MistralIntegration.setup() - client = _get_client() - try: - start = time.time() - response = client.beta.conversations.start( - model=CHAT_MODEL, - inputs="What is 4+4? Reply with just the number.", - ) - end = time.time() - finally: - _restore_method_refs(monkeypatch, original_conversation_methods) - - assert response.outputs - assert "8" in str(response.outputs[0].content) - - spans = memory_logger.pop() - assert len(spans) == 1 - _assert_conversation_span( - spans[0], - "What is 4+4? Reply with just the number.", - start, - end, - expected_content="8", - ) - - def test_chat_complete_wrapper_logs_errors(memory_logger): assert not memory_logger.pop() @@ -1281,6 +1244,5 @@ def test_aggregate_completion_events_merges_tool_calls_and_content(): assert tool_call["function"]["arguments"] == '{"city":"San Francisco"}' -class TestAutoInstrumentMistral: - def test_auto_instrument_mistral(self): - verify_autoinstrument_script("test_auto_mistral.py") +def test_auto_instrument_mistral(): + verify_autoinstrument_script("test_auto_mistral.py") diff --git a/py/src/braintrust/integrations/mistral/tracing.py b/py/src/braintrust/integrations/mistral/tracing.py index b8b2842ae..e4ec84360 100644 --- a/py/src/braintrust/integrations/mistral/tracing.py +++ b/py/src/braintrust/integrations/mistral/tracing.py @@ -49,6 +49,7 @@ def start_span(*args, **kwargs): "temperature", "top_p", "max_tokens", + "min_tokens", "stop", "random_seed", "response_format", @@ -56,6 +57,12 @@ def start_span(*args, **kwargs): "tool_choice", "presence_penalty", "frequency_penalty", + "repetition_penalty", + "top_k", + "logprobs", + "top_logprobs", + "prompt_logprobs", + "top_prompt_logprobs", "n", "prediction", "parallel_tool_calls", @@ -68,6 +75,7 @@ def start_span(*args, **kwargs): "agent_id", "service_tier", "max_tokens", + "min_tokens", "stop", "random_seed", "response_format", @@ -75,6 +83,12 @@ def start_span(*args, **kwargs): "tool_choice", "presence_penalty", "frequency_penalty", + "repetition_penalty", + "top_k", + "logprobs", + "top_logprobs", + "prompt_logprobs", + "top_prompt_logprobs", "n", "prediction", "parallel_tool_calls", @@ -783,6 +797,10 @@ def _aggregate_completion_events(items: list[Any]) -> dict[str, Any]: }, ) message = accumulated["message"] + for key in ("logprobs", "prompt_logprobs"): + logprobs = _normalized_mistral_dict(_get_value(choice, key)) + if logprobs is not None: + accumulated.setdefault(key, {"content": []})["content"].extend(logprobs.get("content") or []) delta = getattr(choice, "delta", None) if delta is None: continue From 0047103e661772a5ad4874daf8dc17c511bd4696 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Tue, 13 Oct 2026 11:32:32 -0400 Subject: [PATCH 2/2] test(mistral): record logprobs against a supported model --- ...est_wrap_mistral_agents_complete_sync.yaml | 125 ++++++++----- .../test_wrap_mistral_chat_complete_sync.yaml | 164 +++++++++++++++--- .../test_wrap_mistral_chat_stream_sync.yaml | 140 +++++++++++++-- .../integrations/mistral/test_mistral.py | 62 ++++--- 4 files changed, 386 insertions(+), 105 deletions(-) diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_sync.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_sync.yaml index b32afef65..895c9be9e 100644 --- a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_sync.yaml +++ b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_agents_complete_sync.yaml @@ -1,27 +1,27 @@ interactions: - request: - body: '{"model":"mistral-small-latest","name":"braintrust-test-agent-1785860160631","instructions":"You + body: '{"model":"zai-glm-5-3","name":"braintrust-test-agent-1791905193535","instructions":"You are concise. Keep responses under five words."}' headers: accept: - application/json accept-encoding: - - gzip, deflate + - gzip, deflate, zstd connection: - keep-alive content-length: - - '144' + - '135' content-type: - application/json host: - api.mistral.ai user-agent: - - mistral-client-python/2.9.0 + - mistral-client-python/3.2.0 method: POST uri: https://api.mistral.ai/v1/agents response: body: - string: '{"instructions":"You are concise. Keep responses under five words.","tools":[],"completion_args":{"stop":null,"presence_penalty":null,"frequency_penalty":null,"temperature":null,"top_p":null,"max_tokens":null,"random_seed":null,"prediction":null,"response_format":null,"tool_choice":"auto","reasoning_effort":null},"guardrails":[],"model":"mistral-small-latest","name":"braintrust-test-agent-1785860160631","description":null,"handoffs":null,"metadata":null,"object":"agent","id":"ag_019fcd8f7d9d7606a9c031bae03d4b78","version":0,"versions":[],"created_at":"2026-08-04T16:16:00.947141Z","updated_at":"2026-08-04T16:16:00.947143Z","deployment_chat":false,"source":"api","version_message":null}' + string: '{"instructions":"You are concise. Keep responses under five words.","tools":[],"completion_args":{"stop":null,"presence_penalty":null,"frequency_penalty":null,"temperature":null,"top_p":null,"max_tokens":null,"random_seed":null,"prediction":null,"response_format":null,"tool_choice":"auto","reasoning_effort":null},"guardrails":[],"model":"zai-glm-5-3","name":"braintrust-test-agent-1791905193535","description":null,"handoffs":null,"metadata":null,"object":"agent","id":"ag_01a135df5fc273f5b16cbdd1bfe55ac7","version":0,"versions":[],"owner_id":"4cf0889c-e6bd-4aba-a664-2e25c0dcd24e","created_at":"2026-10-13T15:26:33.950795Z","updated_at":"2026-10-13T15:26:33.950798Z","deployment_chat":false,"source":"api","version_message":null}' headers: access-control-allow-origin: - '*' @@ -30,17 +30,17 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a25edd35194ff8ac-YYZ + - a49f5d048e97ac28-YYZ connection: - keep-alive content-length: - - '692' + - '733' content-type: - application/json date: - - Tue, 04 Aug 2026 16:16:01 GMT + - Tue, 13 Oct 2026 15:26:34 GMT mistral-correlation-id: - - 019fcd8f-7d6f-7c6e-97d3-5b02736b2232 + - 01a135df-5f9b-7cbb-961a-62f71dfc2674 server: - cloudflare strict-transport-security: @@ -54,34 +54,79 @@ interactions: x-kong-proxy-latency: - '14' x-kong-request-id: - - 019fcd8f-7d6f-7c6e-97d3-5b02736b2232 + - 01a135df-5f9b-7cbb-961a-62f71dfc2674 x-kong-upstream-latency: - - '82' + - '81' status: code: 200 message: OK - request: - body: '{"messages":[{"content":"What is 8+1? Reply with just the number.","role":"user"}],"agent_id":"ag_019fcd8f7d9d7606a9c031bae03d4b78","max_tokens":10,"stream":false}' + body: '{"messages":[{"content":"What is 8+1? Reply with just the number.","role":"user"}],"agent_id":"ag_01a135df5fc273f5b16cbdd1bfe55ac7","max_tokens":256,"stream":false,"min_tokens":1,"repetition_penalty":1.1,"top_k":20,"logprobs":true,"top_logprobs":2,"prompt_logprobs":true,"top_prompt_logprobs":2}' headers: accept: - application/json accept-encoding: - - gzip, deflate + - gzip, deflate, zstd connection: - keep-alive content-length: - - '163' + - '295' content-type: - application/json host: - api.mistral.ai user-agent: - - mistral-client-python/2.9.0 + - mistral-client-python/3.2.0 method: POST uri: https://api.mistral.ai/v1/agents/completions response: body: - string: '{"id":"4023125663914af6a560ac26667a2ec1","created":1785860161,"model":"mistral-small-latest","usage":{"prompt_tokens":40,"total_tokens":42,"completion_tokens":2,"prompt_tokens_details":{"cached_tokens":0},"service_tier":"standard"},"object":"chat.completion","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","tool_calls":null,"content":"9"}}]}' + string: '{"id":"8e508ec5ea8e4314a42e6b498a4458bb","created":1791905194,"model":"zai-glm-5-3","usage":{"prompt_tokens":36,"total_tokens":39,"completion_tokens":3,"prompt_tokens_details":{"cached_tokens":0}},"object":"chat.completion","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","index":0,"content":"9"},"logprobs":{"content":[{"token":"","logprob":-0.0913877934217453,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1,"top_logprobs":[{"token":"","logprob":-0.0913877934217453,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1},{"token":"The","logprob":-3.0913877487182617,"bytes":[84,104,101],"token_id":-1}]},{"token":"9","logprob":-0.0003773453936446458,"bytes":[57],"token_id":-1,"top_logprobs":[{"token":"9","logprob":-0.0003773453936446458,"bytes":[57],"token_id":-1},{"token":"8","logprob":-9.000377655029297,"bytes":[56],"token_id":-1}]},{"token":"<|user|>","logprob":-8.713819261174649e-05,"bytes":[60,124,117,115,101,114,124,62],"token_id":-1,"top_logprobs":[{"token":"<|user|>","logprob":-8.713819261174649e-05,"bytes":[60,124,117,115,101,114,124,62],"token_id":-1},{"token":"\n\n","logprob":-10.375086784362793,"bytes":[10,10],"token_id":-1}]}]},"prompt_logprobs":{"content":[{"token":"","logprob":-25.0937557220459,"bytes":[],"token_id":154824,"top_logprobs":[{"token":"[gMASK]","logprob":-6.318072337307967e-06,"bytes":[],"token_id":154822},{"token":" + comments","logprob":-18.6875057220459,"bytes":[],"token_id":6042}]},{"token":"<|system|>","logprob":-19.04717254638672,"bytes":[],"token_id":154826,"top_logprobs":[{"token":"<|endoftext|>","logprob":-0.10967207700014114,"bytes":[],"token_id":154820},{"token":"##","logprob":-4.5784220695495605,"bytes":[],"token_id":565}]},{"token":"Reason","logprob":-14.288257598876953,"bytes":[],"token_id":25062,"top_logprobs":[{"token":"\n","logprob":-2.2921640872955322,"bytes":[],"token_id":198},{"token":"\n\n","logprob":-3.2921640872955322,"bytes":[],"token_id":271}]},{"token":"ing","logprob":-0.37178710103034973,"bytes":[],"token_id":287,"top_logprobs":[{"token":"ably","logprob":-3.2467870712280273,"bytes":[],"token_id":2845}]},{"token":" + Eff","logprob":-11.087068557739258,"bytes":[],"token_id":29905,"top_logprobs":[{"token":" + with","logprob":-3.7042555809020996,"bytes":[],"token_id":448},{"token":".","logprob":-3.7042555809020996,"bytes":[],"token_id":13}]},{"token":"ort","logprob":-4.112343788146973,"bytes":[],"token_id":371,"top_logprobs":[{"token":" + Level","logprob":-2.8623440265655518,"bytes":[],"token_id":9391},{"token":" + Over","logprob":-3.3623440265655518,"bytes":[],"token_id":6065}]},{"token":":","logprob":-4.009695053100586,"bytes":[],"token_id":25,"top_logprobs":[{"token":" + final","logprob":-1.8846948146820068,"bytes":[],"token_id":1590},{"token":" + Answer","logprob":-2.072194814682007,"bytes":[],"token_id":21754}]},{"token":" + Max","logprob":-10.907561302185059,"bytes":[],"token_id":7487,"top_logprobs":[{"token":" + lon","logprob":-4.266936302185059,"bytes":[],"token_id":17780},{"token":" + ","logprob":-4.641936302185059,"bytes":[],"token_id":220}]},{"token":"<|system|>","logprob":-14.329546928405762,"bytes":[],"token_id":154826,"top_logprobs":[{"token":"\n","logprob":-0.8080625534057617,"bytes":[],"token_id":198},{"token":",","logprob":-3.3080625534057617,"bytes":[],"token_id":11}]},{"token":"You","logprob":-8.809078216552734,"bytes":[],"token_id":2610,"top_logprobs":[{"token":"","logprob":-1.2465780973434448,"bytes":[],"token_id":154842},{"token":"The","logprob":-3.6215782165527344,"bytes":[],"token_id":785}]},{"token":" + are","logprob":-4.619465351104736,"bytes":[],"token_id":525,"top_logprobs":[{"token":"''re","logprob":-0.6194652915000916,"bytes":[],"token_id":2299},{"token":" + need","logprob":-1.4944653511047363,"bytes":[],"token_id":1184}]},{"token":" + concise","logprob":-12.689486503601074,"bytes":[],"token_id":63141,"top_logprobs":[{"token":" + asking","logprob":-1.2832363843917847,"bytes":[],"token_id":10156},{"token":" + asked","logprob":-1.4082363843917847,"bytes":[],"token_id":4588}]},{"token":".","logprob":-2.7664852142333984,"bytes":[],"token_id":13,"top_logprobs":[{"token":" + in","logprob":-0.8914852142333984,"bytes":[],"token_id":304},{"token":" by","logprob":-1.2664852142333984,"bytes":[],"token_id":553}]},{"token":" + Keep","logprob":-4.823254108428955,"bytes":[],"token_id":13646,"top_logprobs":[{"token":" + You","logprob":-0.5107541084289551,"bytes":[],"token_id":1446},{"token":" + Your","logprob":-3.010754108428955,"bytes":[],"token_id":4615}]},{"token":" + responses","logprob":-2.6773171424865723,"bytes":[],"token_id":14497,"top_logprobs":[{"token":" + your","logprob":-1.0523171424865723,"bytes":[],"token_id":697},{"token":" + the","logprob":-1.1773171424865723,"bytes":[],"token_id":279}]},{"token":" + under","logprob":-9.26633071899414,"bytes":[],"token_id":1212,"top_logprobs":[{"token":" + short","logprob":-0.07883025705814362,"bytes":[],"token_id":2805},{"token":" + brief","logprob":-3.0788302421569824,"bytes":[],"token_id":9809}]},{"token":" + five","logprob":-8.460169792175293,"bytes":[],"token_id":4236,"top_logprobs":[{"token":" + ","logprob":-0.02267015166580677,"bytes":[],"token_id":220},{"token":" ~","logprob":-4.647670269012451,"bytes":[],"token_id":3968}]},{"token":" + words","logprob":-4.671149730682373,"bytes":[],"token_id":4244,"top_logprobs":[{"token":" + sentences","logprob":-0.10864964872598648,"bytes":[],"token_id":22809},{"token":" + lines","logprob":-2.733649730682373,"bytes":[],"token_id":5128}]},{"token":".","logprob":-0.9434095621109009,"bytes":[],"token_id":13,"top_logprobs":[{"token":" + when","logprob":-1.3184095621109009,"bytes":[],"token_id":979}]},{"token":"<|user|>","logprob":-8.329792976379395,"bytes":[],"token_id":154827,"top_logprobs":[{"token":"","logprob":-0.8922933340072632,"bytes":[],"token_id":154842},{"token":" + Under","logprob":-2.4547934532165527,"bytes":[],"token_id":9445}]},{"token":"What","logprob":-7.348790168762207,"bytes":[],"token_id":3838,"top_logprobs":[{"token":"","logprob":-1.4737902879714966,"bytes":[],"token_id":154842},{"token":"Let","logprob":-1.4737902879714966,"bytes":[],"token_id":10056}]},{"token":" + is","logprob":-0.7880374789237976,"bytes":[],"token_id":374,"top_logprobs":[{"token":" + are","logprob":-1.9130375385284424,"bytes":[],"token_id":525}]},{"token":" + ","logprob":-2.6452090740203857,"bytes":[],"token_id":220,"top_logprobs":[{"token":" + the","logprob":-0.39520910382270813,"bytes":[],"token_id":279}]},{"token":"8","logprob":-4.051606178283691,"bytes":[],"token_id":23,"top_logprobs":[{"token":"2","logprob":-0.8016062378883362,"bytes":[],"token_id":17},{"token":"1","logprob":-2.4266061782836914,"bytes":[],"token_id":16}]},{"token":"+","logprob":-2.001539945602417,"bytes":[],"token_id":10,"top_logprobs":[{"token":" + divided","logprob":-1.626539945602417,"bytes":[],"token_id":17753}]},{"token":"1","logprob":-3.6469202041625977,"bytes":[],"token_id":16,"top_logprobs":[{"token":"3","logprob":-1.1469203233718872,"bytes":[],"token_id":18},{"token":"2","logprob":-1.7719203233718872,"bytes":[],"token_id":17}]},{"token":"?","logprob":-3.257967233657837,"bytes":[],"token_id":30,"top_logprobs":[{"token":"+","logprob":-1.132967233657837,"bytes":[],"token_id":10},{"token":"-","logprob":-1.757967233657837,"bytes":[],"token_id":12}]},{"token":" + Reply","logprob":-12.022690773010254,"bytes":[],"token_id":17815,"top_logprobs":[{"token":"","logprob":-0.022690312936902046,"bytes":[],"token_id":154842},{"token":"9","logprob":-4.897690296173096,"bytes":[],"token_id":24}]},{"token":" + with","logprob":-1.343369483947754,"bytes":[],"token_id":448,"top_logprobs":[{"token":" + in","logprob":-1.218369483947754,"bytes":[],"token_id":304}]},{"token":" just","logprob":-1.7867549657821655,"bytes":[],"token_id":1101,"top_logprobs":[{"token":" + only","logprob":-1.4117549657821655,"bytes":[],"token_id":1172},{"token":" + a","logprob":-1.5367549657821655,"bytes":[],"token_id":264}]},{"token":" the","logprob":-0.1841922253370285,"bytes":[],"token_id":279,"top_logprobs":[{"token":" + your","logprob":-2.684192180633545,"bytes":[],"token_id":697}]},{"token":" + number","logprob":-0.9484573006629944,"bytes":[],"token_id":1372,"top_logprobs":[{"token":" + answer","logprob":-0.5734573006629944,"bytes":[],"token_id":4226}]},{"token":".","logprob":-0.3525479733943939,"bytes":[],"token_id":13,"top_logprobs":[{"token":".\n\n","logprob":-1.7275480031967163,"bytes":[],"token_id":382}]},{"token":"<|assistant|>","logprob":-12.56834888458252,"bytes":[],"token_id":154828,"top_logprobs":[{"token":"","logprob":-0.00584891252219677,"bytes":[],"token_id":154842},{"token":"9","logprob":-7.0683488845825195,"bytes":[],"token_id":24}]},{"token":"","logprob":-17.316312789916992,"bytes":[],"token_id":154841,"top_logprobs":[{"token":"","logprob":-0.06631205976009369,"bytes":[],"token_id":154842},{"token":"9","logprob":-2.941312074661255,"bytes":[],"token_id":24}]}]},"response_error":{"format_error":null,"content_validation":["valid_text_chunk"],"tool_calls_validation":[],"extra_tags":[]}}]}' headers: access-control-allow-origin: - '*' @@ -90,17 +135,17 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a25edd368df0f8ac-YYZ + - a49f5d06c976ac28-YYZ connection: - keep-alive content-length: - - '369' + - '9063' content-type: - application/json date: - - Tue, 04 Aug 2026 16:16:01 GMT + - Tue, 13 Oct 2026 15:26:35 GMT mistral-correlation-id: - - 019fcd8f-7e5f-7825-8899-55903e98cb67 + - 01a135df-60a0-7d73-8168-0ae619daa745 server: - cloudflare strict-transport-security: @@ -110,25 +155,23 @@ interactions: x-content-type-options: - nosniff x-envoy-upstream-service-time: - - '753' + - '836' x-kong-proxy-latency: - - '16' + - '18' x-kong-request-id: - - 019fcd8f-7e5f-7825-8899-55903e98cb67 + - 01a135df-60a0-7d73-8168-0ae619daa745 x-kong-upstream-latency: - - '754' - x-max-retry-attempts-reached: - - 'false' + - '838' x-ratelimit-limit-req-minute: - - '50' + - '1000' x-ratelimit-limit-tokens-minute: - - '50000' + - '32000000' x-ratelimit-remaining-req-minute: - - '49' + - '997' x-ratelimit-remaining-tokens-minute: - - '49533' + - '31999867' x-ratelimit-tokens-query-cost: - - '42' + - '39' status: code: 200 message: OK @@ -138,15 +181,15 @@ interactions: accept: - application/json accept-encoding: - - gzip, deflate + - gzip, deflate, zstd connection: - keep-alive host: - api.mistral.ai user-agent: - - mistral-client-python/2.9.0 + - mistral-client-python/3.2.0 method: DELETE - uri: https://api.mistral.ai/v1/agents/ag_019fcd8f7d9d7606a9c031bae03d4b78 + uri: https://api.mistral.ai/v1/agents/ag_01a135df5fc273f5b16cbdd1bfe55ac7 response: body: string: '' @@ -158,13 +201,13 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a25edd3cc905f8ac-YYZ + - a49f5d0e2b37ac28-YYZ connection: - keep-alive date: - - Tue, 04 Aug 2026 16:16:02 GMT + - Tue, 13 Oct 2026 15:26:35 GMT mistral-correlation-id: - - 019fcd8f-8235-7d58-8ddf-f8f158612e63 + - 01a135df-6520-7723-9cec-795bfc771b35 server: - cloudflare strict-transport-security: @@ -172,13 +215,13 @@ interactions: x-content-type-options: - nosniff x-envoy-upstream-service-time: - - '121' + - '91' x-kong-proxy-latency: - - '10' + - '14' x-kong-request-id: - - 019fcd8f-8235-7d58-8ddf-f8f158612e63 + - 01a135df-6520-7723-9cec-795bfc771b35 x-kong-upstream-latency: - - '122' + - '94' status: code: 204 message: No Content diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_sync.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_sync.yaml index 69f7fd709..5849019f0 100644 --- a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_sync.yaml +++ b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_complete_sync.yaml @@ -1,27 +1,151 @@ interactions: - request: - body: '{"model":"mistral-small-latest","messages":[{"content":"What is 2+2? Reply - with just the number.","role":"user"}],"max_tokens":10,"stream":false}' + body: '{"model":"zai-glm-5-3","messages":[{"content":"What is 2+2? Reply with + just the number.","role":"user"}],"max_tokens":256,"stream":false,"min_tokens":1,"repetition_penalty":1.1,"top_k":20,"logprobs":true,"top_logprobs":2,"prompt_logprobs":true,"top_prompt_logprobs":2}' headers: accept: - application/json accept-encoding: - - gzip, deflate + - gzip, deflate, zstd connection: - keep-alive content-length: - - '145' + - '268' content-type: - application/json host: - api.mistral.ai user-agent: - - mistral-client-python/2.9.0 + - mistral-client-python/3.2.0 method: POST uri: https://api.mistral.ai/v1/chat/completions response: body: - string: '{"id":"3cc9fec7b43a4465afd589cb3ad74192","created":1785860145,"model":"mistral-small-latest","usage":{"prompt_tokens":28,"total_tokens":30,"completion_tokens":2,"prompt_tokens_details":{"cached_tokens":0},"service_tier":"standard"},"object":"chat.completion","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","tool_calls":null,"content":"4"}}]}' + string: '{"id":"4078d421d467439ba16718dd36c4ac23","created":1791905192,"model":"zai-glm-5-3","usage":{"prompt_tokens":25,"total_tokens":78,"completion_tokens":53,"prompt_tokens_details":{"cached_tokens":0}},"object":"chat.completion","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","index":0,"content":[{"type":"thinking","thinking":[{"type":"text","text":"The + user asks \"What is 2+2?\" and wants me to reply with just the number.\n\n2+2 + = 4\n\nThey explicitly asked for just the number, so I should respond with + only \"4\" without any additional text."}],"closed":true},{"type":"text","text":"4"}]},"logprobs":{"content":[{"token":"The","logprob":-0.0167945995926857,"bytes":[84,104,101],"token_id":-1,"top_logprobs":[{"token":"The","logprob":-0.0167945995926857,"bytes":[84,104,101],"token_id":-1},{"token":"This","logprob":-4.141794681549072,"bytes":[84,104,105,115],"token_id":-1}]},{"token":" + user","logprob":-0.004732123576104641,"bytes":[32,117,115,101,114],"token_id":-1,"top_logprobs":[{"token":" + user","logprob":-0.004732123576104641,"bytes":[32,117,115,101,114],"token_id":-1},{"token":" + question","logprob":-5.379732131958008,"bytes":[32,113,117,101,115,116,105,111,110],"token_id":-1}]},{"token":" + asks","logprob":-1.4847524166107178,"bytes":[32,97,115,107,115],"token_id":-1,"top_logprobs":[{"token":" + asks","logprob":-1.4847524166107178,"bytes":[32,97,115,107,115],"token_id":-1},{"token":" + is","logprob":-0.3597524166107178,"bytes":[32,105,115],"token_id":-1}]},{"token":" + \"","logprob":-1.2936851978302002,"bytes":[32,34],"token_id":-1,"top_logprobs":[{"token":" + \"","logprob":-1.2936851978302002,"bytes":[32,34],"token_id":-1},{"token":" + a","logprob":-0.6686851978302002,"bytes":[32,97],"token_id":-1}]},{"token":"What","logprob":-0.0002317160106031224,"bytes":[87,104,97,116],"token_id":-1,"top_logprobs":[{"token":"What","logprob":-0.0002317160106031224,"bytes":[87,104,97,116],"token_id":-1},{"token":"2","logprob":-8.500231742858887,"bytes":[50],"token_id":-1}]},{"token":" + is","logprob":-1.9073468138230965e-06,"bytes":[32,105,115],"token_id":-1,"top_logprobs":[{"token":" + is","logprob":-1.9073468138230965e-06,"bytes":[32,105,115],"token_id":-1},{"token":" + ","logprob":-13.625001907348633,"bytes":[32],"token_id":-1}]},{"token":" ","logprob":-8.344646857949556e-07,"bytes":[32],"token_id":-1,"top_logprobs":[{"token":" + ","logprob":-8.344646857949556e-07,"bytes":[32],"token_id":-1},{"token":" + the","logprob":-14.500000953674316,"bytes":[32,116,104,101],"token_id":-1}]},{"token":"2","logprob":-2.7418097943154862e-06,"bytes":[50],"token_id":-1,"top_logprobs":[{"token":"2","logprob":-2.7418097943154862e-06,"bytes":[50],"token_id":-1},{"token":"4","logprob":-14.43750286102295,"bytes":[52],"token_id":-1}]},{"token":"+","logprob":-1.9073468138230965e-06,"bytes":[43],"token_id":-1,"top_logprobs":[{"token":"+","logprob":-1.9073468138230965e-06,"bytes":[43],"token_id":-1},{"token":" + +","logprob":-13.500001907348633,"bytes":[32,43],"token_id":-1}]},{"token":"2","logprob":-2.0265558760002023e-06,"bytes":[50],"token_id":-1,"top_logprobs":[{"token":"2","logprob":-2.0265558760002023e-06,"bytes":[50],"token_id":-1},{"token":"?\"","logprob":-14.562501907348633,"bytes":[63,34],"token_id":-1}]},{"token":"?\"","logprob":-1.2238450050354004,"bytes":[63,34],"token_id":-1,"top_logprobs":[{"token":"?\"","logprob":-1.2238450050354004,"bytes":[63,34],"token_id":-1},{"token":"?","logprob":-0.34884506464004517,"bytes":[63],"token_id":-1}]},{"token":" + and","logprob":-0.0008503158460371196,"bytes":[32,97,110,100],"token_id":-1,"top_logprobs":[{"token":" + and","logprob":-0.0008503158460371196,"bytes":[32,97,110,100],"token_id":-1},{"token":" + with","logprob":-7.375850200653076,"bytes":[32,119,105,116,104],"token_id":-1}]},{"token":" + wants","logprob":-1.280053734779358,"bytes":[32,119,97,110,116,115],"token_id":-1,"top_logprobs":[{"token":" + wants","logprob":-1.280053734779358,"bytes":[32,119,97,110,116,115],"token_id":-1},{"token":" + explicitly","logprob":-0.9050537347793579,"bytes":[32,101,120,112,108,105,99,105,116,108,121],"token_id":-1}]},{"token":" + me","logprob":-0.43099257349967957,"bytes":[32,109,101],"token_id":-1,"top_logprobs":[{"token":" + me","logprob":-0.43099257349967957,"bytes":[32,109,101],"token_id":-1},{"token":" + just","logprob":-1.055992603302002,"bytes":[32,106,117,115,116],"token_id":-1}]},{"token":" + to","logprob":-2.3841855067985307e-07,"bytes":[32,116,111],"token_id":-1,"top_logprobs":[{"token":" + to","logprob":-2.3841855067985307e-07,"bytes":[32,116,111],"token_id":-1},{"token":" + \"","logprob":-15.75,"bytes":[32,34],"token_id":-1}]},{"token":" reply","logprob":-0.0007908792467787862,"bytes":[32,114,101,112,108,121],"token_id":-1,"top_logprobs":[{"token":" + reply","logprob":-0.0007908792467787862,"bytes":[32,114,101,112,108,121],"token_id":-1},{"token":" + \"","logprob":-7.250791072845459,"bytes":[32,34],"token_id":-1}]},{"token":" + with","logprob":-2.3841830625315197e-06,"bytes":[32,119,105,116,104],"token_id":-1,"top_logprobs":[{"token":" + with","logprob":-2.3841830625315197e-06,"bytes":[32,119,105,116,104],"token_id":-1},{"token":" + \"","logprob":-13.000001907348633,"bytes":[32,34],"token_id":-1}]},{"token":" + just","logprob":-2.777537883957848e-05,"bytes":[32,106,117,115,116],"token_id":-1,"top_logprobs":[{"token":" + just","logprob":-2.777537883957848e-05,"bytes":[32,106,117,115,116],"token_id":-1},{"token":" + \"","logprob":-10.500027656555176,"bytes":[32,34],"token_id":-1}]},{"token":" + the","logprob":-1.5258672647178173e-05,"bytes":[32,116,104,101],"token_id":-1,"top_logprobs":[{"token":" + the","logprob":-1.5258672647178173e-05,"bytes":[32,116,104,101],"token_id":-1},{"token":" + a","logprob":-11.125015258789062,"bytes":[32,97],"token_id":-1}]},{"token":" + number","logprob":-3.814689989667386e-06,"bytes":[32,110,117,109,98,101,114],"token_id":-1,"top_logprobs":[{"token":" + number","logprob":-3.814689989667386e-06,"bytes":[32,110,117,109,98,101,114],"token_id":-1},{"token":" + answer","logprob":-12.750003814697266,"bytes":[32,97,110,115,119,101,114],"token_id":-1}]},{"token":".\n\n","logprob":-0.5235103368759155,"bytes":[46,10,10],"token_id":-1,"top_logprobs":[{"token":".\n\n","logprob":-0.5235103368759155,"bytes":[46,10,10],"token_id":-1},{"token":".","logprob":-0.8985103368759155,"bytes":[46],"token_id":-1}]},{"token":"2","logprob":-0.11548171937465668,"bytes":[50],"token_id":-1,"top_logprobs":[{"token":"2","logprob":-0.11548171937465668,"bytes":[50],"token_id":-1},{"token":"The","logprob":-2.4904816150665283,"bytes":[84,104,101],"token_id":-1}]},{"token":"+","logprob":-0.043050725013017654,"bytes":[43],"token_id":-1,"top_logprobs":[{"token":"+","logprob":-0.043050725013017654,"bytes":[43],"token_id":-1},{"token":" + +","logprob":-3.168050765991211,"bytes":[32,43],"token_id":-1}]},{"token":"2","logprob":-1.311301275563892e-06,"bytes":[50],"token_id":-1,"top_logprobs":[{"token":"2","logprob":-1.311301275563892e-06,"bytes":[50],"token_id":-1},{"token":"4","logprob":-14.625000953674316,"bytes":[52],"token_id":-1}]},{"token":" + =","logprob":-0.003689507720991969,"bytes":[32,61],"token_id":-1,"top_logprobs":[{"token":" + =","logprob":-0.003689507720991969,"bytes":[32,61],"token_id":-1},{"token":"=","logprob":-5.628689289093018,"bytes":[61],"token_id":-1}]},{"token":" + ","logprob":-4.172316494077677e-06,"bytes":[32],"token_id":-1,"top_logprobs":[{"token":" + ","logprob":-4.172316494077677e-06,"bytes":[32],"token_id":-1},{"token":"","logprob":-13.500003814697266,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1}]},{"token":"4","logprob":-0.00017045476124621928,"bytes":[52],"token_id":-1,"top_logprobs":[{"token":"4","logprob":-0.00017045476124621928,"bytes":[52],"token_id":-1},{"token":"5","logprob":-9.625170707702637,"bytes":[53],"token_id":-1}]},{"token":"\n\n","logprob":-0.19981855154037476,"bytes":[10,10],"token_id":-1,"top_logprobs":[{"token":"\n\n","logprob":-0.19981855154037476,"bytes":[10,10],"token_id":-1},{"token":".\n\n","logprob":-2.0748186111450195,"bytes":[46,10,10],"token_id":-1}]},{"token":"They","logprob":-0.22272346913814545,"bytes":[84,104,101,121],"token_id":-1,"top_logprobs":[{"token":"They","logprob":-0.22272346913814545,"bytes":[84,104,101,121],"token_id":-1},{"token":"The","logprob":-1.7227234840393066,"bytes":[84,104,101],"token_id":-1}]},{"token":" + explicitly","logprob":-0.5044372081756592,"bytes":[32,101,120,112,108,105,99,105,116,108,121],"token_id":-1,"top_logprobs":[{"token":" + explicitly","logprob":-0.5044372081756592,"bytes":[32,101,120,112,108,105,99,105,116,108,121],"token_id":-1},{"token":" + want","logprob":-1.6294372081756592,"bytes":[32,119,97,110,116],"token_id":-1}]},{"token":" + asked","logprob":-0.1208973303437233,"bytes":[32,97,115,107,101,100],"token_id":-1,"top_logprobs":[{"token":" + asked","logprob":-0.1208973303437233,"bytes":[32,97,115,107,101,100],"token_id":-1},{"token":" + said","logprob":-2.7458972930908203,"bytes":[32,115,97,105,100],"token_id":-1}]},{"token":" + for","logprob":-0.5034964680671692,"bytes":[32,102,111,114],"token_id":-1,"top_logprobs":[{"token":" + for","logprob":-0.5034964680671692,"bytes":[32,102,111,114],"token_id":-1},{"token":" + to","logprob":-1.5034964084625244,"bytes":[32,116,111],"token_id":-1}]},{"token":" + just","logprob":-0.0005200940649956465,"bytes":[32,106,117,115,116],"token_id":-1,"top_logprobs":[{"token":" + just","logprob":-0.0005200940649956465,"bytes":[32,106,117,115,116],"token_id":-1},{"token":" + \"","logprob":-7.8755202293396,"bytes":[32,34],"token_id":-1}]},{"token":" + the","logprob":-5.960446742392378e-06,"bytes":[32,116,104,101],"token_id":-1,"top_logprobs":[{"token":" + the","logprob":-5.960446742392378e-06,"bytes":[32,116,104,101],"token_id":-1},{"token":" + a","logprob":-12.125005722045898,"bytes":[32,97],"token_id":-1}]},{"token":" + number","logprob":-2.145764938177308e-06,"bytes":[32,110,117,109,98,101,114],"token_id":-1,"top_logprobs":[{"token":" + number","logprob":-2.145764938177308e-06,"bytes":[32,110,117,109,98,101,114],"token_id":-1},{"token":" + answer","logprob":-13.375001907348633,"bytes":[32,97,110,115,119,101,114],"token_id":-1}]},{"token":",","logprob":-0.20216993987560272,"bytes":[44],"token_id":-1,"top_logprobs":[{"token":",","logprob":-0.20216993987560272,"bytes":[44],"token_id":-1},{"token":".","logprob":-1.7021698951721191,"bytes":[46],"token_id":-1}]},{"token":" + so","logprob":-0.0012226970866322517,"bytes":[32,115,111],"token_id":-1,"top_logprobs":[{"token":" + so","logprob":-0.0012226970866322517,"bytes":[32,115,111],"token_id":-1},{"token":" + no","logprob":-6.751222610473633,"bytes":[32,110,111],"token_id":-1}]},{"token":" + I","logprob":-0.018279075622558594,"bytes":[32,73],"token_id":-1,"top_logprobs":[{"token":" + I","logprob":-0.018279075622558594,"bytes":[32,73],"token_id":-1},{"token":" + my","logprob":-4.143279075622559,"bytes":[32,109,121],"token_id":-1}]},{"token":" + should","logprob":-9.655486064730212e-05,"bytes":[32,115,104,111,117,108,100],"token_id":-1,"top_logprobs":[{"token":" + should","logprob":-9.655486064730212e-05,"bytes":[32,115,104,111,117,108,100],"token_id":-1},{"token":"''ll","logprob":-9.375096321105957,"bytes":[39,108,108],"token_id":-1}]},{"token":" + respond","logprob":-1.8938047885894775,"bytes":[32,114,101,115,112,111,110,100],"token_id":-1,"top_logprobs":[{"token":" + respond","logprob":-1.8938047885894775,"bytes":[32,114,101,115,112,111,110,100],"token_id":-1},{"token":" + reply","logprob":-0.5188048481941223,"bytes":[32,114,101,112,108,121],"token_id":-1}]},{"token":" + with","logprob":-0.003153948113322258,"bytes":[32,119,105,116,104],"token_id":-1,"top_logprobs":[{"token":" + with","logprob":-0.003153948113322258,"bytes":[32,119,105,116,104],"token_id":-1},{"token":" + conc","logprob":-6.0031538009643555,"bytes":[32,99,111,110,99],"token_id":-1}]},{"token":" + only","logprob":-3.2448699474334717,"bytes":[32,111,110,108,121],"token_id":-1,"top_logprobs":[{"token":" + only","logprob":-3.2448699474334717,"bytes":[32,111,110,108,121],"token_id":-1},{"token":" + just","logprob":-0.2448698878288269,"bytes":[32,106,117,115,116],"token_id":-1}]},{"token":" + \"","logprob":-0.0023652929812669754,"bytes":[32,34],"token_id":-1,"top_logprobs":[{"token":" + \"","logprob":-0.0023652929812669754,"bytes":[32,34],"token_id":-1},{"token":" + the","logprob":-6.5023651123046875,"bytes":[32,116,104,101],"token_id":-1}]},{"token":"4","logprob":-2.634490556374658e-05,"bytes":[52],"token_id":-1,"top_logprobs":[{"token":"4","logprob":-2.634490556374658e-05,"bytes":[52],"token_id":-1},{"token":"2","logprob":-11.00002670288086,"bytes":[50],"token_id":-1}]},{"token":"\"","logprob":-0.34894657135009766,"bytes":[34],"token_id":-1,"top_logprobs":[{"token":"\"","logprob":-0.34894657135009766,"bytes":[34],"token_id":-1},{"token":"\".","logprob":-1.2239465713500977,"bytes":[34,46],"token_id":-1}]},{"token":" + without","logprob":-1.4770262241363525,"bytes":[32,119,105,116,104,111,117,116],"token_id":-1,"top_logprobs":[{"token":" + without","logprob":-1.4770262241363525,"bytes":[32,119,105,116,104,111,117,116],"token_id":-1},{"token":" + with","logprob":-1.1020262241363525,"bytes":[32,119,105,116,104],"token_id":-1}]},{"token":" + any","logprob":-0.02602219767868519,"bytes":[32,97,110,121],"token_id":-1,"top_logprobs":[{"token":" + any","logprob":-0.02602219767868519,"bytes":[32,97,110,121],"token_id":-1},{"token":" + additional","logprob":-3.77602219581604,"bytes":[32,97,100,100,105,116,105,111,110,97,108],"token_id":-1}]},{"token":" + additional","logprob":-0.0034779557026922703,"bytes":[32,97,100,100,105,116,105,111,110,97,108],"token_id":-1,"top_logprobs":[{"token":" + additional","logprob":-0.0034779557026922703,"bytes":[32,97,100,100,105,116,105,111,110,97,108],"token_id":-1},{"token":" + extra","logprob":-5.878478050231934,"bytes":[32,101,120,116,114,97],"token_id":-1}]},{"token":" + text","logprob":-0.041043832898139954,"bytes":[32,116,101,120,116],"token_id":-1,"top_logprobs":[{"token":" + text","logprob":-0.041043832898139954,"bytes":[32,116,101,120,116],"token_id":-1},{"token":" + explanation","logprob":-3.666043758392334,"bytes":[32,101,120,112,108,97,110,97,116,105,111,110],"token_id":-1}]},{"token":".","logprob":-0.06692969799041748,"bytes":[46],"token_id":-1,"top_logprobs":[{"token":".","logprob":-0.06692969799041748,"bytes":[46],"token_id":-1},{"token":",","logprob":-3.066929817199707,"bytes":[44],"token_id":-1}]},{"token":"","logprob":-0.01243952102959156,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1,"top_logprobs":[{"token":"","logprob":-0.01243952102959156,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1},{"token":" + This","logprob":-4.512439727783203,"bytes":[32,84,104,105,115],"token_id":-1}]},{"token":"4","logprob":-1.7404405298293568e-05,"bytes":[52],"token_id":-1,"top_logprobs":[{"token":"4","logprob":-1.7404405298293568e-05,"bytes":[52],"token_id":-1},{"token":"**","logprob":-11.875017166137695,"bytes":[42,42],"token_id":-1}]},{"token":"<|user|>","logprob":-8.940656698541716e-06,"bytes":[60,124,117,115,101,114,124,62],"token_id":-1,"top_logprobs":[{"token":"<|user|>","logprob":-8.940656698541716e-06,"bytes":[60,124,117,115,101,114,124,62],"token_id":-1},{"token":"","logprob":-25.0937557220459,"bytes":[],"token_id":154824,"top_logprobs":[{"token":"[gMASK]","logprob":-6.318072337307967e-06,"bytes":[],"token_id":154822},{"token":" + comments","logprob":-18.6875057220459,"bytes":[],"token_id":6042}]},{"token":"<|system|>","logprob":-19.102827072143555,"bytes":[],"token_id":154826,"top_logprobs":[{"token":"<|endoftext|>","logprob":-0.10282739251852036,"bytes":[],"token_id":154820},{"token":"##","logprob":-4.634077548980713,"bytes":[],"token_id":565}]},{"token":"Reason","logprob":-14.232568740844727,"bytes":[],"token_id":25062,"top_logprobs":[{"token":"\n","logprob":-2.3771002292633057,"bytes":[],"token_id":198},{"token":"\n\n","logprob":-3.5021002292633057,"bytes":[],"token_id":271}]},{"token":"ing","logprob":-0.20375163853168488,"bytes":[],"token_id":287,"top_logprobs":[{"token":"er","logprob":-3.453751564025879,"bytes":[],"token_id":261}]},{"token":" + Eff","logprob":-12.249364852905273,"bytes":[],"token_id":29905,"top_logprobs":[{"token":".","logprob":-3.1946768760681152,"bytes":[],"token_id":13},{"token":" + Criteria","logprob":-3.9446768760681152,"bytes":[],"token_id":14233}]},{"token":"ort","logprob":-3.629464864730835,"bytes":[],"token_id":371,"top_logprobs":[{"token":" + Level","logprob":-2.254464864730835,"bytes":[],"token_id":9391},{"token":"atory","logprob":-3.504464864730835,"bytes":[],"token_id":5269}]},{"token":":","logprob":-3.9242122173309326,"bytes":[],"token_id":25,"top_logprobs":[{"token":" + Answer","logprob":-1.8617122173309326,"bytes":[],"token_id":21754},{"token":" + final","logprob":-2.2992122173309326,"bytes":[],"token_id":1590}]},{"token":" + Max","logprob":-10.266630172729492,"bytes":[],"token_id":7487,"top_logprobs":[{"token":" + ","logprob":-4.688505172729492,"bytes":[],"token_id":220},{"token":"<|user|>","logprob":-4.844755172729492,"bytes":[],"token_id":154827}]},{"token":"<|user|>","logprob":-12.945687294006348,"bytes":[],"token_id":154827,"top_logprobs":[{"token":"\n","logprob":-0.5863124132156372,"bytes":[],"token_id":198},{"token":"ing","logprob":-3.0238122940063477,"bytes":[],"token_id":287}]},{"token":"What","logprob":-9.608269691467285,"bytes":[],"token_id":3838,"top_logprobs":[{"token":"","logprob":-1.1863950490951538,"bytes":[],"token_id":154842},{"token":"Let","logprob":-4.373895168304443,"bytes":[],"token_id":10056}]},{"token":" + is","logprob":-0.7828489542007446,"bytes":[],"token_id":374,"top_logprobs":[{"token":" + are","logprob":-1.7828489542007446,"bytes":[],"token_id":525}]},{"token":" + ","logprob":-3.202401638031006,"bytes":[],"token_id":220,"top_logprobs":[{"token":" + the","logprob":-0.32740169763565063,"bytes":[],"token_id":279},{"token":" + a","logprob":-2.952401638031006,"bytes":[],"token_id":264}]},{"token":"2","logprob":-1.7365621328353882,"bytes":[],"token_id":17,"top_logprobs":[{"token":"3","logprob":-2.4865622520446777,"bytes":[],"token_id":18}]},{"token":"+","logprob":-0.7275267839431763,"bytes":[],"token_id":10,"top_logprobs":[{"token":" + +","logprob":-1.4775267839431763,"bytes":[],"token_id":488}]},{"token":"2","logprob":-0.038262300193309784,"bytes":[],"token_id":17,"top_logprobs":[{"token":"3","logprob":-4.413262367248535,"bytes":[],"token_id":18}]},{"token":"?","logprob":-1.4470171928405762,"bytes":[],"token_id":30,"top_logprobs":[{"token":"?\n\n","logprob":-0.4470171332359314,"bytes":[],"token_id":1939}]},{"token":" + Reply","logprob":-11.040724754333496,"bytes":[],"token_id":17815,"top_logprobs":[{"token":" + ","logprob":-1.1032249927520752,"bytes":[],"token_id":220},{"token":"","logprob":-1.2282249927520752,"bytes":[],"token_id":154842}]},{"token":" + with","logprob":-1.31611967086792,"bytes":[],"token_id":448,"top_logprobs":[{"token":" + in","logprob":-0.9411197304725647,"bytes":[],"token_id":304}]},{"token":" + just","logprob":-1.563027262687683,"bytes":[],"token_id":1101,"top_logprobs":[{"token":" + a","logprob":-1.438027262687683,"bytes":[],"token_id":264}]},{"token":" the","logprob":-0.2866719961166382,"bytes":[],"token_id":279,"top_logprobs":[{"token":" + your","logprob":-2.1616721153259277,"bytes":[],"token_id":697}]},{"token":" + number","logprob":-1.7255685329437256,"bytes":[],"token_id":1372,"top_logprobs":[{"token":" + answer","logprob":-0.2255684733390808,"bytes":[],"token_id":4226}]},{"token":".","logprob":-1.6974256038665771,"bytes":[],"token_id":13,"top_logprobs":[{"token":".\n\n","logprob":-0.5724256038665771,"bytes":[],"token_id":382}]},{"token":"<|assistant|>","logprob":-10.813070297241211,"bytes":[],"token_id":154828,"top_logprobs":[{"token":"","logprob":-0.12557074427604675,"bytes":[],"token_id":154842},{"token":" + \n\n","logprob":-3.625570774078369,"bytes":[],"token_id":4710}]},{"token":"","logprob":-16.33489990234375,"bytes":[],"token_id":154841,"top_logprobs":[{"token":"","logprob":-0.49114957451820374,"bytes":[],"token_id":154842},{"token":"The","logprob":-0.9911495447158813,"bytes":[],"token_id":785}]}]},"response_error":{"format_error":null,"content_validation":["valid_think_chunk","valid_text_chunk"],"tool_calls_validation":[],"extra_tags":[]}}]}' headers: access-control-allow-origin: - '*' @@ -30,17 +154,17 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a25edcd60e5febb4-YYZ + - a49f5cf9ee98ac60-YYZ connection: - keep-alive content-length: - - '369' + - '20287' content-type: - application/json date: - - Tue, 04 Aug 2026 16:15:46 GMT + - Tue, 13 Oct 2026 15:26:33 GMT mistral-correlation-id: - - 019fcd8f-4204-7893-a32d-99f5c45e61a3 + - 01a135df-5880-79dc-b275-0691d22a8bf5 server: - cloudflare strict-transport-security: @@ -50,25 +174,23 @@ interactions: x-content-type-options: - nosniff x-envoy-upstream-service-time: - - '330' + - '853' x-kong-proxy-latency: - - '16' + - '14' x-kong-request-id: - - 019fcd8f-4204-7893-a32d-99f5c45e61a3 + - 01a135df-5880-79dc-b275-0691d22a8bf5 x-kong-upstream-latency: - - '332' - x-max-retry-attempts-reached: - - 'false' + - '854' x-ratelimit-limit-req-minute: - - '50' + - '1000' x-ratelimit-limit-tokens-minute: - - '50000' + - '32000000' x-ratelimit-remaining-req-minute: - - '49' + - '999' x-ratelimit-remaining-tokens-minute: - - '49970' + - '31999905' x-ratelimit-tokens-query-cost: - - '30' + - '78' status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_sync.yaml b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_sync.yaml index 91ea747bb..02a450960 100644 --- a/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_sync.yaml +++ b/py/src/braintrust/integrations/mistral/cassettes/latest/test_wrap_mistral_chat_stream_sync.yaml @@ -1,33 +1,141 @@ interactions: - request: - body: '{"model":"mistral-small-latest","messages":[{"content":"What is 5+5? Reply - with just the number.","role":"user"}],"max_tokens":10,"stream":true}' + body: '{"model":"zai-glm-5-3","messages":[{"content":"What is 5+5? Reply with + just the number.","role":"user"}],"max_tokens":256,"stream":true,"min_tokens":1,"repetition_penalty":1.1,"top_k":20,"logprobs":true,"top_logprobs":2}' headers: accept: - text/event-stream accept-encoding: - - gzip, deflate + - gzip, deflate, zstd connection: - keep-alive content-length: - - '144' + - '220' content-type: - application/json host: - api.mistral.ai user-agent: - - mistral-client-python/2.9.0 + - mistral-client-python/3.2.0 method: POST uri: https://api.mistral.ai/v1/chat/completions#stream response: body: - string: 'data: {"id":"b33b8bffce0c4169974d421ab833ef9c","object":"chat.completion.chunk","created":1785860147,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} + string: 'data: {"id":"a354dc28022641a3acbe3583e61f26aa","object":"chat.completion.chunk","created":1791905272,"model":"zai-glm-5-3","choices":[{"index":0,"delta":{"role":"assistant","content":""},"finish_reason":null}]} - data: {"id":"b33b8bffce0c4169974d421ab833ef9c","object":"chat.completion.chunk","created":1785860147,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"content":"1"},"finish_reason":null}],"p":"abcde"} + data: {"id":"a354dc28022641a3acbe3583e61f26aa","object":"chat.completion.chunk","created":1791905273,"model":"zai-glm-5-3","choices":[{"index":0,"delta":{"role":"assistant","index":0,"content":""},"finish_reason":null,"logprobs":null}]} - data: {"id":"b33b8bffce0c4169974d421ab833ef9c","object":"chat.completion.chunk","created":1785860147,"model":"mistral-small-latest","choices":[{"index":0,"delta":{"content":"0"},"finish_reason":"stop"}],"usage":{"prompt_tokens":28,"total_tokens":31,"completion_tokens":3,"prompt_tokens_details":{"cached_tokens":0},"service_tier":"standard"},"p":"abcdefghijklm"} + data: {"id":"a354dc28022641a3acbe3583e61f26aa","object":"chat.completion.chunk","created":1791905273,"model":"zai-glm-5-3","choices":[{"index":0,"delta":{"index":0,"content":[{"type":"thinking","thinking":[{"type":"text","text":"The"}],"closed":true}]},"finish_reason":null,"logprobs":{"content":[{"token":"The","logprob":-0.05786751210689545,"bytes":[84,104,101],"token_id":-1,"top_logprobs":[{"token":"The","logprob":-0.05786751210689545,"bytes":[84,104,101],"token_id":-1},{"token":"This","logprob":-2.9328675270080566,"bytes":[84,104,105,115],"token_id":-1}]}]}}]} + + + data: {"id":"a354dc28022641a3acbe3583e61f26aa","object":"chat.completion.chunk","created":1791905273,"model":"zai-glm-5-3","choices":[{"index":0,"delta":{"index":0,"content":[{"type":"thinking","thinking":[{"type":"text","text":" + user is asking a simple math question and wants just the number as a reply.\n\n5 + + 5 = 10\n\nThey asked me to reply with just the number, so I should give + a very concise response."}],"closed":true},{"type":"text","text":"10"}]},"finish_reason":null,"logprobs":{"content":[{"token":" + user","logprob":-0.00208850996568799,"bytes":[32,117,115,101,114],"token_id":-1,"top_logprobs":[{"token":" + user","logprob":-0.00208850996568799,"bytes":[32,117,115,101,114],"token_id":-1},{"token":" + question","logprob":-6.25208854675293,"bytes":[32,113,117,101,115,116,105,111,110],"token_id":-1}]},{"token":" + is","logprob":-0.4995725750923157,"bytes":[32,105,115],"token_id":-1,"top_logprobs":[{"token":" + is","logprob":-0.4995725750923157,"bytes":[32,105,115],"token_id":-1},{"token":" + asks","logprob":-1.249572515487671,"bytes":[32,97,115,107,115],"token_id":-1}]},{"token":" + asking","logprob":-0.00003540453326422721,"bytes":[32,97,115,107,105,110,103],"token_id":-1,"top_logprobs":[{"token":" + asking","logprob":-0.00003540453326422721,"bytes":[32,97,115,107,105,110,103],"token_id":-1},{"token":" + asked","logprob":-10.625035285949707,"bytes":[32,97,115,107,101,100],"token_id":-1}]},{"token":" + a","logprob":-0.12166183441877365,"bytes":[32,97],"token_id":-1,"top_logprobs":[{"token":" + a","logprob":-0.12166183441877365,"bytes":[32,97],"token_id":-1},{"token":" + what","logprob":-2.996661901473999,"bytes":[32,119,104,97,116],"token_id":-1}]},{"token":" + simple","logprob":-0.012226044200360775,"bytes":[32,115,105,109,112,108,101],"token_id":-1,"top_logprobs":[{"token":" + simple","logprob":-0.012226044200360775,"bytes":[32,115,105,109,112,108,101],"token_id":-1},{"token":" + very","logprob":-4.512226104736328,"bytes":[32,118,101,114,121],"token_id":-1}]},{"token":" + math","logprob":-0.8302291631698608,"bytes":[32,109,97,116,104],"token_id":-1,"top_logprobs":[{"token":" + math","logprob":-0.8302291631698608,"bytes":[32,109,97,116,104],"token_id":-1},{"token":" + arithmetic","logprob":-0.5802291631698608,"bytes":[32,97,114,105,116,104,109,101,116,105,99],"token_id":-1}]},{"token":" + question","logprob":-0.00008105902816168964,"bytes":[32,113,117,101,115,116,105,111,110],"token_id":-1,"top_logprobs":[{"token":" + question","logprob":-0.00008105902816168964,"bytes":[32,113,117,101,115,116,105,111,110],"token_id":-1},{"token":" + problem","logprob":-9.500081062316895,"bytes":[32,112,114,111,98,108,101,109],"token_id":-1}]},{"token":" + and","logprob":-2.9305837154388428,"bytes":[32,97,110,100],"token_id":-1,"top_logprobs":[{"token":" + and","logprob":-2.9305837154388428,"bytes":[32,97,110,100],"token_id":-1},{"token":":","logprob":-0.055583830922842026,"bytes":[58],"token_id":-1}]},{"token":" + wants","logprob":-1.0025783777236938,"bytes":[32,119,97,110,116,115],"token_id":-1,"top_logprobs":[{"token":" + wants","logprob":-1.0025783777236938,"bytes":[32,119,97,110,116,115],"token_id":-1},{"token":" + explicitly","logprob":-0.5025783777236938,"bytes":[32,101,120,112,108,105,99,105,116,108,121],"token_id":-1}]},{"token":" + just","logprob":-0.02939765527844429,"bytes":[32,106,117,115,116],"token_id":-1,"top_logprobs":[{"token":" + just","logprob":-0.02939765527844429,"bytes":[32,106,117,115,116],"token_id":-1},{"token":" + me","logprob":-3.65439772605896,"bytes":[32,109,101],"token_id":-1}]},{"token":" + the","logprob":-0.0004895919119007885,"bytes":[32,116,104,101],"token_id":-1,"top_logprobs":[{"token":" + the","logprob":-0.0004895919119007885,"bytes":[32,116,104,101],"token_id":-1},{"token":" + a","logprob":-7.625489711761475,"bytes":[32,97],"token_id":-1}]},{"token":" + number","logprob":-0.000013112935448589269,"bytes":[32,110,117,109,98,101,114],"token_id":-1,"top_logprobs":[{"token":" + number","logprob":-0.000013112935448589269,"bytes":[32,110,117,109,98,101,114],"token_id":-1},{"token":" + answer","logprob":-11.37501335144043,"bytes":[32,97,110,115,119,101,114],"token_id":-1}]},{"token":" + as","logprob":-0.13183175027370453,"bytes":[32,97,115],"token_id":-1,"top_logprobs":[{"token":" + as","logprob":-0.13183175027370453,"bytes":[32,97,115],"token_id":-1},{"token":".","logprob":-2.131831645965576,"bytes":[46],"token_id":-1}]},{"token":" + a","logprob":-0.35278069972991943,"bytes":[32,97],"token_id":-1,"top_logprobs":[{"token":" + a","logprob":-0.35278069972991943,"bytes":[32,97],"token_id":-1},{"token":" + the","logprob":-1.2277806997299194,"bytes":[32,116,104,101],"token_id":-1}]},{"token":" + reply","logprob":-0.06201007962226868,"bytes":[32,114,101,112,108,121],"token_id":-1,"top_logprobs":[{"token":" + reply","logprob":-0.06201007962226868,"bytes":[32,114,101,112,108,121],"token_id":-1},{"token":" + response","logprob":-2.8120100498199463,"bytes":[32,114,101,115,112,111,110,115,101],"token_id":-1}]},{"token":".\n\n","logprob":-2.127352476119995,"bytes":[46,10,10],"token_id":-1,"top_logprobs":[{"token":".\n\n","logprob":-2.127352476119995,"bytes":[46,10,10],"token_id":-1},{"token":".","logprob":-0.12735237181186676,"bytes":[46],"token_id":-1}]},{"token":"5","logprob":-0.001841517398133874,"bytes":[53],"token_id":-1,"top_logprobs":[{"token":"5","logprob":-0.001841517398133874,"bytes":[53],"token_id":-1},{"token":"10","logprob":-7.2518415451049805,"bytes":[49,48],"token_id":-1}]},{"token":" + +","logprob":-0.7576091885566711,"bytes":[32,43],"token_id":-1,"top_logprobs":[{"token":" + +","logprob":-0.7576091885566711,"bytes":[32,43],"token_id":-1},{"token":"+","logprob":-0.6326091885566711,"bytes":[43],"token_id":-1}]},{"token":" + ","logprob":0.0,"bytes":[32],"token_id":-1,"top_logprobs":[{"token":" ","logprob":0.0,"bytes":[32],"token_id":-1},{"token":" + (","logprob":-18.4375,"bytes":[32,40],"token_id":-1}]},{"token":"5","logprob":-0.000016212332411669195,"bytes":[53],"token_id":-1,"top_logprobs":[{"token":"5","logprob":-0.000016212332411669195,"bytes":[53],"token_id":-1},{"token":"4","logprob":-12.625016212463379,"bytes":[52],"token_id":-1}]},{"token":" + =","logprob":-1.5497195136049413e-6,"bytes":[32,61],"token_id":-1,"top_logprobs":[{"token":" + =","logprob":-1.5497195136049413e-6,"bytes":[32,61],"token_id":-1},{"token":" + +","logprob":-14.250001907348633,"bytes":[32,43],"token_id":-1}]},{"token":" + ","logprob":-4.410734163684538e-6,"bytes":[32],"token_id":-1,"top_logprobs":[{"token":" + ","logprob":-4.410734163684538e-6,"bytes":[32],"token_id":-1},{"token":"","logprob":-13.000004768371582,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1}]},{"token":"10","logprob":-0.000027179348762729205,"bytes":[49,48],"token_id":-1,"top_logprobs":[{"token":"10","logprob":-0.000027179348762729205,"bytes":[49,48],"token_id":-1},{"token":"102","logprob":-12.37502670288086,"bytes":[49,48,50],"token_id":-1}]},{"token":"\n\n","logprob":-0.5239465236663818,"bytes":[10,10],"token_id":-1,"top_logprobs":[{"token":"\n\n","logprob":-0.5239465236663818,"bytes":[10,10],"token_id":-1},{"token":"","logprob":-0.8989465236663818,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1}]},{"token":"They","logprob":-0.08969248086214066,"bytes":[84,104,101,121],"token_id":-1,"top_logprobs":[{"token":"They","logprob":-0.08969248086214066,"bytes":[84,104,101,121],"token_id":-1},{"token":"The","logprob":-2.7146925926208496,"bytes":[84,104,101],"token_id":-1}]},{"token":" + asked","logprob":-0.7079316973686218,"bytes":[32,97,115,107,101,100],"token_id":-1,"top_logprobs":[{"token":" + asked","logprob":-0.7079316973686218,"bytes":[32,97,115,107,101,100],"token_id":-1},{"token":" + explicitly","logprob":-0.8329316973686218,"bytes":[32,101,120,112,108,105,99,105,116,108,121],"token_id":-1}]},{"token":" + me","logprob":-0.4203052818775177,"bytes":[32,109,101],"token_id":-1,"top_logprobs":[{"token":" + me","logprob":-0.4203052818775177,"bytes":[32,109,101],"token_id":-1},{"token":" + for","logprob":-1.2953052520751953,"bytes":[32,102,111,114],"token_id":-1}]},{"token":" + to","logprob":-1.1920922133867862e-6,"bytes":[32,116,111],"token_id":-1,"top_logprobs":[{"token":" + to","logprob":-1.1920922133867862e-6,"bytes":[32,116,111],"token_id":-1},{"token":" + not","logprob":-14.250000953674316,"bytes":[32,110,111,116],"token_id":-1}]},{"token":" + reply","logprob":-0.0018458010163158178,"bytes":[32,114,101,112,108,121],"token_id":-1,"top_logprobs":[{"token":" + reply","logprob":-0.0018458010163158178,"bytes":[32,114,101,112,108,121],"token_id":-1},{"token":" + \"","logprob":-6.376845836639404,"bytes":[32,34],"token_id":-1}]},{"token":" + with","logprob":-2.50339189733495e-6,"bytes":[32,119,105,116,104],"token_id":-1,"top_logprobs":[{"token":" + with","logprob":-2.50339189733495e-6,"bytes":[32,119,105,116,104],"token_id":-1},{"token":" + \"","logprob":-12.87500286102295,"bytes":[32,34],"token_id":-1}]},{"token":" + just","logprob":-0.00009667406266089529,"bytes":[32,106,117,115,116],"token_id":-1,"top_logprobs":[{"token":" + just","logprob":-0.00009667406266089529,"bytes":[32,106,117,115,116],"token_id":-1},{"token":" + \"","logprob":-9.250096321105957,"bytes":[32,34],"token_id":-1}]},{"token":" + the","logprob":-7.152555099310121e-7,"bytes":[32,116,104,101],"token_id":-1,"top_logprobs":[{"token":" + the","logprob":-7.152555099310121e-7,"bytes":[32,116,104,101],"token_id":-1},{"token":" + a","logprob":-15.125000953674316,"bytes":[32,97],"token_id":-1}]},{"token":" + number","logprob":-8.344646857949556e-7,"bytes":[32,110,117,109,98,101,114],"token_id":-1,"top_logprobs":[{"token":" + number","logprob":-8.344646857949556e-7,"bytes":[32,110,117,109,98,101,114],"token_id":-1},{"token":"number","logprob":-15.500000953674316,"bytes":[110,117,109,98,101,114],"token_id":-1}]},{"token":",","logprob":-0.12697488069534302,"bytes":[44],"token_id":-1,"top_logprobs":[{"token":",","logprob":-0.12697488069534302,"bytes":[44],"token_id":-1},{"token":".","logprob":-2.1269748210906982,"bytes":[46],"token_id":-1}]},{"token":" + so","logprob":-0.00005006664650863968,"bytes":[32,115,111],"token_id":-1,"top_logprobs":[{"token":" + so","logprob":-0.00005006664650863968,"bytes":[32,115,111],"token_id":-1},{"token":" + and","logprob":-11.000049591064453,"bytes":[32,97,110,100],"token_id":-1}]},{"token":" + I","logprob":-0.006945987697690725,"bytes":[32,73],"token_id":-1,"top_logprobs":[{"token":" + I","logprob":-0.006945987697690725,"bytes":[32,73],"token_id":-1},{"token":" + my","logprob":-5.131946086883545,"bytes":[32,109,121],"token_id":-1}]},{"token":" + should","logprob":-0.0006308949086815119,"bytes":[32,115,104,111,117,108,100],"token_id":-1,"top_logprobs":[{"token":" + should","logprob":-0.0006308949086815119,"bytes":[32,115,104,111,117,108,100],"token_id":-1},{"token":"''ll","logprob":-7.375630855560303,"bytes":[39,108,108],"token_id":-1}]},{"token":" + give","logprob":-4.385343551635742,"bytes":[32,103,105,118,101],"token_id":-1,"top_logprobs":[{"token":" + give","logprob":-4.385343551635742,"bytes":[32,103,105,118,101],"token_id":-1},{"token":" + just","logprob":-0.13534371554851532,"bytes":[32,106,117,115,116],"token_id":-1}]},{"token":" + a","logprob":-0.480926513671875,"bytes":[32,97],"token_id":-1,"top_logprobs":[{"token":" + a","logprob":-0.480926513671875,"bytes":[32,97],"token_id":-1},{"token":" + just","logprob":-1.480926513671875,"bytes":[32,106,117,115,116],"token_id":-1}]},{"token":" + very","logprob":-0.7497419714927673,"bytes":[32,118,101,114,121],"token_id":-1,"top_logprobs":[{"token":" + very","logprob":-0.7497419714927673,"bytes":[32,118,101,114,121],"token_id":-1},{"token":" + minimal","logprob":-0.8747419714927673,"bytes":[32,109,105,110,105,109,97,108],"token_id":-1}]},{"token":" + concise","logprob":-0.6476334929466248,"bytes":[32,99,111,110,99,105,115,101],"token_id":-1,"top_logprobs":[{"token":" + concise","logprob":-0.6476334929466248,"bytes":[32,99,111,110,99,105,115,101],"token_id":-1},{"token":" + brief","logprob":-1.1476335525512695,"bytes":[32,98,114,105,101,102],"token_id":-1}]},{"token":" + response","logprob":-1.5031237602233887,"bytes":[32,114,101,115,112,111,110,115,101],"token_id":-1,"top_logprobs":[{"token":" + response","logprob":-1.5031237602233887,"bytes":[32,114,101,115,112,111,110,115,101],"token_id":-1},{"token":" + answer","logprob":-0.25312376022338867,"bytes":[32,97,110,115,119,101,114],"token_id":-1}]},{"token":".","logprob":-0.1319911926984787,"bytes":[46],"token_id":-1,"top_logprobs":[{"token":".","logprob":-0.1319911926984787,"bytes":[46],"token_id":-1},{"token":":","logprob":-2.131991147994995,"bytes":[58],"token_id":-1}]},{"token":"","logprob":-0.012163864448666573,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1,"top_logprobs":[{"token":"","logprob":-0.012163864448666573,"bytes":[60,47,116,104,105,110,107,62],"token_id":-1},{"token":" + No","logprob":-5.5121636390686035,"bytes":[32,78,111],"token_id":-1}]},{"token":"10","logprob":-0.00012194366718176752,"bytes":[49,48],"token_id":-1,"top_logprobs":[{"token":"10","logprob":-0.00012194366718176752,"bytes":[49,48],"token_id":-1},{"token":"**","logprob":-9.1251220703125,"bytes":[42,42],"token_id":-1}]},{"token":"<|user|>","logprob":-6.794906312279636e-6,"bytes":[60,124,117,115,101,114,124,62],"token_id":-1,"top_logprobs":[{"token":"<|user|>","logprob":-6.794906312279636e-6,"bytes":[60,124,117,115,101,114,124,62],"token_id":-1},{"token":"","logprob":-13.250006675720215,"bytes":[60,116,111,111,108,95,99,97,108,108,62],"token_id":-1}]}]}}]} + + + data: {"id":"a354dc28022641a3acbe3583e61f26aa","object":"chat.completion.chunk","created":1791905273,"model":"zai-glm-5-3","choices":[{"index":0,"delta":{"index":0,"content":""},"finish_reason":"stop","logprobs":null,"response_error":{"format_error":null,"content_validation":["valid_think_chunk","valid_text_chunk"],"tool_calls_validation":[],"extra_tags":[]}}],"usage":{"prompt_tokens":25,"total_tokens":72,"completion_tokens":47,"prompt_tokens_details":{"cached_tokens":0}}} data: [DONE] @@ -42,15 +150,15 @@ interactions: cf-cache-status: - DYNAMIC cf-ray: - - a25edce15c2bebb6-YYZ + - a49f5eef4f34e930-YYZ connection: - keep-alive content-type: - text/event-stream; charset=utf-8 date: - - Tue, 04 Aug 2026 16:15:48 GMT + - Tue, 13 Oct 2026 15:27:53 GMT mistral-correlation-id: - - 019fcd8f-490e-79e9-a077-425db1b649b6 + - 01a135e0-91ce-7b68-99e3-4beca3f958e2 server: - cloudflare strict-transport-security: @@ -60,15 +168,13 @@ interactions: x-content-type-options: - nosniff x-envoy-upstream-service-time: - - '467' + - '870' x-kong-proxy-latency: - - '13' + - '37' x-kong-request-id: - - 019fcd8f-490e-79e9-a077-425db1b649b6 + - 01a135e0-91ce-7b68-99e3-4beca3f958e2 x-kong-upstream-latency: - - '468' - x-max-retry-attempts-reached: - - 'false' + - '871' status: code: 200 message: OK diff --git a/py/src/braintrust/integrations/mistral/test_mistral.py b/py/src/braintrust/integrations/mistral/test_mistral.py index 5edeb352d..f078eebd5 100644 --- a/py/src/braintrust/integrations/mistral/test_mistral.py +++ b/py/src/braintrust/integrations/mistral/test_mistral.py @@ -43,6 +43,8 @@ PROJECT_NAME = "test-mistral-sdk" CHAT_MODEL = "mistral-small-latest" AGENT_MODEL = CHAT_MODEL +# This model supports logprobs and the additional sampling parameters. +LOGPROBS_MODEL = "zai-glm-5-3" EMBEDDING_MODEL = "mistral-embed" FIM_MODEL = "codestral-latest" OCR_MODEL = "mistral-ocr-latest" @@ -155,10 +157,10 @@ def mistral_client(request, monkeypatch): return wrap_mistral(_get_client()) -def _completion_params(method): +def _completion_params(method, *, stream=False): if "logprobs" not in inspect.signature(method).parameters: return {} - return { + params = { "logprobs": True, "top_logprobs": 2, "prompt_logprobs": True, @@ -167,6 +169,11 @@ def _completion_params(method): "repetition_penalty": 1.1, "top_k": 20, } + if stream: + # This model only returns prompt probabilities for non-streamed calls. + del params["prompt_logprobs"] + del params["top_prompt_logprobs"] + return params def _assert_completion_params(span, params): @@ -180,7 +187,7 @@ def _assert_completion_logprobs(span, response): assert hasattr(expected, "content") and expected.content, ( f"Expected populated {key}; re-record this cassette with Mistral 3.2.0." ) - assert span["output"][0][key] == expected.model_dump(mode="python", by_alias=True) + assert span["output"][0][key] == expected.model_dump(exclude_none=True) def _weather_tool(): @@ -206,12 +213,12 @@ def _assert_ocr_metrics_are_valid(metrics, start, end): @contextmanager -def _temporary_agent(client): +def _temporary_agent(client, *, model=AGENT_MODEL): manager = getattr(getattr(client, "beta", None), "agents", None) assert manager is not None, "Mistral beta.agents is required for agent tests" agent = manager.create( - model=AGENT_MODEL, + model=model, name=f"braintrust-test-agent-{int(time.time() * 1000)}", instructions="You are concise. Keep responses under five words.", ) @@ -230,11 +237,12 @@ def test_wrap_mistral_chat_complete_sync(memory_logger): client = wrap_mistral(_get_client()) params = _completion_params(client.chat.complete) + model = LOGPROBS_MODEL if params else CHAT_MODEL start = time.time() response = client.chat.complete( - model=CHAT_MODEL, + model=model, messages=[{"role": "user", "content": "What is 2+2? Reply with just the number."}], - max_tokens=10, + max_tokens=256 if params else 10, **params, ) end = time.time() @@ -248,7 +256,7 @@ def test_wrap_mistral_chat_complete_sync(memory_logger): assert span["context"]["span_origin"]["instrumentation"]["name"] == "mistral-auto" assert span["input"] == [{"role": "user", "content": "What is 2+2? Reply with just the number."}] assert span["metadata"]["provider"] == "mistral" - assert span["metadata"]["model"] == CHAT_MODEL + assert span["metadata"]["model"] == model _assert_completion_params(span, params) if params: _assert_completion_logprobs(span, response) @@ -310,12 +318,13 @@ def test_wrap_mistral_chat_stream_sync(memory_logger): assert not memory_logger.pop() client = wrap_mistral(_get_client()) - params = _completion_params(client.chat.stream) + params = _completion_params(client.chat.stream, stream=True) + model = LOGPROBS_MODEL if params else CHAT_MODEL start = time.time() with client.chat.stream( - model=CHAT_MODEL, + model=model, messages=[{"role": "user", "content": "What is 5+5? Reply with just the number."}], - max_tokens=10, + max_tokens=256 if params else 10, **params, ) as stream: chunks = list(stream) @@ -323,10 +332,12 @@ def test_wrap_mistral_chat_stream_sync(memory_logger): assert chunks streamed_text = "".join( - choice.delta.content or "" + content + if isinstance(content := choice.delta.content, str) + else "".join(getattr(part, "text", "") for part in content or []) for chunk in chunks for choice in (chunk.data.choices or []) - if getattr(choice, "delta", None) is not None and isinstance(choice.delta.content, str) + if getattr(choice, "delta", None) is not None ) assert "10" in streamed_text @@ -334,20 +345,19 @@ def test_wrap_mistral_chat_stream_sync(memory_logger): assert len(spans) == 1 span = spans[0] assert span["metadata"]["provider"] == "mistral" - assert span["metadata"]["model"] == CHAT_MODEL + assert span["metadata"]["model"] == model assert span["metadata"]["stream"] == True _assert_completion_params(span, params) if params: - for key in ("logprobs", "prompt_logprobs"): - expected_content = [ - token.model_dump(mode="python", by_alias=True) - for chunk in chunks - for choice in chunk.data.choices - if hasattr(logprobs := getattr(choice, key), "content") - for token in logprobs.content - ] - assert expected_content, f"Expected populated {key}; re-record this cassette with Mistral 3.2.0." - assert span["output"][0][key]["content"] == expected_content + expected_content = [ + token.model_dump(mode="python", by_alias=True) + for chunk in chunks + for choice in chunk.data.choices + if hasattr(logprobs := choice.logprobs, "content") + for token in logprobs.content + ] + assert expected_content, "Expected populated logprobs; re-record this cassette with Mistral 3.2.0." + assert span["output"][0]["logprobs"]["content"] == expected_content assert span["metrics"]["time_to_first_token"] >= 0 assert "10" in str(span["output"]) assert_metrics_are_valid(span["metrics"], start, end) @@ -577,12 +587,12 @@ def test_wrap_mistral_agents_complete_sync(memory_logger): client = wrap_mistral(_get_client()) params = _completion_params(client.agents.complete) - with _temporary_agent(client) as agent_id: + with _temporary_agent(client, model=LOGPROBS_MODEL if params else AGENT_MODEL) as agent_id: start = time.time() response = client.agents.complete( agent_id=agent_id, messages=[{"role": "user", "content": "What is 8+1? Reply with just the number."}], - max_tokens=10, + max_tokens=256 if params else 10, **params, ) end = time.time()