Test major merge

.github/workflows/test-integrations-ai.yml

@@ -69,6 +69,10 @@ jobs:
         run: |
           set -x # print commands that are executed
           ./scripts/runtox.sh "py${{ matrix.python-version }}-openai-notiktoken"
+      - name: Test typesafe
+        run: |
+          set -x # print commands that are executed
+          ./scripts/runtox.sh "py${{ matrix.python-version }}-typesafe"
       - name: Upload coverage data
         if: ${{ !cancelled() }}
         continue-on-error: true

pyproject.toml

@@ -81,6 +81,7 @@ typing = [
     "aiohttp>=3.14.3",
     "openai-agents>=0.22.0",
     "mistralai>=2.10.1",
+    "typesafe-sdk>=0.7.2",
 ]
 test = [
     "dataclasses ; python_full_version < '3.7'",

scripts/populate_tox/config.py

@@ -533,6 +533,13 @@
         "package": "typer",
         "num_versions": 2,
     },
+    "typesafe": {
+        "package": "typesafe-sdk",
+        "integration_name": "typesafe",
+        "deps": {
+            "*": ["pytest-asyncio"],
+        },
+    },
     "unleash": {
         "package": "UnleashClient",
         "num_versions": 2,

scripts/split_tox_gh_actions/split_tox_gh_actions.py

@@ -95,6 +95,7 @@
         "mistral",
         "openai-base",
         "openai-notiktoken",
+        "typesafe",
     ],
     "Cloud": [
         "aws_lambda",

sentry_sdk/consts.py

@@ -1187,6 +1187,7 @@ class OP:
     GEN_AI_CHAT = "gen_ai.chat"
     GEN_AI_CREATE_AGENT = "gen_ai.create_agent"
     GEN_AI_EMBEDDINGS = "gen_ai.embeddings"
+    GEN_AI_EVALUATE = "gen_ai.evaluate"
     GEN_AI_EXECUTE_TOOL = "gen_ai.execute_tool"
     GEN_AI_TEXT_COMPLETION = "gen_ai.text_completion"
     GEN_AI_HANDOFF = "gen_ai.handoff"

sentry_sdk/integrations/__init__.py

@@ -177,6 +177,7 @@ def iter_default_integrations(
     "tornado": (6, 0),
     "trytond_wsgi": (5, 4),
     "typer": (0, 15),
+    "typesafe": (0, 7),
     "unleash": (6, 0, 1),
 }
 

sentry_sdk/integrations/typesafe.py

@@ -0,0 +1,262 @@
+import json
+from collections.abc import Mapping
+from functools import wraps
+from typing import TYPE_CHECKING, cast
+
+import sentry_sdk
+from sentry_sdk.consts import OP, SPANDATA
+from sentry_sdk.integrations import DidNotEnable, Integration
+
+if TYPE_CHECKING:
+    from typing import (
+        Any,
+        Callable,
+        Literal,
+        Mapping,
+        NotRequired,
+        Sequence,
+        TypedDict,
+        Union,
+    )
+
+    from typesafe_sdk import JSONContent
+    from typesafe_sdk._core.question_types import Questions
+
+    class NoulModel(TypedDict):
+        type: Literal["noul"]
+        instructions: NotRequired[JSONContent | None]
+
+    class ChoiceModel(TypedDict):
+        type: Literal["choice"]
+        criteria: NotRequired[Mapping[str, JSONContent | None]]
+        instructions: NotRequired[JSONContent | None]
+
+    class ScoreModel(TypedDict):
+        type: Literal["score"]
+        criteria: NotRequired[Sequence[JSONContent]]
+        instructions: NotRequired[JSONContent | None]
+
+    class InputMessageModel(TypedDict):
+        type: Literal["evaluation"]
+        state: NotRequired[JSONContent]
+        questions: NotRequired[dict[str, Union[NoulModel, ChoiceModel, ScoreModel]]]
+
+
+try:
+    from typesafe_sdk import Choice, Noul, Score
+    from typesafe_sdk._core.client.aio.client import AsyncTypeSafeClient
+    from typesafe_sdk._core.client.sync.client import TypeSafeClient
+    from typesafe_sdk._core.response_types import SystemOneResponse
+except ImportError:
+    raise DidNotEnable("typesafe-sdk not installed")
+
+
+class TypeSafeIntegration(Integration):
+    identifier = "typesafe"
+    origin = f"auto.ai.{identifier}"
+
+    @staticmethod
+    def setup_once() -> None:
+        TypeSafeClient.system_one = _wrap_system_one(TypeSafeClient.system_one)  # type: ignore[method-assign]
+        AsyncTypeSafeClient.system_one = _wrap_system_one_async(  # type: ignore[method-assign]
+            AsyncTypeSafeClient.system_one
+        )
+
+
+def _transform_questions(
+    questions: "Questions",
+) -> "dict[str, Union[NoulModel, ChoiceModel, ScoreModel]]":
+    transformed_questions: "dict[str, Union[NoulModel, ChoiceModel, ScoreModel]]" = {}
+    for name, question in questions.items():
+        if isinstance(question, Noul):
+            noul: "NoulModel" = {
+                "type": "noul",
+            }
+
+            if question.instructions is not None:
+                noul["instructions"] = question.instructions
+
+            transformed_questions[name] = noul
+            continue
+
+        if isinstance(question, Choice):
+            choice: "ChoiceModel" = {
+                "type": "choice",
+                "criteria": question.criteria,
+            }
+
+            if question.instructions is not None:
+                choice["instructions"] = question.instructions
+
+            transformed_questions[name] = choice
+            continue
+
+        if isinstance(question, Score):
+            score: "ScoreModel" = {
+                "type": "score",
+                "criteria": question.criteria,
+            }
+
+            if question.instructions is not None:
+                score["instructions"] = question.instructions
+
+            transformed_questions[name] = score
+            continue
+
+        if not isinstance(question, dict):
+            continue
+
+        question_type = question.get("type")
+        if question_type == "noul":
+            noul = {
+                "type": question_type,
+            }
+            if "instructions" in question:
+                noul["instructions"] = question["instructions"]
+
+            transformed_questions[name] = noul
+            continue
+
+        if question_type == "choice":
+            choice = {
+                "type": question_type,
+            }
+            if "criteria" in question:
+                choice["criteria"] = cast(
+                    "Mapping[str, JSONContent | None]", question["criteria"]
+                )
+            if "instructions" in question:
+                choice["instructions"] = question["instructions"]
+
+            transformed_questions[name] = choice
+            continue
+
+        if question_type == "score":
+            score = {
+                "type": question_type,
+            }
+            if "criteria" in question:
+                score["criteria"] = cast("Sequence[JSONContent]", question["criteria"])
+            if "instructions" in question:
+                score["instructions"] = question["instructions"]
+
+            transformed_questions[name] = score
+            continue
+
+    return transformed_questions
+
+
+def _wrap_system_one(f: "Callable[..., Any]") -> "Callable[..., Any]":
+    @wraps(f)
+    def wrap_system_one(self: "TypeSafeClient", *args: "Any", **kwargs: "Any") -> "Any":
+        client = sentry_sdk.get_client()
+        integration = client.get_integration(TypeSafeIntegration)
+        if integration is None:
+            return f(self, *args, **kwargs)
+
+        model = kwargs.get("model")
+        if (
+            model is None
+            and hasattr(self, "_config")
+            and hasattr(self._config, "default_model")
+        ):
+            model = self._config.default_model
+
+        with sentry_sdk.start_span(
+            name=f"evaluate {model}".strip(),
+            attributes={
+                "sentry.op": OP.GEN_AI_EVALUATE,
+                "sentry.origin": TypeSafeIntegration.origin,
+                SPANDATA.GEN_AI_PROVIDER_NAME: "typesafe",
+                SPANDATA.GEN_AI_OPERATION_NAME: "evaluate",
+            },
+        ) as span:
+            if model is not None:
+                span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model)
+
+            if client.options["data_collection"]["gen_ai"]["inputs"]:
+                input_message: "InputMessageModel" = {
+                    "type": "evaluation",
+                }
+
+                state = args[0] if len(args) > 0 else kwargs.get("state")
+                if state is not None:
+                    input_message["state"] = state
+
+                questions = args[1] if len(args) > 1 else kwargs.get("questions")
+                if isinstance(questions, Mapping):
+                    input_message["questions"] = _transform_questions(questions)
+
+                span.set_attribute(
+                    SPANDATA.GEN_AI_INPUT_MESSAGES, json.dumps([input_message])
+                )
+
+            response = f(self, *args, **kwargs)
+
+            if not isinstance(response, SystemOneResponse):
+                return response
+
+            span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, response.model)
+
+            return response
+
+    return wrap_system_one
+
+
+def _wrap_system_one_async(f: "Callable[..., Any]") -> "Callable[..., Any]":
+    @wraps(f)
+    async def wrap_system_one_async(
+        self: "AsyncTypeSafeClient", *args: "Any", **kwargs: "Any"
+    ) -> "Any":
+        client = sentry_sdk.get_client()
+        integration = client.get_integration(TypeSafeIntegration)
+        if integration is None:
+            return await f(self, *args, **kwargs)
+
+        model = kwargs.get("model")
+        if (
+            model is None
+            and hasattr(self, "_config")
+            and hasattr(self._config, "default_model")
+        ):
+            model = self._config.default_model
+
+        with sentry_sdk.start_span(
+            name=f"evaluate {model}".strip(),
+            attributes={
+                "sentry.op": OP.GEN_AI_EVALUATE,
+                "sentry.origin": TypeSafeIntegration.origin,
+                SPANDATA.GEN_AI_PROVIDER_NAME: "typesafe",
+                SPANDATA.GEN_AI_OPERATION_NAME: "evaluate",
+            },
+        ) as span:
+            if model is not None:
+                span.set_attribute(SPANDATA.GEN_AI_REQUEST_MODEL, model)
+
+            if client.options["data_collection"]["gen_ai"]["inputs"]:
+                input_message: "InputMessageModel" = {
+                    "type": "evaluation",
+                }
+
+                state = args[0] if len(args) > 0 else kwargs.get("state")
+                if state is not None:
+                    input_message["state"] = state
+
+                questions = args[1] if len(args) > 1 else kwargs.get("questions")
+                if isinstance(questions, Mapping):
+                    input_message["questions"] = _transform_questions(questions)
+
+                span.set_attribute(
+                    SPANDATA.GEN_AI_INPUT_MESSAGES, json.dumps([input_message])
+                )
+
+            response = await f(self, *args, **kwargs)
+
+            if not isinstance(response, SystemOneResponse):
+                return response
+
+            span.set_attribute(SPANDATA.GEN_AI_RESPONSE_MODEL, response.model)
+
+            return response
+
+    return wrap_system_one_async

sentry_sdk/tracing_utils.py

@@ -1217,13 +1217,10 @@ def _make_sampling_decision(
             sample_rate = client.options["traces_sampler"](sampling_context)
         except Exception:
             logger.warning(
-                "[Tracing] traces_sampler raised; falling back to parent sample rate or traces_sample_rate",
+                "[Tracing] traces_sampler raised; unsampling trace",
                 exc_info=True,
             )
-            if propagation_context.parent_sampled is not None:
-                sample_rate = propagation_context.parent_sampled
-            else:
-                sample_rate = client.options["traces_sample_rate"]
+            return False, None, None, "callback_error"
     else:
         if propagation_context.parent_sampled is not None:
             sample_rate = propagation_context.parent_sampled

tests/integrations/flask/test_flask.py

@@ -355,7 +355,7 @@ def index():
         except ZeroDivisionError:
             pass
 
-    sentry_sdk.get_client().flush()
+    sentry_sdk.flush()
 
     (first_event, error_event, session) = envelopes
     first_event = first_event.get_event()