feat(op): Add gen_ai.evaluate op

javascript/sentry-conventions/src/attributes.ts

@@ -7744,7 +7744,7 @@ export type GEN_AI_FUNCTION_ID_TYPE = string;
 // Path: model/attributes/gen_ai/gen_ai__input__messages.json
 
 /**
- * The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. `gen_ai.input.messages`
+ * The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them. `gen_ai.input.messages`
  *
  * Attribute Value Type: `string` {@link GEN_AI_INPUT_MESSAGES_TYPE}
  *
@@ -7809,7 +7809,7 @@ export type GEN_AI_OPERATION_TYPE_TYPE = string;
 // Path: model/attributes/gen_ai/gen_ai__output__messages.json
 
 /**
- * The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. `gen_ai.output.messages`
+ * The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them. `gen_ai.output.messages`
  *
  * Attribute Value Type: `string` {@link GEN_AI_OUTPUT_MESSAGES_TYPE}
  *
@@ -26582,7 +26582,7 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
   },
   'gen_ai.input.messages': {
     brief:
-      'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.',
+      'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.',
     type: 'string',
     keys: [
       'gen_ai.input.messages',
@@ -26601,6 +26601,7 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
       '[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]',
     aliases: ['ai.texts', 'ai.prompt.messages', 'gen_ai.prompt', 'ai.prompt'],
     changelog: [
+      { version: 'next', prs: [650], description: 'Describe the evaluation message shape for gen_ai.evaluate' },
       { version: '0.21.0', prs: [583], description: 'Added ai.prompt as an alias' },
       { version: '0.5.0', prs: [264] },
       { version: '0.4.0', prs: [221] },
@@ -26640,7 +26641,7 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
   },
   'gen_ai.output.messages': {
     brief:
-      "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.",
+      'The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.',
     type: 'string',
     keys: ['gen_ai.output.messages', 'ai.response.text', 'ai.response.toolCalls', 'ai.responses', 'ai.tool_calls'],
     applyScrubbing: {
@@ -26651,7 +26652,10 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
     example:
       '[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]',
     aliases: ['ai.response.toolCalls', 'ai.response.text'],
-    changelog: [{ version: '0.4.0', prs: [221] }],
+    changelog: [
+      { version: 'next', prs: [650], description: 'Describe the evaluation message shape for gen_ai.evaluate' },
+      { version: '0.4.0', prs: [221] },
+    ],
   },
   'gen_ai.pipeline.name': {
     brief: 'Name of the AI pipeline or chain being executed.',

javascript/sentry-conventions/src/op.ts

@@ -326,6 +326,11 @@ export const GEN_AI_GENERATE_CONTENT = 'gen_ai.generate_content';
  */
 export const GEN_AI_RERANK = 'gen_ai.rerank';
 
+/**
+ * Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model
+ */
+export const GEN_AI_EVALUATE = 'gen_ai.evaluate';
+
 /**
  * Creation of an AI agent that can later be invoked to perform a task
  */

javascript/sentry-conventions/src/search.ts

@@ -8084,7 +8084,7 @@ export const ATTRIBUTE_SEARCH_METADATA: Record<string, AttributeSearchMetadata>
     canonicalName: 'gen_ai.input.messages',
     type: 'string',
     brief:
-      'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.',
+      'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.',
     deprecationChain: [
       'gen_ai.input.messages',
       'ai.input_messages',
@@ -8112,7 +8112,7 @@ export const ATTRIBUTE_SEARCH_METADATA: Record<string, AttributeSearchMetadata>
     canonicalName: 'gen_ai.output.messages',
     type: 'string',
     brief:
-      "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.",
+      'The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.',
     deprecationChain: [
       'gen_ai.output.messages',
       'ai.response.text',

model/attributes/gen_ai/gen_ai__input__messages.json

@@ -1,6 +1,6 @@
 {
   "key": "gen_ai.input.messages",
-  "brief": "The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `\"user\"`, `\"assistant\"`, `\"tool\"`, or `\"system\"`. For messages of the role `\"tool\"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: \"text\", text:\"...\"}`.",
+  "brief": "The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `\"user\"`, `\"assistant\"`, `\"tool\"`, or `\"system\"`. For messages of the role `\"tool\"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: \"text\", text:\"...\"}`. For `gen_ai.evaluate` operations, the array holds one object `{type: \"evaluation\", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.",
   "type": "string",
   "apply_scrubbing": {
     "key": "manual"
@@ -10,6 +10,11 @@
   "alias": ["ai.texts", "ai.prompt.messages", "gen_ai.prompt", "ai.prompt"],
   "visibility": "public",
   "changelog": [
+    {
+      "version": "next",
+      "prs": [650],
+      "description": "Describe the evaluation message shape for gen_ai.evaluate"
+    },
     {
       "version": "0.21.0",
       "prs": [583],

model/attributes/gen_ai/gen_ai__output__messages.json

@@ -1,6 +1,6 @@
 {
   "key": "gen_ai.output.messages",
-  "brief": "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.",
+  "brief": "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: \"evaluation\", answers: {...}}` with the answers keyed by question name, as the provider returned them.",
   "type": "string",
   "apply_scrubbing": {
     "key": "manual"
@@ -10,6 +10,11 @@
   "alias": ["ai.response.toolCalls", "ai.response.text"],
   "visibility": "public",
   "changelog": [
+    {
+      "version": "next",
+      "prs": [650],
+      "description": "Describe the evaluation message shape for gen_ai.evaluate"
+    },
     {
       "version": "0.4.0",
       "prs": [221]

model/description/gen_ai.json

@@ -24,6 +24,7 @@
         "gen_ai",
         "gen_ai.chat",
         "gen_ai.embeddings",
+        "gen_ai.evaluate",
         "gen_ai.generate_content",
         "gen_ai.rerank",
         "gen_ai.responses",
@@ -36,6 +37,7 @@
         "text_completion gpt-3.5-turbo-instruct",
         "generate_content gemini-2.0-flash-001",
         "embeddings text-embedding-3-small",
+        "evaluate jev-latest",
         "chat"
       ]
     },

model/name/gen_ai.json

@@ -31,6 +31,7 @@
         "gen_ai",
         "gen_ai.chat",
         "gen_ai.embeddings",
+        "gen_ai.evaluate",
         "gen_ai.execute_tool",
         "gen_ai.generate_content",
         "gen_ai.rerank",
@@ -48,6 +49,7 @@
         "embeddings huggingface",
         "chat claude-opus-4",
         "execute_tool get_weather",
+        "evaluate jev-latest",
         "chat"
       ]
     }

model/op/gen_ai.json

@@ -30,6 +30,10 @@
       "name": "gen_ai.rerank",
       "description": "Reranking of documents or results by a generative AI model"
     },
+    {
+      "name": "gen_ai.evaluate",
+      "description": "Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model"
+    },
     {
       "name": "gen_ai.create_agent",
       "description": "Creation of an AI agent that can later be invoked to perform a task"

python/src/sentry_conventions/attributes.py

@@ -4776,7 +4776,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
 
     # Path: model/attributes/gen_ai/gen_ai__input__messages.json
     GEN_AI_INPUT_MESSAGES: Literal["gen_ai.input.messages"] = "gen_ai.input.messages"
-    """The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.
+    """The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.
 
     Type: str
     Apply Scrubbing: manual
@@ -4810,7 +4810,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
 
     # Path: model/attributes/gen_ai/gen_ai__output__messages.json
     GEN_AI_OUTPUT_MESSAGES: Literal["gen_ai.output.messages"] = "gen_ai.output.messages"
-    """The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.
+    """The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.
 
     Type: str
     Apply Scrubbing: manual
@@ -17783,7 +17783,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         ],
     ),
     "gen_ai.input.messages": AttributeMetadata(
-        brief='The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.',
+        brief='The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.',
         type=AttributeType.STRING,
         keys=(
             "gen_ai.input.messages",
@@ -17799,6 +17799,11 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         example='[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]',
         aliases=["ai.texts", "ai.prompt.messages", "gen_ai.prompt", "ai.prompt"],
         changelog=[
+            ChangelogEntry(
+                version="next",
+                prs=[650],
+                description="Describe the evaluation message shape for gen_ai.evaluate",
+            ),
             ChangelogEntry(
                 version="0.21.0", prs=[583], description="Added ai.prompt as an alias"
             ),
@@ -17833,7 +17838,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         ],
     ),
     "gen_ai.output.messages": AttributeMetadata(
-        brief="The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.",
+        brief='The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.',
         type=AttributeType.STRING,
         keys=(
             "gen_ai.output.messages",
@@ -17848,6 +17853,11 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         example='[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]',
         aliases=["ai.response.toolCalls", "ai.response.text"],
         changelog=[
+            ChangelogEntry(
+                version="next",
+                prs=[650],
+                description="Describe the evaluation message shape for gen_ai.evaluate",
+            ),
             ChangelogEntry(version="0.4.0", prs=[221]),
         ],
     ),

rust/src/op.rs

@@ -215,6 +215,9 @@ pub const GEN_AI_GENERATE_CONTENT: &str = "gen_ai.generate_content";
 /// Reranking of documents or results by a generative AI model
 pub const GEN_AI_RERANK: &str = "gen_ai.rerank";
 
+/// Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model
+pub const GEN_AI_EVALUATE: &str = "gen_ai.evaluate";
+
 /// Creation of an AI agent that can later be invoked to perform a task
 pub const GEN_AI_CREATE_AGENT: &str = "gen_ai.create_agent";