From a0fd65f68db81fffe5bcfc43bae1285213b1386a Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 15:00:37 +0200 Subject: [PATCH 1/2] feat(op): Add gen_ai.evaluate op Co-Authored-By: Claude Opus 5.5 --- javascript/sentry-conventions/src/attributes.ts | 8 ++++---- javascript/sentry-conventions/src/op.ts | 5 +++++ javascript/sentry-conventions/src/search.ts | 4 ++-- model/attributes/gen_ai/gen_ai__input__messages.json | 2 +- model/attributes/gen_ai/gen_ai__output__messages.json | 2 +- model/description/gen_ai.json | 2 ++ model/name/gen_ai.json | 2 ++ model/op/gen_ai.json | 4 ++++ python/src/sentry_conventions/attributes.py | 8 ++++---- rust/src/op.rs | 3 +++ 10 files changed, 28 insertions(+), 12 deletions(-) diff --git a/javascript/sentry-conventions/src/attributes.ts b/javascript/sentry-conventions/src/attributes.ts index 85eda6231..d37b330cb 100644 --- a/javascript/sentry-conventions/src/attributes.ts +++ b/javascript/sentry-conventions/src/attributes.ts @@ -7744,7 +7744,7 @@ export type GEN_AI_FUNCTION_ID_TYPE = string; // Path: model/attributes/gen_ai/gen_ai__input__messages.json /** - * The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. `gen_ai.input.messages` + * The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them. `gen_ai.input.messages` * * Attribute Value Type: `string` {@link GEN_AI_INPUT_MESSAGES_TYPE} * @@ -7809,7 +7809,7 @@ export type GEN_AI_OPERATION_TYPE_TYPE = string; // Path: model/attributes/gen_ai/gen_ai__output__messages.json /** - * The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. `gen_ai.output.messages` + * The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them. `gen_ai.output.messages` * * Attribute Value Type: `string` {@link GEN_AI_OUTPUT_MESSAGES_TYPE} * @@ -26582,7 +26582,7 @@ export const ATTRIBUTE_METADATA: Record = { }, 'gen_ai.input.messages': { brief: - 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.', + 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.', type: 'string', keys: [ 'gen_ai.input.messages', @@ -26640,7 +26640,7 @@ export const ATTRIBUTE_METADATA: Record = { }, 'gen_ai.output.messages': { brief: - "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + 'The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.', type: 'string', keys: ['gen_ai.output.messages', 'ai.response.text', 'ai.response.toolCalls', 'ai.responses', 'ai.tool_calls'], applyScrubbing: { diff --git a/javascript/sentry-conventions/src/op.ts b/javascript/sentry-conventions/src/op.ts index 92d3daee7..2e85fe8bf 100644 --- a/javascript/sentry-conventions/src/op.ts +++ b/javascript/sentry-conventions/src/op.ts @@ -326,6 +326,11 @@ export const GEN_AI_GENERATE_CONTENT = 'gen_ai.generate_content'; */ export const GEN_AI_RERANK = 'gen_ai.rerank'; +/** + * Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model + */ +export const GEN_AI_EVALUATE = 'gen_ai.evaluate'; + /** * Creation of an AI agent that can later be invoked to perform a task */ diff --git a/javascript/sentry-conventions/src/search.ts b/javascript/sentry-conventions/src/search.ts index 0c772b8f4..32b009961 100644 --- a/javascript/sentry-conventions/src/search.ts +++ b/javascript/sentry-conventions/src/search.ts @@ -8084,7 +8084,7 @@ export const ATTRIBUTE_SEARCH_METADATA: Record canonicalName: 'gen_ai.input.messages', type: 'string', brief: - 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.', + 'The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.', deprecationChain: [ 'gen_ai.input.messages', 'ai.input_messages', @@ -8112,7 +8112,7 @@ export const ATTRIBUTE_SEARCH_METADATA: Record canonicalName: 'gen_ai.output.messages', type: 'string', brief: - "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + 'The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.', deprecationChain: [ 'gen_ai.output.messages', 'ai.response.text', diff --git a/model/attributes/gen_ai/gen_ai__input__messages.json b/model/attributes/gen_ai/gen_ai__input__messages.json index 0e5215641..f1552c5d6 100644 --- a/model/attributes/gen_ai/gen_ai__input__messages.json +++ b/model/attributes/gen_ai/gen_ai__input__messages.json @@ -1,6 +1,6 @@ { "key": "gen_ai.input.messages", - "brief": "The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `\"user\"`, `\"assistant\"`, `\"tool\"`, or `\"system\"`. For messages of the role `\"tool\"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: \"text\", text:\"...\"}`.", + "brief": "The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `\"user\"`, `\"assistant\"`, `\"tool\"`, or `\"system\"`. For messages of the role `\"tool\"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: \"text\", text:\"...\"}`. For `gen_ai.evaluate` operations, the array holds one object `{type: \"evaluation\", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.", "type": "string", "apply_scrubbing": { "key": "manual" diff --git a/model/attributes/gen_ai/gen_ai__output__messages.json b/model/attributes/gen_ai/gen_ai__output__messages.json index 5b80a9245..6131232ca 100644 --- a/model/attributes/gen_ai/gen_ai__output__messages.json +++ b/model/attributes/gen_ai/gen_ai__output__messages.json @@ -1,6 +1,6 @@ { "key": "gen_ai.output.messages", - "brief": "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + "brief": "The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: \"evaluation\", answers: {...}}` with the answers keyed by question name, as the provider returned them.", "type": "string", "apply_scrubbing": { "key": "manual" diff --git a/model/description/gen_ai.json b/model/description/gen_ai.json index 21e0da778..1ee5d21a1 100644 --- a/model/description/gen_ai.json +++ b/model/description/gen_ai.json @@ -24,6 +24,7 @@ "gen_ai", "gen_ai.chat", "gen_ai.embeddings", + "gen_ai.evaluate", "gen_ai.generate_content", "gen_ai.rerank", "gen_ai.responses", @@ -36,6 +37,7 @@ "text_completion gpt-3.5-turbo-instruct", "generate_content gemini-2.0-flash-001", "embeddings text-embedding-3-small", + "evaluate jev-latest", "chat" ] }, diff --git a/model/name/gen_ai.json b/model/name/gen_ai.json index e46c807a2..f84fab001 100644 --- a/model/name/gen_ai.json +++ b/model/name/gen_ai.json @@ -31,6 +31,7 @@ "gen_ai", "gen_ai.chat", "gen_ai.embeddings", + "gen_ai.evaluate", "gen_ai.execute_tool", "gen_ai.generate_content", "gen_ai.rerank", @@ -48,6 +49,7 @@ "embeddings huggingface", "chat claude-opus-4", "execute_tool get_weather", + "evaluate jev-latest", "chat" ] } diff --git a/model/op/gen_ai.json b/model/op/gen_ai.json index e2d5bce37..c82fe5763 100644 --- a/model/op/gen_ai.json +++ b/model/op/gen_ai.json @@ -30,6 +30,10 @@ "name": "gen_ai.rerank", "description": "Reranking of documents or results by a generative AI model" }, + { + "name": "gen_ai.evaluate", + "description": "Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model" + }, { "name": "gen_ai.create_agent", "description": "Creation of an AI agent that can later be invoked to perform a task" diff --git a/python/src/sentry_conventions/attributes.py b/python/src/sentry_conventions/attributes.py index fd9e06c0e..7213683e1 100644 --- a/python/src/sentry_conventions/attributes.py +++ b/python/src/sentry_conventions/attributes.py @@ -4776,7 +4776,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): # Path: model/attributes/gen_ai/gen_ai__input__messages.json GEN_AI_INPUT_MESSAGES: Literal["gen_ai.input.messages"] = "gen_ai.input.messages" - """The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. + """The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them. Type: str Apply Scrubbing: manual @@ -4810,7 +4810,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): # Path: model/attributes/gen_ai/gen_ai__output__messages.json GEN_AI_OUTPUT_MESSAGES: Literal["gen_ai.output.messages"] = "gen_ai.output.messages" - """The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. + """The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them. Type: str Apply Scrubbing: manual @@ -17783,7 +17783,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): ], ), "gen_ai.input.messages": AttributeMetadata( - brief='The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`.', + brief='The messages passed to the model. It has to be a stringified version of an array of objects. The `role` attribute of each object must be `"user"`, `"assistant"`, `"tool"`, or `"system"`. For messages of the role `"tool"`, the `content` can be a string or an arbitrary object with information about the tool call. For other messages the `content` can be either a string or a list of objects in the format `{type: "text", text:"..."}`. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", state: ..., questions: {...}}` with the evaluated state and the questions keyed by name, as the caller passed them.', type=AttributeType.STRING, keys=( "gen_ai.input.messages", @@ -17833,7 +17833,7 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): ], ), "gen_ai.output.messages": AttributeMetadata( - brief="The model's response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls.", + brief='The model\'s response messages. It has to be a stringified version of an array of message objects, which can include text responses and tool calls. For `gen_ai.evaluate` operations, the array holds one object `{type: "evaluation", answers: {...}}` with the answers keyed by question name, as the provider returned them.', type=AttributeType.STRING, keys=( "gen_ai.output.messages", diff --git a/rust/src/op.rs b/rust/src/op.rs index 8974b6af4..a4299e7bb 100644 --- a/rust/src/op.rs +++ b/rust/src/op.rs @@ -215,6 +215,9 @@ pub const GEN_AI_GENERATE_CONTENT: &str = "gen_ai.generate_content"; /// Reranking of documents or results by a generative AI model pub const GEN_AI_RERANK: &str = "gen_ai.rerank"; +/// Evaluation of typed questions (e.g., boolean, choice, or score) about a shared state by a generative AI model +pub const GEN_AI_EVALUATE: &str = "gen_ai.evaluate"; + /// Creation of an AI agent that can later be invoked to perform a task pub const GEN_AI_CREATE_AGENT: &str = "gen_ai.create_agent"; From 7f464e749a80a1145066082e8847bc4e4563a1aa Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 15:01:04 +0200 Subject: [PATCH 2/2] Add changelog entries for evaluation message shape --- javascript/sentry-conventions/src/attributes.ts | 6 +++++- model/attributes/gen_ai/gen_ai__input__messages.json | 5 +++++ model/attributes/gen_ai/gen_ai__output__messages.json | 5 +++++ python/src/sentry_conventions/attributes.py | 10 ++++++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/javascript/sentry-conventions/src/attributes.ts b/javascript/sentry-conventions/src/attributes.ts index d37b330cb..4d85bd6ba 100644 --- a/javascript/sentry-conventions/src/attributes.ts +++ b/javascript/sentry-conventions/src/attributes.ts @@ -26601,6 +26601,7 @@ export const ATTRIBUTE_METADATA: Record = { '[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]', aliases: ['ai.texts', 'ai.prompt.messages', 'gen_ai.prompt', 'ai.prompt'], changelog: [ + { version: 'next', prs: [650], description: 'Describe the evaluation message shape for gen_ai.evaluate' }, { version: '0.21.0', prs: [583], description: 'Added ai.prompt as an alias' }, { version: '0.5.0', prs: [264] }, { version: '0.4.0', prs: [221] }, @@ -26651,7 +26652,10 @@ export const ATTRIBUTE_METADATA: Record = { example: '[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]', aliases: ['ai.response.toolCalls', 'ai.response.text'], - changelog: [{ version: '0.4.0', prs: [221] }], + changelog: [ + { version: 'next', prs: [650], description: 'Describe the evaluation message shape for gen_ai.evaluate' }, + { version: '0.4.0', prs: [221] }, + ], }, 'gen_ai.pipeline.name': { brief: 'Name of the AI pipeline or chain being executed.', diff --git a/model/attributes/gen_ai/gen_ai__input__messages.json b/model/attributes/gen_ai/gen_ai__input__messages.json index f1552c5d6..00b3c4656 100644 --- a/model/attributes/gen_ai/gen_ai__input__messages.json +++ b/model/attributes/gen_ai/gen_ai__input__messages.json @@ -10,6 +10,11 @@ "alias": ["ai.texts", "ai.prompt.messages", "gen_ai.prompt", "ai.prompt"], "visibility": "public", "changelog": [ + { + "version": "next", + "prs": [650], + "description": "Describe the evaluation message shape for gen_ai.evaluate" + }, { "version": "0.21.0", "prs": [583], diff --git a/model/attributes/gen_ai/gen_ai__output__messages.json b/model/attributes/gen_ai/gen_ai__output__messages.json index 6131232ca..754068b10 100644 --- a/model/attributes/gen_ai/gen_ai__output__messages.json +++ b/model/attributes/gen_ai/gen_ai__output__messages.json @@ -10,6 +10,11 @@ "alias": ["ai.response.toolCalls", "ai.response.text"], "visibility": "public", "changelog": [ + { + "version": "next", + "prs": [650], + "description": "Describe the evaluation message shape for gen_ai.evaluate" + }, { "version": "0.4.0", "prs": [221] diff --git a/python/src/sentry_conventions/attributes.py b/python/src/sentry_conventions/attributes.py index 7213683e1..334472fc9 100644 --- a/python/src/sentry_conventions/attributes.py +++ b/python/src/sentry_conventions/attributes.py @@ -17799,6 +17799,11 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): example='[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]', aliases=["ai.texts", "ai.prompt.messages", "gen_ai.prompt", "ai.prompt"], changelog=[ + ChangelogEntry( + version="next", + prs=[650], + description="Describe the evaluation message shape for gen_ai.evaluate", + ), ChangelogEntry( version="0.21.0", prs=[583], description="Added ai.prompt as an alias" ), @@ -17848,6 +17853,11 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta): example='[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]', aliases=["ai.response.toolCalls", "ai.response.text"], changelog=[ + ChangelogEntry( + version="next", + prs=[650], + description="Describe the evaluation message shape for gen_ai.evaluate", + ), ChangelogEntry(version="0.4.0", prs=[221]), ], ),