Align GenAI provider semantic convention

dashboards/datadog-dashboard.json

@@ -85,13 +85,13 @@
     {
       "id": 4,
       "definition": {
-        "title": "Error Rate by System",
+        "title": "Error Rate by Provider",
         "title_size": "16",
         "title_align": "left",
         "type": "query_value",
         "requests": [
           {
-            "q": "sum:gen_ai.client.operation.error{*} by {gen_ai.system}.as_rate() / sum:gen_ai.client.operation.duration{*} by {gen_ai.system}.as_count()",
+            "q": "sum:gen_ai.client.operation.error{*} by {gen_ai.provider.name}.as_rate() / sum:gen_ai.client.operation.duration{*} by {gen_ai.provider.name}.as_count()",
             "aggregator": "last"
           }
         ],
@@ -232,9 +232,9 @@
       "prefix": "gen_ai.request.model"
     },
     {
-      "name": "system",
+      "name": "provider",
       "default": "*",
-      "prefix": "gen_ai.system"
+      "prefix": "gen_ai.provider.name"
     }
   ],
   "layout_type": "ordered",

docs/contract/eval2otel-v1.md

@@ -19,7 +19,7 @@ Every converted evaluation creates one client span named by operation:
 Every span must include:
 
 - `gen_ai.operation.name`
-- `gen_ai.system`
+- `gen_ai.provider.name`
 - `evalops.contract.version`
 - `evalops.semconv.version`
 - `evalops.eval.id`
@@ -28,8 +28,9 @@ Every span must include:
 - `evalops.redacted_content_count`
 - `evalops.truncated_content_count`
 
-Provider-aware conversions should include `gen_ai.provider.name` using the
-normalized provider names exported by `normalizeProviderName`.
+`gen_ai.provider.name` uses the normalized provider names exported by
+`normalizeProviderName`. `gen_ai.system` is intentionally not emitted because it
+is no longer present in the upstream OpenTelemetry GenAI registry.
 
 ## Attribute Registry
 

package.json

@@ -50,7 +50,8 @@
     "jest": "^29.5.0",
     "prettier": "^3.0.0",
     "ts-jest": "^29.1.0",
-    "typescript": "^5.0.0"
+    "typescript": "^5.0.0",
+    "yaml": "^2.9.0"
   },
   "dependencies": {
     "@opentelemetry/api": "^1.9.1",

python/eval2otel/contract.py

@@ -12,6 +12,23 @@
 UNKNOWN_SEMCONV_VERSION = "unspecified"
 
 
+def normalize_provider_name(system: str | None) -> str | None:
+    if not system:
+        return None
+    value = system.lower()
+    if "azure" in value:
+        return "azure.openai"
+    if "bedrock" in value or "aws" in value:
+        return "aws.bedrock"
+    if "vertex" in value or "gemini" in value or "google" in value:
+        return "google.vertex"
+    if "anthropic" in value or "claude" in value:
+        return "anthropic"
+    if "openai" in value:
+        return "openai"
+    return value
+
+
 def sha256_payload(value: Any) -> str:
     encoded = json.dumps(value, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8")
     return hashlib.sha256(encoded).hexdigest()
@@ -302,7 +319,7 @@ def build_span_attributes(
 ) -> MutableMapping[str, str | int | float]:
     attrs: MutableMapping[str, str | int | float] = build_eval2otel_attributes(eval_result, semconv_version)
     attrs["gen_ai.operation.name"] = eval_result.operation
-    attrs["gen_ai.system"] = eval_result.system or "unknown"
+    attrs["gen_ai.provider.name"] = normalize_provider_name(eval_result.system) or "unknown"
     if eval_result.request.get("model") is not None:
         attrs["gen_ai.request.model"] = str(eval_result.request["model"])
     if eval_result.response.get("model") is not None:

python/tests/test_contract.py

@@ -100,7 +100,8 @@ def test_process_evaluation_emits_span_and_content_events_with_injected_tracer(s
 
         self.assertEqual(report.event_count, 2)
         self.assertEqual(tracer.spans[0].name, "gen_ai.chat")
-        self.assertEqual(tracer.spans[0].attributes["gen_ai.system"], "openai")
+        self.assertEqual(tracer.spans[0].attributes["gen_ai.provider.name"], "openai")
+        self.assertNotIn("gen_ai.system", tracer.spans[0].attributes)
         self.assertEqual(tracer.spans[0].attributes["gen_ai.usage.input_tokens"], 3)
         self.assertEqual(tracer.spans[0].events[0][0], "gen_ai.user.message")
         self.assertIn("evalops.content_sha256", tracer.spans[0].events[0][1])

src/converter.ts

@@ -246,18 +246,13 @@ export class Eval2OtelConverter {
     evalResult: EvalResult, 
     additionalAttributes?: Record<string, string | number | boolean>
   ): GenAIAttributes {
+    const provider = normalizeProviderName(evalResult.system) ?? 'unknown';
     const attributes: GenAIAttributes = {
       'gen_ai.operation.name': evalResult.operation,
-      'gen_ai.system': evalResult.system ?? 'unknown',
+      [ATTR.PROVIDER_NAME]: provider,
       ...buildEval2OtelAttributes(evalResult, this.config),
     };
 
-    // Provider discriminator aligned with latest GenAI semconv
-    const provider = normalizeProviderName(evalResult.system);
-    if (provider) {
-      (attributes as any)[ATTR.PROVIDER_NAME] = provider;
-    }
-
     // Add service attributes
     if (this.config.environment) {
       attributes['deployment.environment'] = this.config.environment;
@@ -454,7 +449,6 @@ export class Eval2OtelConverter {
     evalResult.conversation.messages.forEach((message, index) => {
       const eventName = `gen_ai.${message.role}.message`;
       const attributes: Record<string, string | number | boolean> = {
-        'gen_ai.system': evalResult.system ?? 'unknown',
         [ATTR.PROVIDER_NAME]: normalizeProviderName(evalResult.system) ?? 'unknown',
         [ATTR.MESSAGE_ROLE]: message.role,
         [ATTR.MESSAGE_INDEX]: index,
@@ -524,7 +518,6 @@ export class Eval2OtelConverter {
 
     evalResult.response.choices.forEach((choice) => {
       const attributes: Record<string, string | number | boolean> = {
-        'gen_ai.system': evalResult.system ?? 'unknown',
         [ATTR.PROVIDER_NAME]: normalizeProviderName(evalResult.system) ?? 'unknown',
         [ATTR.RESPONSE_CHOICE_INDEX]: choice.index,
         [ATTR.RESPONSE_FINISH_REASON]: choice.finishReason,
@@ -583,7 +576,7 @@ export class Eval2OtelConverter {
             : JSON.stringify(toolCall.function.arguments);
           if (this.canAddEvent(span)) {
             span.addEvent('gen_ai.tool.message', {
-            'gen_ai.system': evalResult.system ?? 'unknown',
+            [ATTR.PROVIDER_NAME]: normalizeProviderName(evalResult.system) ?? 'unknown',
             [ATTR.TOOL_NAME]: toolCall.function.name,
             [ATTR.TOOL_CALL_ID]: toolCall.id,
             [ATTR.RESPONSE_CHOICE_INDEX]: choice.index,

src/metrics.ts

@@ -1,4 +1,5 @@
 import { metrics, Counter, Histogram, Meter, context as otContext } from '@opentelemetry/api';
+import { ATTR } from './attributes';
 import { normalizeProviderName } from './contract';
 import { getRagMetricValue } from './rag';
 import { ConversionReport, EvalResult, OtelConfig, ProcessOptions } from './types';
@@ -208,17 +209,11 @@ export class Eval2OtelMetrics {
   recordMetrics(evalResult: EvalResult, options?: ProcessOptions): void {
     const baseAttributes = {
       'gen_ai.operation.name': evalResult.operation,
-      'gen_ai.system': evalResult.system ?? 'unknown',
+      [ATTR.PROVIDER_NAME]: normalizeProviderName(evalResult.system) ?? 'unknown',
       'gen_ai.request.model': evalResult.request.model,
       'gen_ai.response.model': evalResult.response.model ?? evalResult.request.model,
     };
 
-    // Provider discriminator aligned with latest GenAI semconv
-    const provider = normalizeProviderName(evalResult.system);
-    if (provider) {
-      (baseAttributes as any)['gen_ai.provider.name'] = provider;
-    }
-
     // Add error type if present
     const attributes: Record<string, string | number> = { ...baseAttributes };
     if (evalResult.error) {
@@ -388,9 +383,8 @@ export class Eval2OtelMetrics {
       attrs['gen_ai.operation.name'] = evalResult.operation;
     }
     if (evalResult?.system) {
-      attrs['gen_ai.system'] = evalResult.system;
       const provider = normalizeProviderName(evalResult.system);
-      if (provider) attrs['gen_ai.provider.name'] = provider;
+      if (provider) attrs[ATTR.PROVIDER_NAME] = provider;
     }
     if (evalResult?.request?.model) {
       attrs['gen_ai.request.model'] = evalResult.request.model;
@@ -506,7 +500,7 @@ export class Eval2OtelMetrics {
   }): void {
     const baseAttributes = {
       'gen_ai.operation.name': evalResult.operation,
-      'gen_ai.system': evalResult.system ?? 'unknown',
+      [ATTR.PROVIDER_NAME]: normalizeProviderName(evalResult.system) ?? 'unknown',
       'gen_ai.request.model': evalResult.request.model,
     };
 

src/semconv.ts

@@ -12,7 +12,6 @@ export interface AttributeSpec {
 
 export const ATTRIBUTE_REGISTRY: AttributeSpec[] = [
   { key: 'gen_ai.operation.name', source: 'otel-genai', signal: 'all', stability: 'stable', description: 'GenAI operation name.' },
-  { key: 'gen_ai.system', source: 'otel-genai', signal: 'all', stability: 'stable', description: 'AI system or provider family.' },
   { key: 'error.type', source: 'otel-genai', signal: 'all', stability: 'stable', description: 'Error type for failed operations.' },
   { key: 'deployment.environment', source: 'otel-genai', signal: 'all', stability: 'stable', description: 'Deployment environment resource/context attribute.' },
   { key: ATTR.PROVIDER_NAME, source: 'otel-genai', signal: 'all', stability: 'stable', description: 'Normalized provider name.' },

src/types.ts

@@ -338,7 +338,7 @@ export interface ProcessOptions {
 export interface GenAIAttributes {
   // Required
   'gen_ai.operation.name': string;
-  'gen_ai.system': string;
+  'gen_ai.provider.name': string;
 
   // Conditionally required
   'error.type'?: string;

test-harness/README.md

@@ -74,7 +74,7 @@ curl "http://localhost:16686/api/traces?service=eval2otel-e2e-test"
 ### Traces in Jaeger
 - Service: `eval2otel-e2e-test`
 - Span names: `gen_ai.chat`, `gen_ai.embeddings`, `gen_ai.execute_tool`
-- Attributes: `gen_ai.system=openai`, `gen_ai.request.model=gpt-4`, etc.
+- Attributes: `gen_ai.provider.name=openai`, `gen_ai.request.model=gpt-4`, etc.
 - Events: Message events (when `captureContent=true`)
 
 ### Metrics in Prometheus