From 966e5d27269f4ee44cf95ed9a3d27d04287e4b8a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 25 Feb 2026 23:18:01 +0000 Subject: [PATCH 1/6] [AIC-1636] fix: Improve usage reporting - Rename usage tracking event keys to $ld:ai:usage:* prefix - $ld:ai:config:function:single -> $ld:ai:usage:completion-config - $ld:ai:judge:function:single -> $ld:ai:usage:judge-config - Add SDK info tracking on AIClient construction ($ld:ai:sdk:info) with anonymous ld_ai context and SDK metadata - Rename Config method to CompletionConfig; keep Config as deprecated proxy - Update usage tracking data format to use object with configKey field - Update tests for new event keys, data format, and SDK info event Co-Authored-By: unknown <> --- ldai/client.go | 55 +++++++++++++++++++++++++++++----- ldai/client_test.go | 72 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 112 insertions(+), 15 deletions(-) diff --git a/ldai/client.go b/ldai/client.go index a1e4c9fa..f3ffab6a 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -40,16 +40,43 @@ type Client struct { logger interfaces.LDLoggers } +const ( + // usageCompletionConfig is the event key for completion config usage tracking. + usageCompletionConfig = "$ld:ai:usage:completion-config" + // usageJudgeConfig is the event key for judge config usage tracking. + usageJudgeConfig = "$ld:ai:usage:judge-config" + // sdkInfoEvent is the event key for SDK info tracking. + sdkInfoEvent = "$ld:ai:sdk:info" +) + // NewClient creates a new AI Client. The provided SDK interface must not be nil. The client will use the provided SDK's // loggers to log warnings and errors. +// +// Upon construction, the client fires a single $ld:ai:sdk:info tracking event containing SDK metadata. func NewClient(sdk ServerSDK) (*Client, error) { if sdk == nil { return nil, fmt.Errorf("sdk must not be nil") } - return &Client{ + c := &Client{ sdk: sdk, logger: sdk.Loggers(), - }, nil + } + c.trackSDKInfo() + return c, nil +} + +func (c *Client) trackSDKInfo() { + ctx, err := ldcontext.NewBuilder("ld-internal-tracking").Kind("ld_ai").Anonymous(true).TryBuild() + if err != nil { + c.logger.Warnf("AI Client: failed to build SDK info context: %v", err) + return + } + data := ldvalue.ObjectBuild(). + Set("aiSdkName", ldvalue.String("launchdarkly-go-server-sdk-ai")). + Set("aiSdkVersion", ldvalue.String(Version)). + Set("aiSdkLanguage", ldvalue.String("go")). + Build() + _ = c.sdk.TrackMetric(sdkInfoEvent, ctx, 1, data) } func (c *Client) logConfigWarning(key string, format string, args ...interface{}) { @@ -57,23 +84,36 @@ func (c *Client) logConfigWarning(key string, format string, args ...interface{} c.logger.Warnf(prefix+format, args...) } -// Config evaluates an AI Config named by a given key for the given context. +// CompletionConfig evaluates an AI Completion Config named by a given key for the given context. // // The config's messages will undergo Mustache template interpolation using the provided variables, which may be // nil. If the config cannot be evaluated or LaunchDarkly is unreachable, the default value is returned. Note that // the messages in the default will not undergo template interpolation. // // To send analytic events to LaunchDarkly related to the AI Config, call methods on the returned Tracker. -func (c *Client) Config( +func (c *Client) CompletionConfig( key string, context ldcontext.Context, defaultValue Config, variables map[string]interface{}, ) (Config, *Tracker) { - _ = c.sdk.TrackMetric("$ld:ai:config:function:single", context, 1, ldvalue.String(key)) + data := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(key)).Build() + _ = c.sdk.TrackMetric(usageCompletionConfig, context, 1, data) return c.evaluateConfig(key, context, defaultValue, variables) } +// Config evaluates an AI Config named by a given key for the given context. +// +// Deprecated: Use CompletionConfig instead. +func (c *Client) Config( + key string, + context ldcontext.Context, + defaultValue Config, + variables map[string]interface{}, +) (Config, *Tracker) { + return c.CompletionConfig(key, context, defaultValue, variables) +} + // evaluateConfig fetches and interpolates an AI Config without emitting any metric. // Callers (Config, JudgeConfig) are meant to emit their own metric before calling this. func (c *Client) evaluateConfig( @@ -189,7 +229,7 @@ func interpolateTemplate(template string, variables map[string]interface{}) (str return m.RenderString(variables) } -// JudgeConfig evaluates an AI Config, tracking it as a judge function. See Config for details. +// JudgeConfig evaluates an AI Config, tracking it as a judge function. See CompletionConfig for details. // // This method extends the provided variables with reserved judge variables: // - "message_history": "{{message_history}}" @@ -203,7 +243,8 @@ func (c *Client) JudgeConfig( defaultValue Config, variables map[string]interface{}, ) (Config, *Tracker) { - _ = c.sdk.TrackMetric("$ld:ai:judge:function:single", context, 1, ldvalue.String(key)) + data := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(key)).Build() + _ = c.sdk.TrackMetric(usageJudgeConfig, context, 1, data) // Extend variables with reserved judge placeholders extendedVariables := make(map[string]interface{}) diff --git a/ldai/client_test.go b/ldai/client_test.go index d145e5b5..3a794e1e 100644 --- a/ldai/client_test.go +++ b/ldai/client_test.go @@ -67,9 +67,24 @@ func TestNewClientReturnsErrorWhenSDKIsNil(t *testing.T) { } func TestNewClient(t *testing.T) { - client, err := NewClient(newMockSDK(nil, nil)) + mockSDK := newMockSDK(nil, nil) + client, err := NewClient(mockSDK) require.NoError(t, err) require.NotNil(t, client) + + // Verify SDK info event was fired on construction. + require.Len(t, mockSDK.events, 1) + evt := mockSDK.events[0] + assert.Equal(t, "$ld:ai:sdk:info", evt.eventName) + assert.Equal(t, float64(1), evt.metricValue) + assert.Equal(t, "launchdarkly-go-server-sdk-ai", evt.data.GetByKey("aiSdkName").StringValue()) + assert.Equal(t, Version, evt.data.GetByKey("aiSdkVersion").StringValue()) + assert.Equal(t, "go", evt.data.GetByKey("aiSdkLanguage").StringValue()) + + // Verify the context is anonymous with kind ld_ai. + assert.Equal(t, "ld-internal-tracking", evt.context.Key()) + assert.True(t, evt.context.Anonymous()) + assert.Equal(t, ldcontext.Kind("ld_ai"), evt.context.Kind()) } func TestEvalErrorReturnsDefault(t *testing.T) { @@ -302,12 +317,48 @@ func TestCanSetDefaultConfigFields(t *testing.T) { assert.Equal(t, datamodel.System, msg[1].Role) } -func TestConfigMethodTracking(t *testing.T) { +func TestCompletionConfigMethodTracking(t *testing.T) { + mockSDK := newMockSDK(nil, nil) + client, err := NewClient(mockSDK) + require.NoError(t, err) + require.NotNil(t, client) + + // Clear the SDK info event from construction. + mockSDK.events = nil + + defaultConfig := NewConfig().WithEnabled(false).Build() + context := ldcontext.New("user-key") + configKey := "test-config-key" + + config, tracker := client.CompletionConfig(configKey, context, defaultConfig, nil) + + require.NotNil(t, config) + require.NotNil(t, tracker) + + expectedData := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(configKey)).Build() + expectedEvents := []mockEvent{ + { + eventName: "$ld:ai:usage:completion-config", + context: context, + metricValue: 1, + data: expectedData, + }, + } + + assert.ElementsMatch(t, expectedEvents, mockSDK.events) +} + +// TestDeprecatedConfigDelegatesToCompletionConfig verifies the deprecated Config method delegates +// to CompletionConfig and emits the same usage event. +func TestDeprecatedConfigDelegatesToCompletionConfig(t *testing.T) { mockSDK := newMockSDK(nil, nil) client, err := NewClient(mockSDK) require.NoError(t, err) require.NotNil(t, client) + // Clear the SDK info event from construction. + mockSDK.events = nil + defaultConfig := NewConfig().WithEnabled(false).Build() context := ldcontext.New("user-key") configKey := "test-config-key" @@ -317,12 +368,13 @@ func TestConfigMethodTracking(t *testing.T) { require.NotNil(t, config) require.NotNil(t, tracker) + expectedData := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(configKey)).Build() expectedEvents := []mockEvent{ { - eventName: "$ld:ai:config:function:single", + eventName: "$ld:ai:usage:completion-config", context: context, metricValue: 1, - data: ldvalue.String(configKey), + data: expectedData, }, } @@ -330,7 +382,7 @@ func TestConfigMethodTracking(t *testing.T) { } // TestJudgeConfigMethodTracking verifies that JudgeConfig emits only the judge metric, -// not the config metric, so judge evaluations are not double-counted on the dashboard. +// not the completion-config metric, so judge evaluations are not double-counted on the dashboard. func TestJudgeConfigMethodTracking(t *testing.T) { json := []byte(`{ "_ldMeta": {"variationKey": "1", "enabled": true}, @@ -343,6 +395,9 @@ func TestJudgeConfigMethodTracking(t *testing.T) { require.NoError(t, err) require.NotNil(t, client) + // Clear the SDK info event from construction. + mockSDK.events = nil + defaultConfig := Disabled() context := ldcontext.New("user-key") configKey := "judge-config-key" @@ -353,16 +408,17 @@ func TestJudgeConfigMethodTracking(t *testing.T) { require.NotNil(t, tracker) // Only the judge metric should be emitted; evaluateConfig does not emit any metric. + expectedData := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(configKey)).Build() expectedEvents := []mockEvent{ { - eventName: "$ld:ai:judge:function:single", + eventName: "$ld:ai:usage:judge-config", context: context, metricValue: 1, - data: ldvalue.String(configKey), + data: expectedData, }, } assert.ElementsMatch(t, expectedEvents, mockSDK.events, - "JudgeConfig must not emit $ld:ai:config:function:single to avoid double-counting") + "JudgeConfig must not emit $ld:ai:usage:completion-config to avoid double-counting") } func TestCanSetModelParameters(t *testing.T) { From ecb53088c54fb2a07c0889d82d5043934b4bdf67 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:07:48 +0000 Subject: [PATCH 2/6] Address PR feedback: alphabetize constants, remove unnecessary comment Co-Authored-By: unknown <> --- ldai/client.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ldai/client.go b/ldai/client.go index f3ffab6a..50b3172d 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -41,18 +41,13 @@ type Client struct { } const ( - // usageCompletionConfig is the event key for completion config usage tracking. + sdkInfoEvent = "$ld:ai:sdk:info" usageCompletionConfig = "$ld:ai:usage:completion-config" - // usageJudgeConfig is the event key for judge config usage tracking. - usageJudgeConfig = "$ld:ai:usage:judge-config" - // sdkInfoEvent is the event key for SDK info tracking. - sdkInfoEvent = "$ld:ai:sdk:info" + usageJudgeConfig = "$ld:ai:usage:judge-config" ) // NewClient creates a new AI Client. The provided SDK interface must not be nil. The client will use the provided SDK's // loggers to log warnings and errors. -// -// Upon construction, the client fires a single $ld:ai:sdk:info tracking event containing SDK metadata. func NewClient(sdk ServerSDK) (*Client, error) { if sdk == nil { return nil, fmt.Errorf("sdk must not be nil") From 756024ace316a5e12ffd47ede2a3ddc07ed14a72 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:12:25 +0000 Subject: [PATCH 3/6] Update JudgeConfig doc comment with better description Co-Authored-By: unknown <> --- ldai/client.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ldai/client.go b/ldai/client.go index 50b3172d..f76d7336 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -224,7 +224,8 @@ func interpolateTemplate(template string, variables map[string]interface{}) (str return m.RenderString(variables) } -// JudgeConfig evaluates an AI Config, tracking it as a judge function. See CompletionConfig for details. +// JudgeConfig retrieves and processes a Judge AI Config based on the provided key, LaunchDarkly context, and +// variables. This includes the model configuration and the customized messages for evaluation. // // This method extends the provided variables with reserved judge variables: // - "message_history": "{{message_history}}" @@ -232,6 +233,8 @@ func interpolateTemplate(template string, variables map[string]interface{}) (str // // These literal placeholder strings preserve the Mustache templates through the first interpolation // (during config fetch), allowing Judge.Evaluate() to perform a second interpolation with actual values. +// +// To send analytic events to LaunchDarkly related to the AI Config, call methods on the returned Tracker. func (c *Client) JudgeConfig( key string, context ldcontext.Context, From 948c8cb7587f5d031e21c84df71cb8a6e3cbcb59 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:13:01 +0000 Subject: [PATCH 4/6] Update CompletionConfig doc comment to match JudgeConfig style Co-Authored-By: unknown <> --- ldai/client.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ldai/client.go b/ldai/client.go index f76d7336..095361b2 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -79,7 +79,8 @@ func (c *Client) logConfigWarning(key string, format string, args ...interface{} c.logger.Warnf(prefix+format, args...) } -// CompletionConfig evaluates an AI Completion Config named by a given key for the given context. +// CompletionConfig retrieves and processes a Completion AI Config based on the provided key, LaunchDarkly context, +// and variables. This includes the model configuration and the customized messages. // // The config's messages will undergo Mustache template interpolation using the provided variables, which may be // nil. If the config cannot be evaluated or LaunchDarkly is unreachable, the default value is returned. Note that From c6ed67cc6f43dd295106f0f1585177d3d28f7d4f Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:18:57 +0000 Subject: [PATCH 5/6] Move SDK name and language constants to package_info.go Co-Authored-By: unknown <> --- ldai/client.go | 4 ++-- ldai/package_info.go | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/ldai/client.go b/ldai/client.go index 095361b2..76cc30f2 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -67,9 +67,9 @@ func (c *Client) trackSDKInfo() { return } data := ldvalue.ObjectBuild(). - Set("aiSdkName", ldvalue.String("launchdarkly-go-server-sdk-ai")). + Set("aiSdkName", ldvalue.String(SDKName)). Set("aiSdkVersion", ldvalue.String(Version)). - Set("aiSdkLanguage", ldvalue.String("go")). + Set("aiSdkLanguage", ldvalue.String(SDKLanguage)). Build() _ = c.sdk.TrackMetric(sdkInfoEvent, ctx, 1, data) } diff --git a/ldai/package_info.go b/ldai/package_info.go index 117a425e..2b24d250 100644 --- a/ldai/package_info.go +++ b/ldai/package_info.go @@ -1,5 +1,13 @@ // Package ldai contains an AI SDK suitable for usage with generative AI applications. package ldai -// Version is the current version string of the ldai package. This is updated by our release scripts. -const Version = "0.8.0" // {{ x-release-please-version }} +const ( + // Version is the current version string of the ldai package. This is updated by our release scripts. + Version = "0.8.0" // {{ x-release-please-version }} + + // SDKName is the canonical name of this AI SDK package. + SDKName = "launchdarkly-go-server-sdk-ai" + + // SDKLanguage is the programming language of this AI SDK. + SDKLanguage = "go" +) From 5d2ca436c933d12db5acffc440edd35fbe73a8f0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 26 Feb 2026 15:30:29 +0000 Subject: [PATCH 6/6] Refactor trackSDKInfo to return error with single logging point Co-Authored-By: unknown <> --- ldai/client.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ldai/client.go b/ldai/client.go index 76cc30f2..b5c797cd 100644 --- a/ldai/client.go +++ b/ldai/client.go @@ -56,22 +56,23 @@ func NewClient(sdk ServerSDK) (*Client, error) { sdk: sdk, logger: sdk.Loggers(), } - c.trackSDKInfo() + if err := c.trackSDKInfo(); err != nil { + c.logger.Warnf("AI Client: failed to track SDK info: %v", err) + } return c, nil } -func (c *Client) trackSDKInfo() { +func (c *Client) trackSDKInfo() error { ctx, err := ldcontext.NewBuilder("ld-internal-tracking").Kind("ld_ai").Anonymous(true).TryBuild() if err != nil { - c.logger.Warnf("AI Client: failed to build SDK info context: %v", err) - return + return err } data := ldvalue.ObjectBuild(). Set("aiSdkName", ldvalue.String(SDKName)). Set("aiSdkVersion", ldvalue.String(Version)). Set("aiSdkLanguage", ldvalue.String(SDKLanguage)). Build() - _ = c.sdk.TrackMetric(sdkInfoEvent, ctx, 1, data) + return c.sdk.TrackMetric(sdkInfoEvent, ctx, 1, data) } func (c *Client) logConfigWarning(key string, format string, args ...interface{}) {