diff --git a/.github/ISSUE_TEMPLATE/bug_report--ldai.md b/.github/ISSUE_TEMPLATE/bug_report--ldai.md deleted file mode 100644 index f711d2e8..00000000 --- a/.github/ISSUE_TEMPLATE/bug_report--ldai.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: Bug report for the ldai module -about: Create a report to help us improve -title: '' -labels: 'ldai, enhancement' -assignees: '' - ---- - -**Is this a support request?** -This issue tracker is maintained by LaunchDarkly SDK developers and is intended for feedback on the SDK code. If you're not sure whether the problem you are having is specifically related to the SDK, or to the LaunchDarkly service overall, it may be more appropriate to contact the LaunchDarkly support team; they can help to investigate the problem and will consult the SDK team if necessary. You can submit a support request by going [here](https://support.launchdarkly.com/hc/en-us/requests/new) or by emailing support@launchdarkly.com. - -Note that issues filed on this issue tracker are publicly accessible. Do not provide any private account information on your issues. If your problem is specific to your account, you should submit a support request as described above. - -**Describe the bug** -A clear and concise description of what the bug is. - -**To reproduce** -Steps to reproduce the behavior. - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Logs** -If applicable, add any log output related to your problem. - -**SDK version** -The version of this SDK that you are using. - -**Language version, developer tools** -For instance, Go 1.22. - -**OS/platform** -For instance, Ubuntu 16.04, or Windows 10. - -**Additional context** -Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request--ldai.md b/.github/ISSUE_TEMPLATE/feature_request--ldai.md deleted file mode 100644 index 65733fc8..00000000 --- a/.github/ISSUE_TEMPLATE/feature_request--ldai.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Feature request for the ldai module -about: Suggest an idea for this project -title: '' -labels: 'ldai, enhancement' -assignees: '' - ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I would love to see the SDK [...does something new...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context about the feature request here. diff --git a/.github/workflows/ldai-ci.yml b/.github/workflows/ldai-ci.yml deleted file mode 100644 index 8caa2e26..00000000 --- a/.github/workflows/ldai-ci.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: Build and Test ldai -permissions: - contents: read -on: - push: - branches: [ 'v7', 'feat/**' ] - paths-ignore: - - '**.md' # Don't run CI on markdown changes. - pull_request: - branches: [ 'v7', 'feat/**' ] - paths-ignore: - - '**.md' - -jobs: - go-versions: - uses: ./.github/workflows/go-versions.yml - - # Runs the common tasks (unit tests, lint, contract tests) for each Go version. - test-linux: - name: ${{ format('ldai Linux, Go {0}', matrix.go-version) }} - needs: go-versions - strategy: - # Let jobs fail independently, in case it's a single version that's broken. - fail-fast: false - matrix: - go-version: ${{ fromJSON(needs.go-versions.outputs.matrix) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Setup Go ${{ inputs.go-version }} - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go-version }} - - uses: ./.github/actions/unit-tests - with: - lint: ${{ matrix.go-version == needs.go-versions.outputs.latest }} - test-target: ldai-test - - uses: ./.github/actions/coverage - with: - enforce: 'false' diff --git a/.release-please-manifest.json b/.release-please-manifest.json index ed938ce2..0bc318d4 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,6 +1,5 @@ { ".": "7.17.0", "ldotel": "1.3.5", - "ldai": "0.9.4", "ldmiddleware": "0.1.3" } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0b2a0b28..b1a0baab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ The SDK is tested against three Go versions: the latest, penultimate, and a mini Whereas the latest and penultimate are updates on a regular cadence to track upstream Go releases, the minimum version may be bumped at the discretion of the SDK maintainers to take advantage of new features. -Invoke the following make command, which will update `go.mod`, `ldotel/go.mod`, `ldai/go.mod`, `ldmiddleware/go.mod`, +Invoke the following make command, which will update `go.mod`, `ldotel/go.mod`, `ldmiddleware/go.mod`, `testservice/go.mod`, and `.github/variables/go-versions.env` (pass the desired Go version): ```shell make bump-min-go-version MIN_GO_VERSION=1.18 diff --git a/Makefile b/Makefile index 231e459a..8dad8f5b 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ COVERAGE_ENFORCER_FLAGS=-package github.com/launchdarkly/go-server-sdk/v7 \ -skipcode "// COVERAGE" \ -packagestats -filestats -showcode -ALL_BUILD_TARGETS=sdk ldotel ldai ldmiddleware +ALL_BUILD_TARGETS=sdk ldotel ldmiddleware ALL_TEST_TARGETS = $(addsuffix -test, $(ALL_BUILD_TARGETS)) ALL_LINT_TARGETS = $(addsuffix -lint, $(ALL_BUILD_TARGETS)) @@ -35,7 +35,6 @@ test: $(ALL_TEST_TARGETS) bump-min-go-version: go mod edit -go=$(MIN_GO_VERSION) go.mod cd ldotel && go mod edit -go=$(MIN_GO_VERSION) go.mod - cd ldai && go mod edit -go=$(MIN_GO_VERSION) go.mod cd ldmiddleware && go mod edit -go=$(MIN_GO_VERSION) go.mod cd testservice && go mod edit -go=$(MIN_GO_VERSION) go.mod cd ./.github/variables && sed -i.bak "s#min=[^ ]*#min=$(MIN_GO_VERSION)#g" go-versions.env && rm go-versions.env.bak @@ -83,33 +82,6 @@ ldotel-lint: $(LINTER_VERSION_FILE) cd ldotel && ../$(LINTER) run .; \ fi -ldai: - @if [ -f go.work ]; then \ - echo "Building ldai with workspace"; \ - go build ./ldai; \ - else \ - echo "Building ldai without workspace"; \ - cd ldai && go build .; \ - fi - -ldai-test: - @if [ -f go.work ]; then \ - echo "Testing ldai with workspace"; \ - go test -v -race ./ldai; \ - else \ - echo "Testing ldai without workspace"; \ - cd ldai && go test -v -race .; \ - fi - -ldai-lint: $(LINTER_VERSION_FILE) - @if [ -f go.work ]; then \ - echo "Linting ldai with workspace"; \ - $(LINTER) run ./ldai; \ - else \ - echo "Linting ldai without workspace"; \ - cd ldai && ../$(LINTER) run .; \ - fi - ldmiddleware: @if [ -f go.work ]; then \ echo "Building ldmiddleware with workspace"; \ @@ -186,7 +158,6 @@ workspace: go.work go.work: go work init ./ go work use ./ldotel - go work use ./ldai go work use ./ldmiddleware go work use ./testservice diff --git a/ldai/LICENSE.txt b/ldai/LICENSE.txt deleted file mode 100644 index 50add35e..00000000 --- a/ldai/LICENSE.txt +++ /dev/null @@ -1,13 +0,0 @@ -Copyright 2024 Catamorphic, Co. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. diff --git a/ldai/README.md b/ldai/README.md index 619f5a73..6b5a7cb9 100644 --- a/ldai/README.md +++ b/ldai/README.md @@ -1,81 +1,18 @@ LaunchDarkly Server-side AI SDK for Go ============================================== -[![Actions Status](https://github.com/launchdarkly/go-server-sdk/actions/workflows/ldai-ci.yml/badge.svg?branch=v7)](https://github.com/launchdarkly/go-server-sdk/actions/workflows/ldai-ci.yml) +This package has moved to its own repository: +**[launchdarkly/go-server-sdk-ai](https://github.com/launchdarkly/go-server-sdk-ai)** -> [!CAUTION] -> This AI SDK is in pre-release and not subject to backwards compatibility guarantees. The API may change based on feedback. -> -> Pin to a specific minor version and review the [changelog] before upgrading. -> -> Active feature development is ongoing in the [Python][python-ai-sdk] and [Node.js][node-ai-sdk] AI SDKs, so this SDK will receive new features at a slower pace. Refer to those for the latest capabilities. +Update your import path: -[changelog]: https://github.com/launchdarkly/go-server-sdk/blob/v7/ldai/CHANGELOG.md -[python-ai-sdk]: https://github.com/launchdarkly/python-server-sdk-ai/tree/main/packages/sdk/server-ai -[node-ai-sdk]: https://github.com/launchdarkly/js-core/tree/main/packages/sdk/server-ai - -LaunchDarkly overview -------------------------- -[LaunchDarkly](https://www.launchdarkly.com) is a feature management platform that serves trillions of feature flags daily to help teams build better software, faster. [Get started](https://docs.launchdarkly.com/home/getting-started) using LaunchDarkly today! - -[![Twitter Follow](https://img.shields.io/twitter/follow/launchdarkly.svg?style=social&label=Follow&maxAge=2592000)](https://twitter.com/intent/follow?screen_name=launchdarkly) - -Getting started ------------ - -Import the module: - -```go -import ( - ld "github.com/launchdarkly/go-server-sdk/v7" - "github.com/launchdarkly/go-server-sdk/ldai" -) -``` - -Configure the base LaunchDarkly Server SDK: - -```go -sdkClient, _ = ld.MakeClient("your-sdk-key", 5*time.Second) -``` - -Instantiate the AI client, passing in the base Server SDK: -```go -aiClient, err := ldai.NewClient(sdkClient) -``` - -Fetch a model configuration for a specific LaunchDarkly context: ```go -// The default value 'ldai.Disabled()' be returned if LaunchDarkly is unavailable or the config -// cannot be fetched. To customize the default value, use ldai.NewConfig(). -config := aiClient.CompletionConfig("your-model-key", ldcontext.New("user-key"), ldai.Disabled(), nil) - -// Access the methods on config. To generate analytic events related to usage of the model config, -// obtain a tracker by calling config.CreateTracker(). -tracker := config.CreateTracker() +import "github.com/launchdarkly/go-server-sdk-ai/ldai" ``` -Learn more ------------ - -Read our [documentation](http://docs.launchdarkly.com) for in-depth instructions on configuring and using LaunchDarkly. -You can also head straight to the [complete reference guide for this SDK](https://docs.launchdarkly.com/sdk/ai/go). - - -Contributing ------------- - -We encourage pull requests and other contributions from the community. Check out our [contributing guidelines](../CONTRIBUTING.md) for instructions on how to contribute to this library. -About LaunchDarkly ------------ +Version numbers continue from where this package stopped. The last release from +this repository was `0.9.4`. The next release is `0.10.0`, from the new +repository. -* LaunchDarkly is a continuous delivery platform that provides feature flags as a service and allows developers to iterate quickly and safely. We allow you to easily flag your features and manage them from the LaunchDarkly dashboard. With LaunchDarkly, you can: - * Roll out a new feature to a subset of your users (like a group of users who opt-in to a beta tester group), gathering feedback and bug reports from real-world use cases. - * Gradually roll out a feature to an increasing percentage of users, and track the effect that the feature has on key metrics (for instance, how likely is a user to complete a purchase if they have feature A versus feature B?). - * Turn off a feature that you realize is causing performance problems in production, without needing to re-deploy, or even restart the application with a changed configuration file. - * Grant access to certain features based on user attributes, like payment plan (eg: users on the ‘gold’ plan get access to more features than users in the ‘silver’ plan). Disable parts of your application to facilitate maintenance, without taking everything offline. -* LaunchDarkly provides feature flag SDKs for a wide variety of languages and technologies. Read [our documentation](https://docs.launchdarkly.com/sdk) for a complete list. -* Explore LaunchDarkly - * [launchdarkly.com](https://www.launchdarkly.com/ "LaunchDarkly Main Website") for more information - * [docs.launchdarkly.com](https://docs.launchdarkly.com/ "LaunchDarkly Documentation") for our documentation and SDK reference guides - * [apidocs.launchdarkly.com](https://apidocs.launchdarkly.com/ "LaunchDarkly API Documentation") for our API documentation - * [blog.launchdarkly.com](https://blog.launchdarkly.com/ "LaunchDarkly Blog Documentation") for the latest product updates +The [CHANGELOG](./CHANGELOG.md) in this directory keeps the release history up +to `0.9.4`. diff --git a/ldai/client.go b/ldai/client.go deleted file mode 100644 index 76eddc5e..00000000 --- a/ldai/client.go +++ /dev/null @@ -1,273 +0,0 @@ -package ldai - -import ( - "encoding/json" - "fmt" - - "github.com/launchdarkly/go-server-sdk/ldai/datamodel" - - "github.com/alexkappa/mustache" - - "github.com/launchdarkly/go-sdk-common/v3/ldcontext" - "github.com/launchdarkly/go-sdk-common/v3/ldvalue" - "github.com/launchdarkly/go-server-sdk/v7/interfaces" -) - -// Defines the Mustache variable name used to access the provided context. -const ldContextVariable = "ldctx" - -// JudgePlaceholderMessageHistory and JudgePlaceholderResponseToEvaluate are the literal placeholder -// strings shared between JudgeConfig (pass 1) and Judge.buildMessages (pass 2). Both must use the -// same values or substitution silently fails. -const ( - JudgePlaceholderMessageHistory = "{{message_history}}" - JudgePlaceholderResponseToEvaluate = "{{response_to_evaluate}}" -) - -// ServerSDK defines the required methods for the AI SDK to interact with LaunchDarkly. These methods are -// satisfied by the LaunchDarkly Go Server SDK. -type ServerSDK interface { - JSONVariation( - key string, - context ldcontext.Context, - defaultVal ldvalue.Value, - ) (ldvalue.Value, error) - Loggers() interfaces.LDLoggers - TrackMetric( - eventName string, - context ldcontext.Context, - metricValue float64, - data ldvalue.Value, - ) error -} - -// Client is the main entrypoint for the AI SDK. A client can be used to obtain an AI Config from LaunchDarkly. -// Unless otherwise noted, the Client's method are not safe for concurrent use. -type Client struct { - sdk ServerSDK - logger interfaces.LDLoggers -} - -const ( - sdkInfoEvent = "$ld:ai:sdk:info" - usageCompletionConfig = "$ld:ai:usage:completion-config" - usageJudgeConfig = "$ld:ai:usage:judge-config" -) - -// NewClient creates a new AI Client. The provided SDK interface must not be nil. The client will use the provided SDK's -// loggers to log warnings and errors. -func NewClient(sdk ServerSDK) (*Client, error) { - if sdk == nil { - return nil, fmt.Errorf("sdk must not be nil") - } - c := &Client{ - sdk: sdk, - logger: sdk.Loggers(), - } - if err := c.trackSDKInfo(); err != nil { - c.logger.Warnf("AI Client: failed to track SDK info: %v", err) - } - return c, nil -} - -func (c *Client) trackSDKInfo() error { - ctx, err := ldcontext.NewBuilder("ld-internal-tracking").Kind("ld_ai").Anonymous(true).TryBuild() - if err != nil { - return err - } - data := ldvalue.ObjectBuild(). - Set("aiSdkName", ldvalue.String(SDKName)). - Set("aiSdkVersion", ldvalue.String(Version)). - Set("aiSdkLanguage", ldvalue.String(SDKLanguage)). - Build() - return c.sdk.TrackMetric(sdkInfoEvent, ctx, 1, data) -} - -func (c *Client) logConfigWarning(key string, format string, args ...interface{}) { - prefix := "AI Config '" + key + "': " - c.logger.Warnf(prefix+format, args...) -} - -// CompletionConfig retrieves an AI Config and interpolates its message templates using the provided -// variables. Returns the default value if the config cannot be evaluated. Template interpolation is -// not applied to the default value's messages. -// -// To send analytic events to LaunchDarkly, call CreateTracker on the returned Config to obtain a Tracker. -func (c *Client) CompletionConfig( - key string, - context ldcontext.Context, - defaultValue Config, - variables map[string]interface{}, -) Config { - data := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(key)).Build() - _ = c.sdk.TrackMetric(usageCompletionConfig, context, 1, data) - return c.evaluateConfig(key, context, defaultValue, variables) -} - -// CreateTracker reconstructs a Tracker from a resumption token and the given context. -// This delegates to TrackerFromResumptionToken. See that function for details. -func (c *Client) CreateTracker(token string, context ldcontext.Context) (*Tracker, error) { - return TrackerFromResumptionToken(token, c.sdk, context) -} - -// returnDefault sets a tracker factory on a copy of def (so CreateTracker always works) and -// returns the resulting Config. Used for all error-path returns in evaluateConfig. -func (c *Client) returnDefault(key string, context ldcontext.Context, def Config) Config { - def.trackerFactory = func() *Tracker { - return newTracker(c.sdk, newRunID(), key, "", 1, context, &def, c.logger) - } - return def -} - -// evaluateConfig fetches and interpolates an AI Config without emitting any metric. -// Callers (CompletionConfig, JudgeConfig) are meant to emit their own metric before calling this. -func (c *Client) evaluateConfig( - key string, - context ldcontext.Context, - defaultValue Config, - variables map[string]interface{}, -) Config { - result, _ := c.sdk.JSONVariation(key, context, defaultValue.AsLdValue()) - - // The spec requires the config to at least be an object (although all properties are optional, so it may be an - // empty object.) - if result.Type() != ldvalue.ObjectType { - c.logConfigWarning(key, "unmarshalling failed, expected JSON object but got %s", result.Type().String()) - return c.returnDefault(key, context, defaultValue) - } - - var parsed datamodel.Config - if err := json.Unmarshal([]byte(result.JSONString()), &parsed); err != nil { - c.logConfigWarning(key, "unmarshalling failed: %v", err) - return c.returnDefault(key, context, defaultValue) - } - - mergedVariables := map[string]interface{}{ - ldContextVariable: getAllAttributes(context), - } - - for k, v := range variables { - if k == ldContextVariable { - c.logConfigWarning(key, "config variables contains 'ldctx', which is reserved and cannot be overwritten") - continue - } - mergedVariables[k] = v - } - - builder := NewConfig(). - WithModelName(parsed.Model.Name). - WithProviderName(parsed.Provider.Name). - WithEnabled(parsed.Meta.Enabled). - WithMode(parsed.Mode). - WithEvaluationMetricKey(parsed.EvaluationMetricKey). - WithEvaluationMetricKeys(parsed.EvaluationMetricKeys). - WithJudgeConfiguration(parsed.JudgeConfiguration) - - for k, v := range parsed.Model.Parameters { - builder.WithModelParam(k, v) - } - - for k, v := range parsed.Model.Custom { - builder.WithCustomModelParam(k, v) - } - - for i, msg := range parsed.Messages { - content, err := interpolateTemplate(msg.Content, mergedVariables) - if err != nil { - c.logConfigWarning(key, - "malformed message at index %d: %v", i, err, - ) - return c.returnDefault(key, context, defaultValue) - } - builder.WithMessage(content, msg.Role) - } - - cfg := builder.Build() - - version := 1 - if parsed.Meta.Version != nil { - version = *parsed.Meta.Version - } - - variationKey := parsed.Meta.VariationKey - cfg.trackerFactory = func() *Tracker { - return newTracker(c.sdk, newRunID(), key, variationKey, version, context, &cfg, c.logger) - } - - return cfg -} - -func getAllAttributes(context ldcontext.Context) map[string]interface{} { - if !context.Multiple() { - return addContextAttributes(context, false) - } - - attributes := map[string]interface{}{ - "kind": context.Kind(), - "key": context.FullyQualifiedKey(), - } - - for _, ctx := range context.GetAllIndividualContexts(nil) { - attributes[string(ctx.Kind())] = addContextAttributes(ctx, true) - } - - return attributes -} - -func addContextAttributes(context ldcontext.Context, omitKind bool) map[string]interface{} { - attributes := map[string]interface{}{ - "key": context.Key(), - "anonymous": context.Anonymous(), - } - - if !omitKind { - attributes["kind"] = context.Kind() - } - - for _, attr := range context.GetOptionalAttributeNames(nil) { - attributes[attr] = context.GetValue(attr).AsArbitraryValue() - } - - return attributes -} - -func interpolateTemplate(template string, variables map[string]interface{}) (string, error) { - m := mustache.New() - if err := m.ParseString(template); err != nil { - return "", err - } - return m.RenderString(variables) -} - -// JudgeConfig retrieves a Judge AI Config and interpolates its message templates. The reserved -// variables message_history and response_to_evaluate are preserved as literal placeholders for -// substitution by Judge.buildMessages during evaluation. -// -// To send analytic events to LaunchDarkly, call CreateTracker on the returned Config to obtain a Tracker. -func (c *Client) JudgeConfig( - key string, - context ldcontext.Context, - defaultValue Config, - variables map[string]interface{}, -) Config { - data := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(key)).Build() - _ = c.sdk.TrackMetric(usageJudgeConfig, context, 1, data) - - // Extend variables with reserved judge placeholders - extendedVariables := make(map[string]interface{}) - for k, v := range variables { - // Warn if user tries to override reserved variables - if k == "message_history" || k == "response_to_evaluate" { - c.logger.Warnf("AI Config '%s': variable '%s' is reserved by judge and will be ignored", key, k) - continue - } - extendedVariables[k] = v - } - - // Inject reserved variables as literal placeholder strings - // These will be preserved through the first interpolation and resolved during Judge.Evaluate() - extendedVariables["message_history"] = JudgePlaceholderMessageHistory - extendedVariables["response_to_evaluate"] = JudgePlaceholderResponseToEvaluate - - return c.evaluateConfig(key, context, defaultValue, extendedVariables) -} diff --git a/ldai/client_test.go b/ldai/client_test.go deleted file mode 100644 index 01805f1e..00000000 --- a/ldai/client_test.go +++ /dev/null @@ -1,1043 +0,0 @@ -package ldai - -import ( - "encoding/base64" - "encoding/json" - "errors" - "testing" - - "github.com/launchdarkly/go-server-sdk/ldai/datamodel" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/launchdarkly/go-sdk-common/v3/ldcontext" - "github.com/launchdarkly/go-sdk-common/v3/ldlog" - "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" - "github.com/launchdarkly/go-sdk-common/v3/ldvalue" - "github.com/launchdarkly/go-server-sdk/v7/interfaces" -) - -type mockServerSDK struct { - log *ldlogtest.MockLog - json []byte - err error - events []mockEvent -} - -type mockEvent struct { - eventName string - context ldcontext.Context - metricValue float64 - data ldvalue.Value -} - -func newMockSDK(json []byte, err error) *mockServerSDK { - return &mockServerSDK{json: json, err: err, log: ldlogtest.NewMockLog(), events: []mockEvent{}} -} - -func (m *mockServerSDK) JSONVariation( - key string, - context ldcontext.Context, - defaultVal ldvalue.Value, -) (ldvalue.Value, error) { - - if m.err != nil { - return defaultVal, m.err - } - - return ldvalue.Parse(m.json), nil -} - -func (m *mockServerSDK) Loggers() interfaces.LDLoggers { - return m.log.Loggers -} - -func (m *mockServerSDK) TrackMetric(eventName string, context ldcontext.Context, metricValue float64, data ldvalue.Value) error { - m.events = append(m.events, mockEvent{ - eventName: eventName, - context: context, - metricValue: metricValue, - data: data, - }) - return nil -} - -func TestNewClientReturnsErrorWhenSDKIsNil(t *testing.T) { - _, err := NewClient(nil) - require.Error(t, err) -} - -func TestNewClient(t *testing.T) { - mockSDK := newMockSDK(nil, nil) - client, err := NewClient(mockSDK) - require.NoError(t, err) - require.NotNil(t, client) - - // Verify SDK info event was fired on construction. - require.Len(t, mockSDK.events, 1) - evt := mockSDK.events[0] - assert.Equal(t, "$ld:ai:sdk:info", evt.eventName) - assert.Equal(t, float64(1), evt.metricValue) - assert.Equal(t, "go-server-sdk/ldai", evt.data.GetByKey("aiSdkName").StringValue()) - assert.Equal(t, Version, evt.data.GetByKey("aiSdkVersion").StringValue()) - assert.Equal(t, "go", evt.data.GetByKey("aiSdkLanguage").StringValue()) - - // Verify the context is anonymous with kind ld_ai. - assert.Equal(t, "ld-internal-tracking", evt.context.Key()) - assert.True(t, evt.context.Anonymous()) - assert.Equal(t, ldcontext.Kind("ld_ai"), evt.context.Kind()) -} - -func TestEvalErrorReturnsDefault(t *testing.T) { - client, err := NewClient(newMockSDK(nil, errors.New("client is offline"))) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - assert.NotNil(t, cfg.CreateTracker()) - assert.Equal(t, defaultVal.Enabled(), cfg.Enabled()) - assert.Equal(t, defaultVal.Messages(), cfg.Messages()) - assert.Equal(t, defaultVal.ModelName(), cfg.ModelName()) - assert.Equal(t, defaultVal.ProviderName(), cfg.ProviderName()) -} - -func TestParseMultipleMessages(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "messages": [ - {"content": "hello", "role": "user"}, - {"content": "world", "role": "system"} - ] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - - assert.ElementsMatch(t, cfg.Messages(), []datamodel.Message{ - {Content: "hello", Role: datamodel.User}, - {Content: "world", Role: datamodel.System}, - }) -} - -func TestParseModelName(t *testing.T) { - tests := []struct { - name string - json []byte - expected string - }{ - {"missing", []byte(`{"model": {}}`), ""}, - {"empty string", []byte(`{"model": {"name": ""}}`), ""}, - {"non-empty string", []byte(`{"model": {"name": "my-model"}}`), "my-model"}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client, err := NewClient(newMockSDK(test.json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - assert.Equal(t, test.expected, cfg.ModelName()) - }) - } -} - -func TestParseProviderName(t *testing.T) { - tests := []struct { - name string - json []byte - expected string - }{ - {"missing", []byte(`{"provider": {}}`), ""}, - {"empty string", []byte(`{"provider": {"name": ""}}`), ""}, - {"non-empty string", []byte(`{"provider": {"name": "my-provider"}}`), "my-provider"}} - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client, err := NewClient(newMockSDK(test.json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - assert.Equal(t, test.expected, cfg.ProviderName()) - }) - } -} - -func TestParseInvalidConfigReturnsDefault(t *testing.T) { - tests := []struct { - name string - json []byte - }{ - {"null value", []byte("null")}, - {"invalid json", []byte("invalid")}, - {"is a number", []byte("42")}, - {"is a string", []byte(`"hello"`)}, - {"is an array", []byte(`["hello"]`)}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - sdk := newMockSDK(test.json, nil) - client, err := NewClient(sdk) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - // Verify config data matches the default - assert.Equal(t, defaultVal.AsLdValue(), cfg.AsLdValue()) - // Verify CreateTracker() now works (returnDefault always injects a factory) - assert.NotNil(t, cfg.CreateTracker()) - - sdk.log.AssertMessageMatch(t, true, ldlog.Warn, "AI Config 'key':") - }) - } -} - -func TestParseDisabledConfigs(t *testing.T) { - tests := []struct { - name string - json []byte - }{ - {"empty object", []byte("{}")}, - {"missing meta field", []byte(`{"model": {}, "messages": []}`)}, - {"meta disabled explicitly", []byte(`{"meta": {"enabled": false, "variationKey": "1"}, "model": {}, "messages": []}`)}, - {"meta disable implicitly", []byte(`{"meta": { "variationKey": "1"}, "model": {}, "messages": []}`)}, - } - - defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client, err := NewClient(newMockSDK(test.json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - // We *shouldn't* be getting the default value, because these are all valid configs that should - // be parsed as disabled. - assert.False(t, cfg.Enabled()) - }) - } -} - -func TestParseModelParams(t *testing.T) { - tests := []struct { - name string - json []byte - expected map[string]ldvalue.Value - }{ - {"omitted", []byte(`{"model": {"name": "model"}}`), nil}, - {"empty", []byte(`{"model": {"name": "model", "parameters": {}}}`), map[string]ldvalue.Value{}}, - {"single", []byte(`{"model": {"name": "model", "parameters": {"foo": "bar"}}}`), - map[string]ldvalue.Value{"foo": ldvalue.String("bar")}}, - {"multiple", []byte(`{"model": {"name": "model", "parameters": {"foo": "bar", "baz": 42}}}`), - map[string]ldvalue.Value{"foo": ldvalue.String("bar"), "baz": ldvalue.Int(42)}}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client, err := NewClient(newMockSDK(test.json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - for k, v := range test.expected { - p, ok := cfg.ModelParam(k) - if assert.True(t, ok) { - assert.Equal(t, v, p) - } - } - }) - } -} - -func TestParseCustomModelParams(t *testing.T) { - tests := []struct { - name string - json []byte - expected map[string]ldvalue.Value - }{ - {"omitted", []byte(`{"model": {"name": "model"}}`), nil}, - {"empty", []byte(`{"model": {"name": "model", "custom": {}}}`), map[string]ldvalue.Value{}}, - {"single", []byte(`{"model": {"name": "model", "custom": {"foo": "bar"}}}`), - map[string]ldvalue.Value{"foo": ldvalue.String("bar")}}, - {"multiple", []byte(`{"model": {"name": "model", "custom": {"foo": "bar", "baz": 42}}}`), - map[string]ldvalue.Value{"foo": ldvalue.String("bar"), "baz": ldvalue.Int(42)}}, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client, err := NewClient(newMockSDK(test.json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - for k, v := range test.expected { - p, ok := cfg.CustomModelParam(k) - if assert.True(t, ok) { - assert.Equal(t, v, p) - } - } - }) - } -} - -func TestCanSetDefaultConfigFields(t *testing.T) { - client, err := NewClient(newMockSDK(nil, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().Enable(). - WithMessage("hello", datamodel.User). - WithMessage("world", datamodel.System). - WithProviderName("provider"). - WithModelName("model").Build() - - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - assert.True(t, cfg.Enabled()) - assert.Equal(t, "provider", cfg.ProviderName()) - assert.Equal(t, "model", cfg.ModelName()) - assert.Equal(t, 2, len(cfg.Messages())) - - msg := cfg.Messages() - assert.Equal(t, "hello", msg[0].Content) - assert.Equal(t, datamodel.User, msg[0].Role) - assert.Equal(t, "world", msg[1].Content) - assert.Equal(t, datamodel.System, msg[1].Role) -} - -func TestCompletionConfigMethodTracking(t *testing.T) { - mockSDK := newMockSDK(nil, nil) - client, err := NewClient(mockSDK) - require.NoError(t, err) - require.NotNil(t, client) - - // Clear the SDK info event from construction. - mockSDK.events = nil - - defaultConfig := NewConfig().WithEnabled(false).Build() - context := ldcontext.New("user-key") - configKey := "test-config-key" - - config := client.CompletionConfig(configKey, context, defaultConfig, nil) - - require.NotNil(t, config.CreateTracker()) - - expectedData := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(configKey)).Build() - expectedEvents := []mockEvent{ - { - eventName: "$ld:ai:usage:completion-config", - context: context, - metricValue: 1, - data: expectedData, - }, - } - - assert.ElementsMatch(t, expectedEvents, mockSDK.events) -} - -// TestJudgeConfigMethodTracking verifies that JudgeConfig emits only the judge metric, -// not the completion-config metric, so judge evaluations are not double-counted on the dashboard. -func TestJudgeConfigMethodTracking(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "mode": "judge", - "evaluationMetricKey": "toxicity", - "messages": [{"content": "test", "role": "system"}] - }`) - mockSDK := newMockSDK(json, nil) - client, err := NewClient(mockSDK) - require.NoError(t, err) - require.NotNil(t, client) - - // Clear the SDK info event from construction. - mockSDK.events = nil - - defaultConfig := Disabled() - context := ldcontext.New("user-key") - configKey := "judge-config-key" - - config := client.JudgeConfig(configKey, context, defaultConfig, nil) - - require.NotNil(t, config.CreateTracker()) - - // Only the judge metric should be emitted; evaluateConfig does not emit any metric. - expectedData := ldvalue.ObjectBuild().Set("configKey", ldvalue.String(configKey)).Build() - expectedEvents := []mockEvent{ - { - eventName: "$ld:ai:usage:judge-config", - context: context, - metricValue: 1, - data: expectedData, - }, - } - assert.ElementsMatch(t, expectedEvents, mockSDK.events, - "JudgeConfig must not emit $ld:ai:usage:completion-config to avoid double-counting") -} - -func TestCanSetModelParameters(t *testing.T) { - client, err := NewClient(newMockSDK(nil, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().WithModelParam("foo", ldvalue.String("bar")).Build() - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - t.Run("param is present", func(t *testing.T) { - p, ok := cfg.ModelParam("foo") - assert.True(t, ok) - assert.Equal(t, "bar", p.StringValue()) - }) - - t.Run("param is missing", func(t *testing.T) { - p, ok := cfg.ModelParam("missing") - assert.False(t, ok) - assert.Equal(t, ldvalue.Null(), p) - }) -} - -func TestCanSetCustomModelParameters(t *testing.T) { - client, err := NewClient(newMockSDK(nil, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig().WithCustomModelParam("foo", ldvalue.String("bar")).Build() - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - t.Run("param is present", func(t *testing.T) { - p, ok := cfg.CustomModelParam("foo") - assert.True(t, ok) - assert.Equal(t, "bar", p.StringValue()) - }) - - t.Run("param is missing", func(t *testing.T) { - p, ok := cfg.CustomModelParam("missing") - assert.False(t, ok) - assert.Equal(t, ldvalue.Null(), p) - }) -} - -func TestNormalAndCustomParamsDoNotInterfere(t *testing.T) { - client, err := NewClient(newMockSDK(nil, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig(). - WithModelParam("foo", ldvalue.String("bar")). - WithCustomModelParam("foo", ldvalue.String("baz")).Build() - - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - foo1, ok := cfg.ModelParam("foo") - require.True(t, ok) - assert.Equal(t, "bar", foo1.StringValue()) - - foo2, ok := cfg.CustomModelParam("foo") - require.True(t, ok) - assert.Equal(t, "baz", foo2.StringValue()) -} - -func TestCannotOverwriteMessages(t *testing.T) { - client, err := NewClient(newMockSDK(nil, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - defaultVal := NewConfig(). - WithMessage("hello", datamodel.Assistant).Build() - - cfg := client.CompletionConfig("key", ldcontext.New("user"), defaultVal, nil) - - cfg.Messages()[0].Content = "changed" - cfg.Messages()[0].Role = datamodel.User - - assert.ElementsMatch(t, []datamodel.Message{{Content: "hello", Role: datamodel.Assistant}}, cfg.Messages()) -} - -func eval(t *testing.T, prompt string, ctx ldcontext.Context, variables map[string]interface{}) (string, error) { - t.Helper() - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "messages": [ - {"content": "` + prompt + `", "role": "user"} - ] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - cfg := client.CompletionConfig("key", ctx, Disabled(), variables) - if len(cfg.Messages()) == 0 { - return "", errors.New("no messages interpolated") - } - return cfg.Messages()[0].Content, nil -} - -func TestInterpolation(t *testing.T) { - t.Run("missing variables", func(t *testing.T) { - cases := []string{ - "{{ adjective }}", - "{{ adjective.nested.deep }}", - "{{ ldctx.this_is_not_a_variable }}", - } - - for _, c := range cases { - t.Run(c, func(t *testing.T) { - result, err := eval(t, "I am an ("+c+") LLM", ldcontext.New("user"), nil) - require.NoError(t, err) - assert.Equal(t, "I am an () LLM", result) - }) - } - }) - - t.Run("simple variables", func(t *testing.T) { - cases := []string{ - "awesome", - "slow", - "all powerful", - } - - for _, c := range cases { - t.Run(c, func(t *testing.T) { - result, err := eval(t, "I am an {{ adjective }} LLM", ldcontext.New("user"), map[string]interface{}{"adjective": c}) - require.NoError(t, err) - assert.Equal(t, "I am an "+c+" LLM", result) - }) - } - }) - - t.Run("multiple variables", func(t *testing.T) { - vars := map[string]interface{}{ - "adjective": "awesome", - "noun": "robot", - "stats": map[string]interface{}{ - "power": "9000", - }, - } - result, err := eval(t, "I am an {{ adjective }} {{ noun }} with power over {{ stats.power }}", ldcontext.New("user"), vars) - require.NoError(t, err) - assert.Equal(t, "I am an awesome robot with power over 9000", result) - }) - - t.Run("interpolation with array indices does not work", func(t *testing.T) { - vars := map[string]interface{}{ - "adjectives": []string{"awesome", "slow", "all powerful"}, - } - - t.Run("dot syntax interpolates as empty string", func(t *testing.T) { - result, err := eval(t, "I am an ({{ adjectives.0 }}) LLM", ldcontext.New("user"), vars) - require.NoError(t, err) - assert.Equal(t, "I am an () LLM", result) - }) - - t.Run("bracket syntax returns error", func(t *testing.T) { - _, err := eval(t, "I am an ({{ adjectives[0] }}) LLM", ldcontext.New("user"), vars) - assert.Error(t, err) - }) - }) - - t.Run("array sections", func(t *testing.T) { - vars := map[string]interface{}{ - "adjectives": []string{"hello", "world", "!"}, - } - - result, err := eval(t, "{{#adjectives }}{{ . }} {{/adjectives }}", ldcontext.New("user"), vars) - require.NoError(t, err) - assert.Equal(t, "hello world ! ", result) - }) - - t.Run("malformed syntax", func(t *testing.T) { - _, err := eval(t, "This is a {{ malformed }]} prompt", ldcontext.New("user"), nil) - require.Error(t, err) - }) - - t.Run("interpolate single kind context", func(t *testing.T) { - context := ldcontext.NewBuilder("123").Name("Sandy").Build() - result, err := eval(t, "I'm a {{ ldctx.kind}} with key {{ ldctx.key }}, named {{ ldctx.name }}", context, nil) - require.NoError(t, err) - assert.Equal(t, "I'm a user with key 123, named Sandy", result) - }) - - t.Run("interpolation with nested context attributes", func(t *testing.T) { - context := ldcontext.NewBuilder("123"). - SetValue("stats", ldvalue.ObjectBuild().Set("power", ldvalue.Int(9000)).Build()).Build() - result, err := eval(t, "I can ingest over {{ ldctx.stats.power }} tokens per second!", context, nil) - require.NoError(t, err) - assert.Equal(t, "I can ingest over 9000 tokens per second!", result) - }) - - t.Run("interpolation with multi kind context", func(t *testing.T) { - user := ldcontext.NewBuilder("123"). - SetValue("cat_ownership", ldvalue.ObjectBuild().Set("count", ldvalue.Int(12)).Build()).Build() - - cat := ldcontext.NewBuilder("456").Kind("cat"). - SetValue("health", ldvalue.ObjectBuild().Set("hunger", ldvalue.String("off the charts")).Build()).Build() - - context := ldcontext.NewMulti(user, cat) - - result, err := eval(t, "As an owner of {{ ldctx.user.cat_ownership.count }} cats, I must report that my cat's hunger level is {{ ldctx.cat.health.hunger }}!", context, nil) - require.NoError(t, err) - assert.Equal(t, "As an owner of 12 cats, I must report that my cat's hunger level is off the charts!", result) - }) - - t.Run("interpolation with multi kind context does not have anonymous attribute", func(t *testing.T) { - user := ldcontext.NewBuilder("123"). - SetValue("cat_ownership", ldvalue.ObjectBuild().Set("count", ldvalue.Int(12)).Build()).Build() - - cat := ldcontext.NewBuilder("456").Kind("cat"). - SetValue("health", ldvalue.ObjectBuild().Set("hunger", ldvalue.String("off the charts")).Build()).Build() - - context := ldcontext.NewMulti(user, cat) - - result, err := eval(t, "anonymous=<{{ ldctx.anonymous }}>", context, nil) - require.NoError(t, err) - assert.Equal(t, "anonymous=<>", result) - }) - - t.Run("interpolation with multi kind context has kind multi", func(t *testing.T) { - user := ldcontext.NewBuilder("123"). - SetValue("cat_ownership", ldvalue.ObjectBuild().Set("count", ldvalue.Int(12)).Build()).Build() - - cat := ldcontext.NewBuilder("456").Kind("cat"). - SetValue("health", ldvalue.ObjectBuild().Set("hunger", ldvalue.String("off the charts")).Build()).Build() - - context := ldcontext.NewMulti(user, cat) - - result, err := eval(t, "kind=<{{ ldctx.kind }}>", context, nil) - require.NoError(t, err) - assert.Equal(t, "kind=", result) - }) - - t.Run("interpolation with multi kind context does not have child kinds", func(t *testing.T) { - - // The idea here is that in a multi-kind context, we can access ldctx.kind (== "multi"), but you can't - // access the kind field of the individual nested contexts since this doesn't match the actual data model. - // That is, you can't access ldctx.user.kind or ldctx.cat.kind, only ldctx.kind. - - user := ldcontext.NewBuilder("123"). - SetValue("cat_ownership", ldvalue.ObjectBuild().Set("count", ldvalue.Int(12)).Build()).Build() - - cat := ldcontext.NewBuilder("456").Kind("cat"). - SetValue("health", ldvalue.ObjectBuild().Set("hunger", ldvalue.String("off the charts")).Build()).Build() - - context := ldcontext.NewMulti(user, cat) - - result, err := eval(t, "user_kind=<{{ ldctx.user.kind}}>,cat_kind=<{{ ldctx.cat.kind }}>", context, nil) - require.NoError(t, err) - assert.Equal(t, "user_kind=<>,cat_kind=<>", result) - }) -} - -func TestParseJudgeSpecificFields(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "mode": "judge", - "evaluationMetricKey": "toxicity", - "judgeConfiguration": { - "judges": [ - {"key": "judge1", "samplingRate": 0.5}, - {"key": "judge2", "samplingRate": 1.0} - ] - }, - "messages": [ - {"content": "test", "role": "system"} - ] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - - assert.Equal(t, "judge", cfg.Mode()) - assert.Equal(t, "toxicity", cfg.EvaluationMetricKey()) - - judgeConfig := cfg.JudgeConfiguration() - require.NotNil(t, judgeConfig) - require.Len(t, judgeConfig.Judges, 2) - assert.Equal(t, "judge1", judgeConfig.Judges[0].Key) - assert.Equal(t, 0.5, judgeConfig.Judges[0].SamplingRate) - assert.Equal(t, "judge2", judgeConfig.Judges[1].Key) - assert.Equal(t, 1.0, judgeConfig.Judges[1].SamplingRate) -} - -func TestParseEvaluationMetricKeys(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "mode": "judge", - "evaluationMetricKeys": ["relevance", "accuracy"], - "messages": [ - {"content": "test", "role": "system"} - ] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - - assert.Equal(t, "judge", cfg.Mode()) - assert.Equal(t, "", cfg.EvaluationMetricKey()) - assert.Equal(t, []string{"relevance", "accuracy"}, cfg.EvaluationMetricKeys()) -} - -func TestParseEvaluationMetricKeyPriority(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "mode": "judge", - "evaluationMetricKey": "toxicity", - "evaluationMetricKeys": ["relevance", "accuracy"], - "messages": [ - {"content": "test", "role": "system"} - ] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - - assert.Equal(t, "judge", cfg.Mode()) - // Both fields should be parsed - assert.Equal(t, "toxicity", cfg.EvaluationMetricKey()) - assert.Equal(t, []string{"relevance", "accuracy"}, cfg.EvaluationMetricKeys()) -} - -func TestJudgeConfigurationImmutable(t *testing.T) { - // Test that mutations to JudgeConfiguration don't affect the Config - judgeConfig := &datamodel.JudgeConfiguration{ - Judges: []datamodel.Judge{ - {Key: "judge1", SamplingRate: 0.5}, - {Key: "judge2", SamplingRate: 1.0}, - }, - } - - builder := NewConfig(). - Enable(). - WithJudgeConfiguration(judgeConfig) - cfg := builder.Build() - - // Mutate the original - judgeConfig.Judges[0].Key = "mutated" - judgeConfig.Judges = append(judgeConfig.Judges, datamodel.Judge{Key: "judge3", SamplingRate: 0.3}) - - // Config should not be affected - retrieved := cfg.JudgeConfiguration() - require.NotNil(t, retrieved) - require.Len(t, retrieved.Judges, 2) - assert.Equal(t, "judge1", retrieved.Judges[0].Key) // Should still be original value - assert.Equal(t, "judge2", retrieved.Judges[1].Key) - - // Mutate the retrieved config - retrieved.Judges[0].Key = "mutated_again" - retrieved.Judges = append(retrieved.Judges, datamodel.Judge{Key: "judge4", SamplingRate: 0.4}) - - // Config should still not be affected - retrieved2 := cfg.JudgeConfiguration() - require.NotNil(t, retrieved2) - require.Len(t, retrieved2.Judges, 2) - assert.Equal(t, "judge1", retrieved2.Judges[0].Key) // Should still be original value - assert.Equal(t, "judge2", retrieved2.Judges[1].Key) -} - -// TestJudgeConfig_PreservesReservedPlaceholders verifies that JudgeConfig injects reserved variables -// so that {{message_history}} and {{response_to_evaluate}} are preserved for the second interpolation -// pass during Judge.Evaluate(). Without this, Config's first Mustache pass would render them as empty. -func TestJudgeConfig_PreservesReservedPlaceholders(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "mode": "judge", - "evaluationMetricKey": "toxicity", - "messages": [ - {"content": "You are a judge.", "role": "system"}, - {"content": "Input: {{message_history}}\nOutput: {{response_to_evaluate}}", "role": "user"} - ] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - cfg := client.JudgeConfig("judge-key", ldcontext.New("user"), Disabled(), nil) - - msgs := cfg.Messages() - require.Len(t, msgs, 2) - assert.Equal(t, "You are a judge.", msgs[0].Content) - assert.Contains(t, msgs[1].Content, "{{message_history}}", "JudgeConfig must preserve placeholder for second interpolation") - assert.Contains(t, msgs[1].Content, "{{response_to_evaluate}}", "JudgeConfig must preserve placeholder for second interpolation") - assert.Equal(t, "Input: {{message_history}}\nOutput: {{response_to_evaluate}}", msgs[1].Content) -} - -// TestConfig_WithoutReservedVarsWipesJudgePlaceholders documents that Config (without reserved vars) -// renders {{message_history}} and {{response_to_evaluate}} as empty when used for judge templates. -func TestConfig_WithoutReservedVarsWipesJudgePlaceholders(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "messages": [ - {"content": "Input: {{message_history}}\nOutput: {{response_to_evaluate}}", "role": "user"} - ] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - require.NotNil(t, client) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - - msgs := cfg.Messages() - require.Len(t, msgs, 1) - assert.Equal(t, "Input: \nOutput: ", msgs[0].Content, "Config without reserved vars renders placeholders as empty") -} - -func TestCreateTracker_ManuallyBuiltConfig_ReturnsNil(t *testing.T) { - cfg := NewConfig().Enable().WithMessage("hello", datamodel.User).Build() - assert.Nil(t, cfg.CreateTracker(), "manually built config should not have a tracker factory") -} - -func TestCreateTracker_DisabledConfig_ReturnsTracker(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": false}, - "messages": [{"content": "hello", "role": "user"}] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - assert.False(t, cfg.Enabled()) - assert.NotNil(t, cfg.CreateTracker(), "disabled config should still have a tracker factory") -} - -func TestCreateTracker_EnabledConfig_ReturnsTracker(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "model": {"name": "gpt-4"}, - "provider": {"name": "openai"}, - "messages": [{"content": "hello", "role": "user"}] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - assert.True(t, cfg.Enabled()) - - tracker := cfg.CreateTracker() - require.NotNil(t, tracker, "enabled config should have a tracker factory") -} - -func TestCreateTracker_FreshRunIdPerCall(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "messages": [{"content": "hello", "role": "user"}] - }`) - - mockSDK := newMockSDK(json, nil) - client, err := NewClient(mockSDK) - require.NoError(t, err) - - // Clear SDK info event - mockSDK.events = nil - - cfg := client.CompletionConfig("key", ldcontext.New("user"), Disabled(), nil) - - tracker1 := cfg.CreateTracker() - tracker2 := cfg.CreateTracker() - require.NotNil(t, tracker1) - require.NotNil(t, tracker2) - - // Each tracker should be able to track independently. Track success on both to emit events. - _ = tracker1.TrackSuccess() - _ = tracker2.TrackSuccess() - - // Filter out the usage event; we only want the generation events. - var genEvents []mockEvent - for _, e := range mockSDK.events { - if e.eventName == "$ld:ai:generation:success" { - genEvents = append(genEvents, e) - } - } - - require.Len(t, genEvents, 2, "each tracker should emit its own event") - - runId1 := genEvents[0].data.GetByKey("runId").StringValue() - runId2 := genEvents[1].data.GetByKey("runId").StringValue() - assert.NotEmpty(t, runId1) - assert.NotEmpty(t, runId2) - assert.NotEqual(t, runId1, runId2, "each tracker must have a unique runId") -} - -func TestCreateTracker_TrackerHasCorrectMetadata(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "var-1", "enabled": true, "version": 5}, - "model": {"name": "gpt-4"}, - "provider": {"name": "openai"}, - "messages": [{"content": "hello", "role": "user"}] - }`) - - mockSDK := newMockSDK(json, nil) - client, err := NewClient(mockSDK) - require.NoError(t, err) - - // Clear SDK info event - mockSDK.events = nil - - cfg := client.CompletionConfig("my-config", ldcontext.New("user"), Disabled(), nil) - - tracker := cfg.CreateTracker() - require.NotNil(t, tracker) - - _ = tracker.TrackSuccess() - - // Filter for the generation event (skip usage event) - var genEvent *mockEvent - for i, e := range mockSDK.events { - if e.eventName == "$ld:ai:generation:success" { - genEvent = &mockSDK.events[i] - break - } - } - require.NotNil(t, genEvent) - - data := genEvent.data - assert.Equal(t, "my-config", data.GetByKey("configKey").StringValue()) - assert.Equal(t, "var-1", data.GetByKey("variationKey").StringValue()) - assert.Equal(t, 5, data.GetByKey("version").IntValue()) - assert.Equal(t, "openai", data.GetByKey("providerName").StringValue()) - assert.Equal(t, "gpt-4", data.GetByKey("modelName").StringValue()) - assert.NotEmpty(t, data.GetByKey("runId").StringValue()) -} - -func TestCreateTracker_JudgeConfigHasFactory(t *testing.T) { - json := []byte(`{ - "_ldMeta": {"variationKey": "1", "enabled": true}, - "mode": "judge", - "evaluationMetricKey": "toxicity", - "messages": [{"content": "test", "role": "system"}] - }`) - - client, err := NewClient(newMockSDK(json, nil)) - require.NoError(t, err) - - cfg := client.JudgeConfig("judge-key", ldcontext.New("user"), Disabled(), nil) - assert.True(t, cfg.Enabled()) - - tracker := cfg.CreateTracker() - require.NotNil(t, tracker, "enabled judge config should have a tracker factory") -} - -func TestClient_CreateTracker_RoundTrip(t *testing.T) { - configJSON := []byte(`{ - "_ldMeta": {"variationKey": "var-1", "enabled": true, "version": 5}, - "model": {"name": "gpt-4"}, - "provider": {"name": "openai"}, - "messages": [{"content": "hello", "role": "user"}] - }`) - - mockSDK := newMockSDK(configJSON, nil) - client, err := NewClient(mockSDK) - require.NoError(t, err) - - // Clear SDK info event - mockSDK.events = nil - - cfg := client.CompletionConfig("my-config", ldcontext.New("user"), Disabled(), nil) - originalTracker := cfg.CreateTracker() - require.NotNil(t, originalTracker) - - token := originalTracker.ResumptionToken() - require.NotEmpty(t, token) - - // Reconstruct from token with a different context - newContext := ldcontext.New("other-user") - reconstructed, err := client.CreateTracker(token, newContext) - require.NoError(t, err) - require.NotNil(t, reconstructed) - - // The reconstructed tracker should produce the same resumption token - assert.Equal(t, token, reconstructed.ResumptionToken()) - - // Track feedback on the reconstructed tracker and verify it uses the original runId - _ = originalTracker.TrackSuccess() - _ = reconstructed.TrackFeedback(FeedbackPositive) - - var successEvent, feedbackEvent *mockEvent - for i, e := range mockSDK.events { - switch e.eventName { - case "$ld:ai:generation:success": - successEvent = &mockSDK.events[i] - case "$ld:ai:feedback:user:positive": - feedbackEvent = &mockSDK.events[i] - } - } - require.NotNil(t, successEvent) - require.NotNil(t, feedbackEvent) - - // Both events should share the same runId - originalRunId := successEvent.data.GetByKey("runId").StringValue() - reconstructedRunId := feedbackEvent.data.GetByKey("runId").StringValue() - assert.Equal(t, originalRunId, reconstructedRunId, "reconstructed tracker must reuse the original runId") - - // Reconstructed tracker should use the new context - assert.Equal(t, newContext, feedbackEvent.context) - - // Verify metadata preserved - assert.Equal(t, "my-config", feedbackEvent.data.GetByKey("configKey").StringValue()) - assert.Equal(t, "var-1", feedbackEvent.data.GetByKey("variationKey").StringValue()) - assert.Equal(t, 5, feedbackEvent.data.GetByKey("version").IntValue()) - - // modelName and providerName should be empty on reconstructed tracker - assert.Equal(t, "", feedbackEvent.data.GetByKey("modelName").StringValue()) - assert.Equal(t, "", feedbackEvent.data.GetByKey("providerName").StringValue()) -} - -func TestClient_CreateTracker_InvalidToken(t *testing.T) { - mockSDK := newMockSDK(nil, nil) - client, err := NewClient(mockSDK) - require.NoError(t, err) - - t.Run("invalid base64", func(t *testing.T) { - _, err := client.CreateTracker("not-valid-base64!!!", ldcontext.New("user")) - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid resumption token") - }) - - t.Run("valid base64 but invalid JSON", func(t *testing.T) { - token := base64.RawURLEncoding.EncodeToString([]byte("not json")) - _, err := client.CreateTracker(token, ldcontext.New("user")) - assert.Error(t, err) - assert.Contains(t, err.Error(), "invalid resumption token") - }) - - t.Run("valid token with missing fields uses zero values", func(t *testing.T) { - payload, _ := json.Marshal(map[string]interface{}{"runId": "test-run"}) - token := base64.RawURLEncoding.EncodeToString(payload) - tracker, err := client.CreateTracker(token, ldcontext.New("user")) - require.NoError(t, err) - require.NotNil(t, tracker) - - // Should work with partial data - resumeToken := tracker.ResumptionToken() - assert.NotEmpty(t, resumeToken) - }) -} diff --git a/ldai/config.go b/ldai/config.go deleted file mode 100644 index 8fa58bfd..00000000 --- a/ldai/config.go +++ /dev/null @@ -1,250 +0,0 @@ -package ldai - -import ( - "maps" - - "slices" - - "github.com/launchdarkly/go-sdk-common/v3/ldvalue" - "github.com/launchdarkly/go-server-sdk/ldai/datamodel" -) - -// Config represents an AI Config. -type Config struct { - c datamodel.Config - trackerFactory func() *Tracker -} - -// VariationKey is used internally by LaunchDarkly. -func (c *Config) VariationKey() string { - return c.c.Meta.VariationKey -} - -// Version is used internally by LaunchDarkly. -func (c *Config) Version() int { - if c.c.Meta.Version == nil { - return 1 - } - return *c.c.Meta.Version -} - -// Messages returns the messages defined by the config. The series of messages may be -// passed to an AI model provider. -func (c *Config) Messages() []datamodel.Message { - return slices.Clone(c.c.Messages) -} - -// Enabled returns whether the config is enabled. -func (c *Config) Enabled() bool { - return c.c.Meta.Enabled -} - -// ProviderName returns the provider name associated with the config. -func (c *Config) ProviderName() string { - return c.c.Provider.Name -} - -// ModelName returns the model name associated with the config. -func (c *Config) ModelName() string { - return c.c.Model.Name -} - -// ModelParam returns the model parameter named by key. The second parameter is true if the key exists. -func (c *Config) ModelParam(key string) (ldvalue.Value, bool) { - val, ok := c.c.Model.Parameters[key] - return val, ok -} - -// CustomModelParam returns the custom model parameter named by key. The second parameter is true if the key exists. -func (c *Config) CustomModelParam(key string) (ldvalue.Value, bool) { - val, ok := c.c.Model.Custom[key] - return val, ok -} - -// Mode returns the AI Config mode (e.g., "completion", "agent", "judge"). -func (c *Config) Mode() string { - return c.c.Mode -} - -// EvaluationMetricKey returns the evaluation metric key for judge mode configs. -func (c *Config) EvaluationMetricKey() string { - return c.c.EvaluationMetricKey -} - -// EvaluationMetricKeys returns the deprecated array of evaluation metric keys. -// Use EvaluationMetricKey instead. -func (c *Config) EvaluationMetricKeys() []string { - return slices.Clone(c.c.EvaluationMetricKeys) -} - -// JudgeConfiguration returns the judge configuration attached to this config, if any. -// Returns a defensive copy to prevent mutations. -func (c *Config) JudgeConfiguration() *datamodel.JudgeConfiguration { - if c.c.JudgeConfiguration == nil { - return nil - } - return &datamodel.JudgeConfiguration{ - Judges: slices.Clone(c.c.JudgeConfiguration.Judges), - } -} - -// CreateTracker returns a new Tracker for a fresh AI run. Each call mints -// a new runId (a UUIDv4) that LaunchDarkly uses to correlate the run's -// events in metrics views. Call this once per AI run; metrics from -// different runIds cannot be combined. -// -// Returns nil if the config was not obtained via the Client. -func (c *Config) CreateTracker() *Tracker { - if c.trackerFactory == nil { - return nil - } - return c.trackerFactory() -} - -// AsLdValue is used internally. -func (c *Config) AsLdValue() ldvalue.Value { - return ldvalue.FromJSONMarshal(c.c) -} - -// ConfigBuilder is used to define a default AI Config, returned when LaunchDarkly is unreachable or there -// is an error evaluating the Config. -type ConfigBuilder struct { - messages []datamodel.Message - enabled bool - providerName string - modelName string - modelParams map[string]ldvalue.Value - modelCustomParams map[string]ldvalue.Value - mode string - evaluationMetricKey string - evaluationMetricKeys []string - judgeConfiguration *datamodel.JudgeConfiguration -} - -// NewConfig returns a new ConfigBuilder. By default, the Config is disabled. -func NewConfig() *ConfigBuilder { - return &ConfigBuilder{ - modelParams: make(map[string]ldvalue.Value), - modelCustomParams: make(map[string]ldvalue.Value), - } -} - -// Disabled is a helper that returns a built Config that is disabled and contains no messages. -func Disabled() Config { - return NewConfig().Disable().Build() -} - -// WithMessage appends a message to the config with the given role. -func (cb *ConfigBuilder) WithMessage(content string, role datamodel.Role) *ConfigBuilder { - cb.messages = append(cb.messages, datamodel.Message{ - Content: content, - Role: role, - }) - return cb -} - -// WithEnabled sets whether the config is enabled. See also Enable and Disable. -func (cb *ConfigBuilder) WithEnabled(enabled bool) *ConfigBuilder { - cb.enabled = enabled - return cb -} - -// Enable enables the config. -func (cb *ConfigBuilder) Enable() *ConfigBuilder { - return cb.WithEnabled(true) -} - -// Disable disables the config. -func (cb *ConfigBuilder) Disable() *ConfigBuilder { - return cb.WithEnabled(false) -} - -// WithModelName sets the model name associated with the config. -func (cb *ConfigBuilder) WithModelName(modelName string) *ConfigBuilder { - cb.modelName = modelName - return cb -} - -// WithProviderName sets the provider name associated with the config. -func (cb *ConfigBuilder) WithProviderName(providerName string) *ConfigBuilder { - cb.providerName = providerName - return cb -} - -// WithModelParam sets a model parameter named by key to the given value. If the key already exists, it will be -// overwritten. Model parameters are generally set by LaunchDarkly; for custom parameters not recognized by -// LaunchDarkly, use WithModelCustomParam. -func (cb *ConfigBuilder) WithModelParam(key string, value ldvalue.Value) *ConfigBuilder { - cb.modelParams[key] = value - return cb -} - -// WithCustomModelParam sets a custom model parameter named by key to the given value. If the key already exists, it -// will be overwritten. -func (cb *ConfigBuilder) WithCustomModelParam(key string, value ldvalue.Value) *ConfigBuilder { - cb.modelCustomParams[key] = value - return cb -} - -// WithMode sets the AI Config mode (e.g., "completion", "agent", "judge"). -func (cb *ConfigBuilder) WithMode(mode string) *ConfigBuilder { - cb.mode = mode - return cb -} - -// WithEvaluationMetricKey sets the evaluation metric key for judge mode configs. -func (cb *ConfigBuilder) WithEvaluationMetricKey(key string) *ConfigBuilder { - cb.evaluationMetricKey = key - return cb -} - -// WithEvaluationMetricKeys sets the deprecated array of evaluation metric keys. -// Use WithEvaluationMetricKey instead. -func (cb *ConfigBuilder) WithEvaluationMetricKeys(keys []string) *ConfigBuilder { - cb.evaluationMetricKeys = slices.Clone(keys) - return cb -} - -// WithJudgeConfiguration sets the judge configuration for this config. -// The provided judgeConfig is defensively copied. -func (cb *ConfigBuilder) WithJudgeConfiguration(judgeConfig *datamodel.JudgeConfiguration) *ConfigBuilder { - if judgeConfig == nil { - cb.judgeConfiguration = nil - return cb - } - cb.judgeConfiguration = &datamodel.JudgeConfiguration{ - Judges: slices.Clone(judgeConfig.Judges), - } - return cb -} - -// Build creates a Config from the current builder state. -func (cb *ConfigBuilder) Build() Config { - var judgeConfig *datamodel.JudgeConfiguration - if cb.judgeConfiguration != nil { - judgeConfig = &datamodel.JudgeConfiguration{ - Judges: slices.Clone(cb.judgeConfiguration.Judges), - } - } - - return Config{ - c: datamodel.Config{ - Messages: slices.Clone(cb.messages), - Meta: datamodel.Meta{ - Enabled: cb.enabled, - }, - Model: datamodel.Model{ - Name: cb.modelName, - Parameters: maps.Clone(cb.modelParams), - Custom: maps.Clone(cb.modelCustomParams), - }, - Provider: datamodel.Provider{ - Name: cb.providerName, - }, - Mode: cb.mode, - EvaluationMetricKey: cb.evaluationMetricKey, - EvaluationMetricKeys: slices.Clone(cb.evaluationMetricKeys), - JudgeConfiguration: judgeConfig, - }, - } -} diff --git a/ldai/datamodel/datamodel.go b/ldai/datamodel/datamodel.go deleted file mode 100644 index b5de1961..00000000 --- a/ldai/datamodel/datamodel.go +++ /dev/null @@ -1,123 +0,0 @@ -package datamodel - -import "github.com/launchdarkly/go-sdk-common/v3/ldvalue" - -// Meta defines the serialization format for config metadata. -type Meta struct { - // VariationKey is the variation key. - VariationKey string `json:"variationKey,omitempty"` - - // Enabled is true if the config is enabled. - Enabled bool `json:"enabled,omitempty"` - - // Version is the version of the Variation. - Version *int `json:"version,omitempty"` -} - -// Model defines the serialization format for a model. -type Model struct { - // Name identifies the model. - Name string `json:"name"` - - // Parameters are the model parameters, generally provided by LaunchDarkly. - Parameters map[string]ldvalue.Value `json:"parameters,omitempty"` - - // Custom are custom model parameters, generally provided by the user. - Custom map[string]ldvalue.Value `json:"custom,omitempty"` -} - -// Provider defines the serialization format for a model provider. -type Provider struct { - // Name identifies the provider. - Name string `json:"name"` -} - -// Role defines the role of a message. -type Role string - -const ( - // User represents the user. - User Role = "user" - - // System represents the system. - System Role = "system" - - // Assistant represents an assistant. - Assistant Role = "assistant" -) - -// Message defines the serialization format for a message which may be passed to an AI model provider. -type Message struct { - // Content is the message content. - Content string `json:"content"` - - // Role is the role of the message. - Role Role `json:"role"` -} - -// Config defines the serialization format for an AI Config. -type Config struct { - // Messages is a list of messages. The messages received from LaunchDarkly are uninterpolated. - Messages []Message `json:"messages,omitempty"` - - // Meta is the config metadata. - Meta Meta `json:"_ldMeta,omitempty"` - - // Model is the model. - Model Model `json:"model,omitempty"` - - // Provider is the provider. - Provider Provider `json:"provider,omitempty"` - - // Mode is the AI Config mode (e.g., "completion", "agent", "judge"). - Mode string `json:"mode,omitempty"` - - // EvaluationMetricKey is the evaluation metric key for judge mode configs. - EvaluationMetricKey string `json:"evaluationMetricKey,omitempty"` - - // EvaluationMetricKeys is a deprecated array of evaluation metric keys. - // Use EvaluationMetricKey instead. - EvaluationMetricKeys []string `json:"evaluationMetricKeys,omitempty"` - - // JudgeConfiguration specifies judges attached to this config. - JudgeConfiguration *JudgeConfiguration `json:"judgeConfiguration,omitempty"` -} - -// JudgeConfiguration defines the configuration for judges attached to a config. -type JudgeConfiguration struct { - // Judges is a list of judges to evaluate this config's outputs. - Judges []Judge `json:"judges,omitempty"` -} - -// Judge defines a single judge reference with key and sampling rate. -type Judge struct { - // Key is the judge config key. - Key string `json:"key"` - - // SamplingRate is the probability (0.0-1.0) that the judge will evaluate. - SamplingRate float64 `json:"samplingRate"` -} - -// EvalScore represents a single evaluation metric result. -type EvalScore struct { - // Score is the evaluation score between 0.0 and 1.0. - Score float64 `json:"score"` - - // Reasoning is the explanation for the score. - Reasoning string `json:"reasoning"` -} - -// JudgeResponse represents the response from a judge evaluation. -type JudgeResponse struct { - // Evals contains the evaluation results keyed by metric name. - Evals map[string]EvalScore `json:"evals"` - - // Success indicates whether the evaluation completed successfully. - Success bool `json:"success"` - - // JudgeConfigKey is the key of the judge config that produced this response. - JudgeConfigKey string `json:"judgeConfigKey,omitempty"` - - // Error contains the error message if the evaluation failed. - Error string `json:"error,omitempty"` -} diff --git a/ldai/go.mod b/ldai/go.mod deleted file mode 100644 index 80fd5fb8..00000000 --- a/ldai/go.mod +++ /dev/null @@ -1,24 +0,0 @@ -// Deprecated: this module has moved to github.com/launchdarkly/go-server-sdk-ai. -module github.com/launchdarkly/go-server-sdk/ldai - -go 1.24.0 - -require ( - github.com/alexkappa/mustache v1.0.0 - github.com/google/uuid v1.1.1 - github.com/launchdarkly/go-sdk-common/v3 v3.5.0 - github.com/launchdarkly/go-server-sdk/v7 v7.15.3 - github.com/stretchr/testify v1.9.0 -) - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/launchdarkly/go-jsonstream/v3 v3.1.1 // indirect - github.com/launchdarkly/go-sdk-events/v3 v3.6.2 // indirect - github.com/mailru/easyjson v0.7.7 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) - -retract v0.9.1 // Introduced unintentional breaking changes; use version v0.9.2 or later. diff --git a/ldai/go.sum b/ldai/go.sum deleted file mode 100644 index 810553f7..00000000 --- a/ldai/go.sum +++ /dev/null @@ -1,35 +0,0 @@ -github.com/alexkappa/mustache v1.0.0 h1:GeF7AKKpKKVq8emIwYRQPsKPDWSGYQGWWSsddge62M4= -github.com/alexkappa/mustache v1.0.0/go.mod h1:6v0WNoCZEQ8K5OZAv82ScIARg2bDqFD+Jl0LWxnApas= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/launchdarkly/go-jsonstream/v3 v3.1.1 h1:ugupp2eNtwVbr69KCdeUrm1vUf1/3ju4Wdliaob95uY= -github.com/launchdarkly/go-jsonstream/v3 v3.1.1/go.mod h1:ZBjhKq8mhArCtqotGRGnteY6eXpNm1GaOdUSZHh+ZjM= -github.com/launchdarkly/go-sdk-common/v3 v3.5.0 h1:DsfTimg4BZO2hQPeeeEZYZVsP25pAWEdq/rKv/b7HJU= -github.com/launchdarkly/go-sdk-common/v3 v3.5.0/go.mod h1:6MNeeP8b2VtsM6I3TbShCHW/+tYh2c+p5dB+ilS69sg= -github.com/launchdarkly/go-sdk-events/v3 v3.6.2 h1:X5sxi1Hz5Cj8wOyXiNnJzCcCGiGIy9IOSEk31lCfDXM= -github.com/launchdarkly/go-sdk-events/v3 v3.6.2/go.mod h1:s86HRgbVAHjoQjQEOzgOB+ZKWlU4wwHAeDa/5N6MwO0= -github.com/launchdarkly/go-server-sdk/v7 v7.15.3 h1:t+UaW8TjGwLJTBLfsPrCLLt3KSifECH8Othfq+wRz/w= -github.com/launchdarkly/go-server-sdk/v7 v7.15.3/go.mod h1:oRUugzU7uSGm2vg7iLdJg2R/aZbqgrt07layMPLyVsw= -github.com/launchdarkly/go-test-helpers/v3 v3.1.0 h1:E3bxJMzMoA+cJSF3xxtk2/chr1zshl1ZWa0/oR+8bvg= -github.com/launchdarkly/go-test-helpers/v3 v3.1.0/go.mod h1:Ake5+hZFS/DmIGKx/cizhn5W9pGA7pplcR7xCxWiLIo= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ldai/judge/judge.go b/ldai/judge/judge.go deleted file mode 100644 index 1cad7d5d..00000000 --- a/ldai/judge/judge.go +++ /dev/null @@ -1,217 +0,0 @@ -package judge - -import ( - "fmt" - "math/rand" - "strings" - - "github.com/launchdarkly/go-sdk-common/v3/ldvalue" - "github.com/launchdarkly/go-server-sdk/ldai" - "github.com/launchdarkly/go-server-sdk/ldai/datamodel" - "github.com/launchdarkly/go-server-sdk/v7/interfaces" -) - -type Config interface { - Messages() []datamodel.Message - ModelParam(key string) (ldvalue.Value, bool) - CustomModelParam(key string) (ldvalue.Value, bool) - EvaluationMetricKey() string - EvaluationMetricKeys() []string -} - -type Tracker interface { - TrackJudgeResponse(response datamodel.JudgeResponse) error - TrackTokens(usage ldai.TokenUsage) error -} - -type StructuredResponse struct { - Content map[string]interface{} - Usage ldai.TokenUsage -} - -type Provider interface { - InvokeStructuredModel(messages []datamodel.Message, schema map[string]interface{}) (StructuredResponse, error) -} - -type Judge struct { - config Config - tracker Tracker - provider Provider - metricKey string - judgeConfigKey string - logger interfaces.LDLoggers -} - -func New(config Config, tracker Tracker, provider Provider, configKey string, logger interfaces.LDLoggers) (*Judge, error) { - if config == nil { - return nil, fmt.Errorf("config must not be nil") - } - if tracker == nil { - return nil, fmt.Errorf("tracker must not be nil") - } - if provider == nil { - return nil, fmt.Errorf("provider must not be nil") - } - - metricKey, err := getMetricKey(config, logger, configKey) - if err != nil { - return nil, err - } - - return &Judge{ - config: config, - tracker: tracker, - provider: provider, - metricKey: metricKey, - judgeConfigKey: configKey, - logger: logger, - }, nil -} - -func (j *Judge) Evaluate(input, output string, samplingRate float64) (*datamodel.JudgeResponse, error) { - if len(j.config.Messages()) == 0 { - return nil, nil - } - - if samplingRate < 1.0 && rand.Float64() > samplingRate { - return nil, nil - } - - messages := j.buildMessages(input, output) - schema := buildSchema(j.metricKey) - - response, err := j.provider.InvokeStructuredModel(messages, schema) - if err != nil { - return &datamodel.JudgeResponse{ - Evals: map[string]datamodel.EvalScore{}, - Success: false, - Error: err.Error(), - JudgeConfigKey: j.judgeConfigKey, - }, nil - } - - if response.Usage.Total > 0 || response.Usage.Input > 0 || response.Usage.Output > 0 { - _ = j.tracker.TrackTokens(response.Usage) - } - - result := j.parseResponse(response.Content) - // Note: Judge response tracking should be done by the caller (AI config being evaluated) - // not by the judge itself. This matches Python and JavaScript SDK behavior. - - return result, nil -} - -func (j *Judge) EvaluateMessages(messages []datamodel.Message, response string, samplingRate float64) (*datamodel.JudgeResponse, error) { - parts := make([]string, len(messages)) - for i, msg := range messages { - parts[i] = msg.Content - } - input := strings.Join(parts, "\r\n") - return j.Evaluate(input, response, samplingRate) -} - -func (j *Judge) GetConfig() Config { - return j.config -} - -func (j *Judge) GetTracker() Tracker { - return j.tracker -} - -func (j *Judge) GetProvider() Provider { - return j.provider -} - -func (j *Judge) buildMessages(input, output string) []datamodel.Message { - // Use string replacement to prevent context attributes like {{=[ ]=}}) from - // influencing judge template parsing. - replacer := strings.NewReplacer( - ldai.JudgePlaceholderMessageHistory, input, - ldai.JudgePlaceholderResponseToEvaluate, output, - ) - - messages := j.config.Messages() - result := make([]datamodel.Message, len(messages)) - - for i, msg := range messages { - result[i] = datamodel.Message{Content: replacer.Replace(msg.Content), Role: msg.Role} - } - - return result -} - -func (j *Judge) parseResponse(data map[string]interface{}) *datamodel.JudgeResponse { - evaluations, ok := data["evaluations"].(map[string]interface{}) - if !ok { - return &datamodel.JudgeResponse{ - Evals: map[string]datamodel.EvalScore{}, - Success: false, - Error: "missing evaluations object", - JudgeConfigKey: j.judgeConfigKey, - } - } - - evalData, ok := evaluations[j.metricKey].(map[string]interface{}) - if !ok { - return &datamodel.JudgeResponse{ - Evals: map[string]datamodel.EvalScore{}, - Success: false, - Error: fmt.Sprintf("missing evaluation for %s", j.metricKey), - JudgeConfigKey: j.judgeConfigKey, - } - } - - score, ok := evalData["score"].(float64) - if !ok || score < 0 || score > 1 { - return &datamodel.JudgeResponse{ - Evals: map[string]datamodel.EvalScore{}, - Success: false, - Error: "invalid score", - JudgeConfigKey: j.judgeConfigKey, - } - } - - reasoning, ok := evalData["reasoning"].(string) - if !ok { - return &datamodel.JudgeResponse{ - Evals: map[string]datamodel.EvalScore{}, - Success: false, - Error: "invalid reasoning", - JudgeConfigKey: j.judgeConfigKey, - } - } - - return &datamodel.JudgeResponse{ - Evals: map[string]datamodel.EvalScore{ - j.metricKey: { - Score: score, - Reasoning: reasoning, - }, - }, - Success: true, - JudgeConfigKey: j.judgeConfigKey, - } -} - -func getMetricKey(config Config, logger interfaces.LDLoggers, configKey string) (string, error) { - // Priority 1: Check top-level evaluationMetricKey field (primary field) - if metricKey := config.EvaluationMetricKey(); strings.TrimSpace(metricKey) != "" { - return strings.TrimSpace(metricKey), nil - } - - // Priority 2: Check top-level evaluationMetricKeys array (deprecated) - keys := config.EvaluationMetricKeys() - for _, key := range keys { - if trimmed := strings.TrimSpace(key); trimmed != "" { - if logger != nil { - logger.Warnf("Judge config %q: using deprecated evaluationMetricKeys; use evaluationMetricKey instead", configKey) - } - return trimmed, nil - } - } - - if configKey != "" { - return "", fmt.Errorf("judge config %q: missing evaluationMetricKey", configKey) - } - return "", fmt.Errorf("missing evaluationMetricKey") -} diff --git a/ldai/judge/judge_test.go b/ldai/judge/judge_test.go deleted file mode 100644 index ca0e71eb..00000000 --- a/ldai/judge/judge_test.go +++ /dev/null @@ -1,1056 +0,0 @@ -package judge - -import ( - "fmt" - "testing" - - "github.com/launchdarkly/go-sdk-common/v3/ldvalue" - "github.com/launchdarkly/go-server-sdk/ldai" - "github.com/launchdarkly/go-server-sdk/ldai/datamodel" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type mockConfig struct { - messages []datamodel.Message - modelParam map[string]ldvalue.Value - customParam map[string]ldvalue.Value - evaluationMetricKey string - evaluationMetricKeys []string -} - -func (m *mockConfig) Messages() []datamodel.Message { - return m.messages -} - -func (m *mockConfig) ModelParam(key string) (ldvalue.Value, bool) { - val, ok := m.modelParam[key] - return val, ok -} - -func (m *mockConfig) CustomModelParam(key string) (ldvalue.Value, bool) { - val, ok := m.customParam[key] - return val, ok -} - -func (m *mockConfig) EvaluationMetricKey() string { - return m.evaluationMetricKey -} - -func (m *mockConfig) EvaluationMetricKeys() []string { - return m.evaluationMetricKeys -} - -type mockTracker struct { - judgeResponses []datamodel.JudgeResponse - usages []ldai.TokenUsage -} - -func (m *mockTracker) TrackJudgeResponse(response datamodel.JudgeResponse) error { - m.judgeResponses = append(m.judgeResponses, response) - return nil -} - -func (m *mockTracker) TrackTokens(usage ldai.TokenUsage) error { - m.usages = append(m.usages, usage) - return nil -} - -type mockProvider struct { - response StructuredResponse - err error - calls [][]datamodel.Message -} - -func (m *mockProvider) InvokeStructuredModel(messages []datamodel.Message, schema map[string]interface{}) (StructuredResponse, error) { - m.calls = append(m.calls, messages) - return m.response, m.err -} - -func TestNew(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - } - tracker := &mockTracker{} - provider := &mockProvider{} - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - assert.NotNil(t, judge) - assert.Equal(t, "$ld:ai:judge:relevance", judge.metricKey) - assert.Equal(t, "test-judge", judge.judgeConfigKey) -} - -func TestNew_MissingMetricKey(t *testing.T) { - config := &mockConfig{} - tracker := &mockTracker{} - provider := &mockProvider{} - - judge, err := New(config, tracker, provider, "test-judge", nil) - assert.Error(t, err) - assert.Nil(t, judge) - assert.Contains(t, err.Error(), "missing evaluationMetricKey") -} - -func TestNew_NilInputs(t *testing.T) { - config := &mockConfig{evaluationMetricKey: "test"} - tracker := &mockTracker{} - provider := &mockProvider{} - - _, err := New(nil, tracker, provider, "test", nil) - assert.Error(t, err) - - _, err = New(config, nil, provider, "test", nil) - assert.Error(t, err) - - _, err = New(config, tracker, nil, "test", nil) - assert.Error(t, err) -} - -func TestEvaluate_Success(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{ - {Role: datamodel.System, Content: "Evaluate this"}, - {Role: datamodel.User, Content: "Input: {{message_history}}"}, - {Role: datamodel.User, Content: "Output: {{response_to_evaluate}}"}, - }, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.85, - "reasoning": "Highly relevant", - }, - }, - }, - Usage: ldai.TokenUsage{Total: 100, Input: 60, Output: 40}, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("test input", "test output", 1.0) - require.NoError(t, err) - assert.NotNil(t, result) - assert.True(t, result.Success) - assert.Equal(t, "test-judge", result.JudgeConfigKey) - assert.Len(t, result.Evals, 1) - assert.Equal(t, 0.85, result.Evals["$ld:ai:judge:relevance"].Score) - assert.Equal(t, "Highly relevant", result.Evals["$ld:ai:judge:relevance"].Reasoning) - - assert.Len(t, tracker.usages, 1) - assert.Equal(t, 100, tracker.usages[0].Total) - // Note: Judge should NOT track responses internally - this is caller's responsibility - // The judge's tracker is only used for usage/duration metrics - assert.Len(t, tracker.judgeResponses, 0, "Judge should not track responses internally") - - require.Len(t, provider.calls, 1) - assert.Equal(t, "Evaluate this", provider.calls[0][0].Content) - assert.Equal(t, "Input: test input", provider.calls[0][1].Content) - assert.Equal(t, "Output: test output", provider.calls[0][2].Content) -} - -func TestEvaluate_NoMessages(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{}, - } - tracker := &mockTracker{} - provider := &mockProvider{} - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.Nil(t, result) -} - -func TestEvaluate_Sampling(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{} - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - sampled := 0 - for i := 0; i < 100; i++ { - result, _ := judge.Evaluate("input", "output", 0.0) - if result != nil { - sampled++ - } - } - assert.Equal(t, 0, sampled) -} - -func TestEvaluate_ProviderError(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{err: fmt.Errorf("provider error")} - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Equal(t, "test-judge", result.JudgeConfigKey) - assert.Contains(t, result.Error, "provider error") -} - -func TestEvaluate_InvalidResponse(t *testing.T) { - tests := []struct { - name string - response map[string]interface{} - }{ - { - name: "missing evaluations", - response: map[string]interface{}{}, - }, - { - name: "missing metric key", - response: map[string]interface{}{ - "evaluations": map[string]interface{}{}, - }, - }, - { - name: "invalid score type", - response: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": "not a number", - "reasoning": "test", - }, - }, - }, - }, - { - name: "score out of range", - response: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 1.5, - "reasoning": "test", - }, - }, - }, - }, - { - name: "invalid reasoning type", - response: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.5, - "reasoning": 123, - }, - }, - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{Content: tt.response}, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.NotEmpty(t, result.Error) - }) - } -} - -func TestEvaluateMessages(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "{{message_history}}"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.9, - "reasoning": "Excellent", - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - messages := []datamodel.Message{ - {Role: datamodel.User, Content: "Hello"}, - {Role: datamodel.Assistant, Content: "Hi there"}, - } - - result, err := judge.EvaluateMessages(messages, "response", 1.0) - require.NoError(t, err) - assert.NotNil(t, result) - assert.True(t, result.Success) - - require.Len(t, provider.calls, 1) - assert.Contains(t, provider.calls[0][0].Content, "Hello\r\nHi there") -} - -func TestGetMetricKey(t *testing.T) { - tests := []struct { - name string - evaluationMetricKey string - evaluationMetricKeys []string - want string - wantErr bool - }{ - { - name: "from top-level field (primary)", - evaluationMetricKey: "$ld:ai:judge:toplevel", - want: "$ld:ai:judge:toplevel", - }, - { - name: "top-level field has priority over array", - evaluationMetricKey: "$ld:ai:judge:toplevel", - evaluationMetricKeys: []string{"$ld:ai:judge:array"}, - want: "$ld:ai:judge:toplevel", - }, - { - name: "missing", - wantErr: true, - }, - { - name: "trim whitespace from top-level", - evaluationMetricKey: " $ld:ai:judge:toplevel ", - want: "$ld:ai:judge:toplevel", - }, - { - name: "from evaluationMetricKeys array", - evaluationMetricKeys: []string{"$ld:ai:judge:relevance", "$ld:ai:judge:accuracy"}, - want: "$ld:ai:judge:relevance", - }, - { - name: "skip empty strings in array", - evaluationMetricKeys: []string{"", " ", "$ld:ai:judge:relevance"}, - want: "$ld:ai:judge:relevance", - }, - { - name: "trim whitespace from array entry", - evaluationMetricKeys: []string{" $ld:ai:judge:relevance "}, - want: "$ld:ai:judge:relevance", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: tt.evaluationMetricKey, - evaluationMetricKeys: tt.evaluationMetricKeys, - } - got, err := getMetricKey(config, nil, "test-judge") - if tt.wantErr { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.Equal(t, tt.want, got) - } - }) - } -} - -func TestBuildSchema(t *testing.T) { - schema := buildSchema("$ld:ai:judge:relevance") - - assert.Equal(t, "object", schema["type"]) - assert.Contains(t, schema, "properties") - assert.Contains(t, schema, "required") - - props := schema["properties"].(map[string]interface{}) - evals := props["evaluations"].(map[string]interface{}) - evalProps := evals["properties"].(map[string]interface{}) - - assert.Contains(t, evalProps, "$ld:ai:judge:relevance") - - metricSchema := evalProps["$ld:ai:judge:relevance"].(map[string]interface{}) - metricProps := metricSchema["properties"].(map[string]interface{}) - - assert.Contains(t, metricProps, "score") - assert.Contains(t, metricProps, "reasoning") - - scoreSchema := metricProps["score"].(map[string]interface{}) - assert.Equal(t, "number", scoreSchema["type"]) - assert.Equal(t, 0.0, scoreSchema["minimum"]) - assert.Equal(t, 1.0, scoreSchema["maximum"]) -} - -func TestBuildSchema_Empty(t *testing.T) { - schema := buildSchema("") - assert.Empty(t, schema) -} - -func TestEvaluate_NegativeScore(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": -0.5, - "reasoning": "negative score", - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Contains(t, result.Error, "invalid score") -} - -func TestEvaluate_ScoreGreaterThanOne(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 1.5, - "reasoning": "over limit", - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Contains(t, result.Error, "invalid score") -} - -func TestEvaluate_NullEvaluationValue(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": nil, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Contains(t, result.Error, "missing evaluation") -} - -func TestEvaluate_NonObjectEvaluationValue(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": "not an object", - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Contains(t, result.Error, "missing evaluation") -} - -func TestEvaluate_ScoreAsString(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": "0.5", - "reasoning": "test", - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Contains(t, result.Error, "invalid score") -} - -func TestEvaluate_ReasoningAsNumber(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.5, - "reasoning": 123, - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Contains(t, result.Error, "invalid reasoning") -} - -func TestEvaluate_EmptyEvaluationsObject(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{}, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Contains(t, result.Error, "missing evaluation") -} - -func TestGetMetricKey_EmptyArray(t *testing.T) { - config := &mockConfig{ - evaluationMetricKeys: []string{}, - } - - _, err := getMetricKey(config, nil, "test-judge") - assert.Error(t, err) - assert.Contains(t, err.Error(), "missing evaluationMetricKey") -} - -func TestGetMetricKey_ArrayWithOnlyEmptyStrings(t *testing.T) { - config := &mockConfig{ - evaluationMetricKeys: []string{"", " ", "\t"}, - } - - _, err := getMetricKey(config, nil, "test-judge") - assert.Error(t, err) - assert.Contains(t, err.Error(), "missing evaluationMetricKey") -} - -func TestEvaluate_ReturnsCorrectResponse(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.75, - "reasoning": "Good response", - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "my-judge-config", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - require.NoError(t, err) - require.NotNil(t, result) - - // Verify the returned response has correct values - assert.Equal(t, "my-judge-config", result.JudgeConfigKey) - assert.True(t, result.Success) - assert.Equal(t, 0.75, result.Evals["$ld:ai:judge:relevance"].Score) - assert.Equal(t, "Good response", result.Evals["$ld:ai:judge:relevance"].Reasoning) - - // Judge should NOT track responses internally - this is caller's responsibility - assert.Len(t, tracker.judgeResponses, 0, "Judge should not track responses internally") -} - -func TestEvaluate_TokenUsageTracked(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.5, - "reasoning": "test", - }, - }, - }, - Usage: ldai.TokenUsage{Total: 150, Input: 90, Output: 60}, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - _, err = judge.Evaluate("input", "output", 1.0) - require.NoError(t, err) - - require.Len(t, tracker.usages, 1) - assert.Equal(t, 150, tracker.usages[0].Total) - assert.Equal(t, 90, tracker.usages[0].Input) - assert.Equal(t, 60, tracker.usages[0].Output) -} - -func TestEvaluate_NoTokenUsageWhenZero(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.5, - "reasoning": "test", - }, - }, - }, - Usage: ldai.TokenUsage{}, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - _, err = judge.Evaluate("input", "output", 1.0) - require.NoError(t, err) - - assert.Len(t, tracker.usages, 0) -} - -func TestEvaluate_ErrorResponseIncludesJudgeConfigKey(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - tracker := &mockTracker{} - provider := &mockProvider{err: fmt.Errorf("test error")} - - judge, err := New(config, tracker, provider, "error-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - assert.NoError(t, err) - assert.NotNil(t, result) - assert.False(t, result.Success) - assert.Equal(t, "error-judge", result.JudgeConfigKey) -} - -// Integration Tests - These verify end-to-end behavior and patterns not caught by unit tests - -// TestDoubleInterpolation_ReservedVariables verifies that the double interpolation pattern works: -// 1. During config fetch, pass literal strings "{{message_history}}" and "{{response_to_evaluate}}" -// 2. First interpolation preserves these placeholders in the template -// 3. During evaluation, second interpolation replaces placeholders with actual values -func TestDoubleInterpolation_ReservedVariables(t *testing.T) { - // Simulate what the client does when fetching a judge config - // The config from LaunchDarkly has templates with {{message_history}} and {{response_to_evaluate}} - rawTemplate := "Input: {{message_history}}\nOutput: {{response_to_evaluate}}" - - // Simulate first interpolation (done by client.JudgeConfig when fetching judge config) - // Variables passed should include literal placeholder strings - variablesForFirstInterpolation := map[string]interface{}{ - "message_history": "{{message_history}}", // Literal string! - "response_to_evaluate": "{{response_to_evaluate}}", // Literal string! - } - - // First interpolation - should preserve placeholders - firstInterpolated := interpolateTemplateForTest(rawTemplate, variablesForFirstInterpolation) - assert.Equal(t, rawTemplate, firstInterpolated, "First interpolation should preserve placeholders") - - // Now simulate what the judge does during Evaluate() - // Second interpolation with actual values - actualInput := "What is LaunchDarkly?" - actualOutput := "LaunchDarkly is a feature management platform." - - variablesForSecondInterpolation := map[string]interface{}{ - "message_history": actualInput, - "response_to_evaluate": actualOutput, - } - - // Second interpolation - should replace with actual values - secondInterpolated := interpolateTemplateForTest(firstInterpolated, variablesForSecondInterpolation) - expected := "Input: What is LaunchDarkly?\nOutput: LaunchDarkly is a feature management platform." - assert.Equal(t, expected, secondInterpolated, "Second interpolation should replace with actual values") -} - -// TestJudgeEvaluation_WithTemplateInterpolation verifies end-to-end template interpolation -func TestJudgeEvaluation_WithTemplateInterpolation(t *testing.T) { - // Config with templates that should be interpolated during evaluation - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:test", - messages: []datamodel.Message{ - {Role: datamodel.System, Content: "You are a judge"}, - {Role: datamodel.User, Content: "Input: {{message_history}}"}, - {Role: datamodel.User, Content: "Output: {{response_to_evaluate}}"}, - }, - } - - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:test": map[string]interface{}{ - "score": 0.9, - "reasoning": "Good response", - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - // Evaluate with actual input/output - actualInput := "What is AI?" - actualOutput := "AI is artificial intelligence" - - result, err := judge.Evaluate(actualInput, actualOutput, 1.0) - require.NoError(t, err) - require.NotNil(t, result) - - // Verify the provider received interpolated messages - require.Len(t, provider.calls, 1) - messages := provider.calls[0] - - assert.Equal(t, "You are a judge", messages[0].Content) - assert.Equal(t, "Input: What is AI?", messages[1].Content, "message_history should be interpolated") - assert.Equal(t, "Output: AI is artificial intelligence", messages[2].Content, "response_to_evaluate should be interpolated") -} - -// TestJudgeTracking_ShouldNotTrackInternally verifies that the judge does NOT track responses internally. -// Tracking should be the responsibility of the caller (AI config being evaluated). -func TestJudgeTracking_ShouldNotTrackInternally(t *testing.T) { - config := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:test", - messages: []datamodel.Message{{Role: datamodel.User, Content: "test"}}, - } - - tracker := &mockTracker{} - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:test": map[string]interface{}{ - "score": 0.5, - "reasoning": "Test", - }, - }, - }, - }, - } - - judge, err := New(config, tracker, provider, "test-judge", nil) - require.NoError(t, err) - - result, err := judge.Evaluate("input", "output", 1.0) - require.NoError(t, err) - require.NotNil(t, result) - - // CRITICAL: Judge should NOT track responses internally - // This should be done by the caller (AI config being evaluated) - assert.Len(t, tracker.judgeResponses, 0, "Judge should not track responses internally - caller's responsibility") - - // Verify usage is still tracked (this is judge-specific) - assert.Len(t, tracker.usages, 0, "No usage was set in this test") -} - -// TestIntegration_AIConfigTracksJudgeResults simulates the real-world pattern where -// an AI config evaluates with a judge and tracks results on its own tracker. -func TestIntegration_AIConfigTracksJudgeResults(t *testing.T) { - // Simulate AI config's tracker - aiConfigTracker := &mockTracker{} - - // Simulate judge's tracker (should NOT be used for judge response tracking) - judgeTracker := &mockTracker{} - - // Judge configuration - judgeConfig := &mockConfig{ - evaluationMetricKey: "$ld:ai:judge:relevance", - messages: []datamodel.Message{ - {Role: datamodel.User, Content: "Evaluate: {{message_history}} -> {{response_to_evaluate}}"}, - }, - } - - provider := &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "$ld:ai:judge:relevance": map[string]interface{}{ - "score": 0.85, - "reasoning": "Highly relevant", - }, - }, - }, - }, - } - - // Create judge with its own tracker - judge, err := New(judgeConfig, judgeTracker, provider, "test-judge", nil) - require.NoError(t, err) - - // AI config evaluates with the judge - result, err := judge.Evaluate("What is AI?", "AI is artificial intelligence", 1.0) - require.NoError(t, err) - require.NotNil(t, result) - - // AI config tracks the result on its own tracker (NOT the judge's tracker) - err = aiConfigTracker.TrackJudgeResponse(*result) - require.NoError(t, err) - - // Verify tracking happened on AI config's tracker - assert.Len(t, aiConfigTracker.judgeResponses, 1, "AI config should track judge response") - assert.Equal(t, 0.85, aiConfigTracker.judgeResponses[0].Evals["$ld:ai:judge:relevance"].Score) - - // Verify judge did NOT track on its own tracker - assert.Len(t, judgeTracker.judgeResponses, 0, "Judge should not track responses internally") -} - -// Helper function to test template interpolation -func interpolateTemplateForTest(template string, vars map[string]interface{}) string { - // Simple string replacement for testing - // Real code uses: mustache.New().ParseString(template).RenderString(vars) - result := template - for key, value := range vars { - placeholder := "{{" + key + "}}" - if str, ok := value.(string); ok { - result = replaceAllForTest(result, placeholder, str) - } - } - return result -} - -func replaceAllForTest(s, old, new string) string { - result := "" - for { - i := indexOfForTest(s, old) - if i == -1 { - result += s - break - } - result += s[:i] + new - s = s[i+len(old):] - } - return result -} - -func indexOfForTest(s, substr string) int { - for i := 0; i <= len(s)-len(substr); i++ { - if s[i:i+len(substr)] == substr { - return i - } - } - return -1 -} - -func newMinimalJudge(t *testing.T, content string) *Judge { - t.Helper() - config := &mockConfig{ - evaluationMetricKey: "metric", - messages: []datamodel.Message{{Role: datamodel.User, Content: content}}, - } - j, err := New(config, &mockTracker{}, &mockProvider{ - response: StructuredResponse{ - Content: map[string]interface{}{ - "evaluations": map[string]interface{}{ - "metric": map[string]interface{}{"score": 1.0, "reasoning": "ok"}, - }, - }, - }, - }, "key", nil) - require.NoError(t, err) - return j -} - -// TestBuildMessages_InjectionVariants is a regression test for HackerOne report #3591852. -// It covers the full set of Mustache control sequences an attacker could inject via a -// user-controlled context variable. All variants must be treated as inert literal text by -// pass 2 — none should cause a placeholder to go unsubstituted. -func TestBuildMessages_InjectionVariants(t *testing.T) { - variants := []struct { - name string - payload string // injected via {{ldctx.user.name}} in the template - }{ - {"delimiter change brackets", "{{=[ ]=}}"}, - {"delimiter change angle", "{{=<% %>=}}"}, - {"partial", "{{> evil}}"}, - {"comment", "{{! drop everything }}"}, - {"triple stache", "{{{raw}}}"}, - {"section", "{{#section}}inject{{/section}}"}, - {"inverted section", "{{^section}}inject{{/section}}"}, - } - - for _, tt := range variants { - t.Run(tt.name, func(t *testing.T) { - // Hand-craft the pass-1 output: user.name resolved to the attack payload, - // reserved placeholder survived as a literal string. - afterPass1 := "Auditing " + tt.payload + ": " + ldai.JudgePlaceholderMessageHistory - - // Pass 2: judge substitutes placeholders - judge := newMinimalJudge(t, afterPass1) - actualHistory := "ACTUAL MESSAGE HISTORY" - messages := judge.buildMessages(actualHistory, "some output") - - require.Len(t, messages, 1) - assert.Contains(t, messages[0].Content, actualHistory, - "payload %q must not blind the judge to the actual history", tt.payload) - assert.NotContains(t, messages[0].Content, ldai.JudgePlaceholderMessageHistory, - "placeholder must be fully substituted after payload %q", tt.payload) - }) - } -} - -// TestBuildMessages_InjectionViaResponse verifies that injection payloads in the response -// being evaluated (not just the history) are equally neutralized. -func TestBuildMessages_InjectionViaResponse(t *testing.T) { - // Pass-1 output: no user vars resolved, both placeholders survived as literal strings. - afterPass1 := "History: " + ldai.JudgePlaceholderMessageHistory + - "\nResponse: " + ldai.JudgePlaceholderResponseToEvaluate - - judge := newMinimalJudge(t, afterPass1) - maliciousResponse := "{{=[ ]=}} INJECTION ATTEMPT" - messages := judge.buildMessages("normal history", maliciousResponse) - - require.Len(t, messages, 1) - assert.Contains(t, messages[0].Content, maliciousResponse, - "malicious content in response must appear verbatim — not silently dropped") - assert.NotContains(t, messages[0].Content, ldai.JudgePlaceholderResponseToEvaluate, - "response placeholder must be fully substituted") -} - -// TestBuildMessages_MultiplePlaceholderOccurrences verifies that when a template contains -// the same placeholder more than once, every occurrence is substituted. -func TestBuildMessages_MultiplePlaceholderOccurrences(t *testing.T) { - template := ldai.JudgePlaceholderMessageHistory + " | " + ldai.JudgePlaceholderMessageHistory - judge := newMinimalJudge(t, template) - messages := judge.buildMessages("HISTORY", "RESPONSE") - - require.Len(t, messages, 1) - assert.Equal(t, "HISTORY | HISTORY", messages[0].Content) -} - -// TestBuildMessages_MustacheSyntaxInContent verifies that Mustache-like syntax inside the -// actual history or response values is treated as literal text and not silently consumed. -// The old Mustache-based pass 2 would have rendered unrecognized tags (e.g. {{user}}) as -// empty strings, corrupting content such as code samples or questions about templating. -func TestBuildMessages_MustacheSyntaxInContent(t *testing.T) { - template := "History: " + ldai.JudgePlaceholderMessageHistory + - "\nResponse: " + ldai.JudgePlaceholderResponseToEvaluate - judge := newMinimalJudge(t, template) - - historyWithMustache := "How do I use {{user}} in Mustache?" - responseWithMustache := "Use {{user}} like this: {{#user}}Hello{{/user}}" - - messages := judge.buildMessages(historyWithMustache, responseWithMustache) - - require.Len(t, messages, 1) - assert.Contains(t, messages[0].Content, historyWithMustache, - "Mustache-like syntax in history must be preserved verbatim") - assert.Contains(t, messages[0].Content, responseWithMustache, - "Mustache-like syntax in response must be preserved verbatim") -} diff --git a/ldai/judge/schema.go b/ldai/judge/schema.go deleted file mode 100644 index 864e28a8..00000000 --- a/ldai/judge/schema.go +++ /dev/null @@ -1,40 +0,0 @@ -package judge - -func buildSchema(metricKey string) map[string]interface{} { - if metricKey == "" { - return map[string]interface{}{} - } - - return map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "evaluations": map[string]interface{}{ - "type": "object", - "description": "Object containing evaluation results for " + metricKey + " metric", - "properties": map[string]interface{}{ - metricKey: map[string]interface{}{ - "type": "object", - "properties": map[string]interface{}{ - "score": map[string]interface{}{ - "type": "number", - "minimum": 0.0, - "maximum": 1.0, - "description": "Score between 0.0 and 1.0 for " + metricKey, - }, - "reasoning": map[string]interface{}{ - "type": "string", - "description": "Reasoning behind the score for " + metricKey, - }, - }, - "required": []string{"score", "reasoning"}, - "additionalProperties": false, - }, - }, - "required": []string{metricKey}, - "additionalProperties": false, - }, - }, - "required": []string{"evaluations"}, - "additionalProperties": false, - } -} diff --git a/ldai/package_info.go b/ldai/package_info.go deleted file mode 100644 index c3312c12..00000000 --- a/ldai/package_info.go +++ /dev/null @@ -1,13 +0,0 @@ -// Package ldai contains an AI SDK suitable for usage with generative AI applications. -package ldai - -const ( - // Version is the current version string of the ldai package. This is updated by our release scripts. - Version = "0.9.4" // {{ x-release-please-version }} - - // SDKName is the canonical name of this AI SDK package. - SDKName = "go-server-sdk/ldai" - - // SDKLanguage is the programming language of this AI SDK. - SDKLanguage = "go" -) diff --git a/ldai/tracker.go b/ldai/tracker.go deleted file mode 100644 index 66473315..00000000 --- a/ldai/tracker.go +++ /dev/null @@ -1,522 +0,0 @@ -package ldai - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "time" - - "github.com/google/uuid" - - ldcommon "github.com/launchdarkly/go-sdk-common/v3" - "github.com/launchdarkly/go-sdk-common/v3/ldcontext" - "github.com/launchdarkly/go-sdk-common/v3/ldvalue" - "github.com/launchdarkly/go-server-sdk/ldai/datamodel" - "github.com/launchdarkly/go-server-sdk/v7/interfaces" -) - -const ( - duration = "$ld:ai:duration:total" - feedbackPositive = "$ld:ai:feedback:user:positive" - feedbackNegative = "$ld:ai:feedback:user:negative" - generationSuccess = "$ld:ai:generation:success" - generationError = "$ld:ai:generation:error" - //nolint:gosec - timeToFirstToken = "$ld:ai:tokens:ttf" - //nolint:gosec - tokenTotal = "$ld:ai:tokens:total" - //nolint:gosec - tokenInput = "$ld:ai:tokens:input" - //nolint:gosec - tokenOutput = "$ld:ai:tokens:output" -) - -// newRunID returns a fresh UUIDv4 that LaunchDarkly uses to group all metric -// events emitted by one Tracker into a single AI run, so they can be analyzed -// together. Each call to CreateTracker on the AI Config mints a new runId. -func newRunID() string { - return uuid.New().String() -} - -// TokenUsage represents the token usage returned by a model provider for a specific request. -type TokenUsage struct { - // Total is the total number of tokens used. - Total int - // Input is the number of input tokens used. - Input int - - // Output is the number of output tokens used. - Output int -} - -// MetricSummary represents a summary of metrics tracked by the tracker. -type MetricSummary struct { - // Duration is the tracked duration in milliseconds. - Duration ldcommon.Option[time.Duration] - // Feedback is the tracked user feedback (positive or negative). - Feedback ldcommon.Option[Feedback] - // Tokens contains information about token usage. - Tokens ldcommon.Option[TokenUsage] - // Success indicates whether the operation was successful. - Success ldcommon.Option[bool] - // TimeToFirstToken is the time to the first token in milliseconds. - TimeToFirstToken ldcommon.Option[time.Duration] -} - -// Set returns true if any of the fields are non-zero. -func (t TokenUsage) Set() bool { - return t.Total > 0 || t.Input > 0 || t.Output > 0 -} - -// Metrics represents the metrics returned by a model provider for a specific request. -type Metrics struct { - // Latency is the latency of the request. - Latency time.Duration - // TimeToFirstToken is the time to the first token of the streamed response. - TimeToFirstToken time.Duration -} - -// ProviderResponse represents the response from a model provider for a specific request. -type ProviderResponse struct { - // Usage is the token usage. - Usage TokenUsage - // Metrics is the request metrics. - Metrics Metrics -} - -// Feedback represents the feedback provided by a user for a model evaluation. -type Feedback string - -const ( - // FeedbackPositive is positive feedback. - FeedbackPositive Feedback = "positive" - // FeedbackNegative is negative feedback. - FeedbackNegative Feedback = "negative" -) - -// EventSink represents the Tracker's requirements for delivering analytic events. This is generally satisfied -// by the LaunchDarkly SDK's TrackMetric method. -type EventSink interface { - // TrackMetric sends a named analytic event to LaunchDarkly relevant to a particular context, and containing a - // metric value and additional data. - TrackMetric( - eventName string, - context ldcontext.Context, - metricValue float64, - data ldvalue.Value, - ) error -} - -// Stopwatch is used to measure the duration of a task. Start will always be called before Stop. -// If an implementation is not provided, the Tracker uses a default implementation that delegates to -// time.Now and time.Since. -type Stopwatch interface { - // Start starts the stopwatch. - Start() - // Stop stops the stopwatch and returns the duration since Start was called. - Stop() time.Duration -} - -// resumptionPayload is the JSON structure encoded into a resumption token. -type resumptionPayload struct { - RunID string `json:"runId"` - ConfigKey string `json:"configKey"` - VariationKey string `json:"variationKey,omitempty"` - Version int `json:"version"` -} - -// Tracker records metrics for a single AI run. -// Unless otherwise noted, the Tracker's methods are not safe for concurrent use. -// -// All events a Tracker emits share a runId (a UUIDv4) so LaunchDarkly can -// correlate them in metrics views. See individual track methods for their -// specific semantics. Call CreateTracker on the AI Config to start a new run. -// A ResumptionToken preserves the runId, so events emitted by a Tracker -// reconstructed in another process correlate with the original run. -type Tracker struct { - key string - runID string - variationKey string - version int - config *Config - context ldcontext.Context - events EventSink - trackData ldvalue.Value - logger interfaces.LDLoggers - stopwatch Stopwatch - - duration ldcommon.Option[time.Duration] - feedback ldcommon.Option[Feedback] - tokens ldcommon.Option[TokenUsage] - success ldcommon.Option[bool] - timeToFirstToken ldcommon.Option[time.Duration] -} - -// Used if a custom Stopwatch is not provided. -type defaultStopwatch struct { - start time.Time -} - -// Start saves the current time using time.Now. -func (d *defaultStopwatch) Start() { - d.start = time.Now() -} - -// Stop returns the duration since Start was called using time.Since. -func (d *defaultStopwatch) Stop() time.Duration { - return time.Since(d.start) -} - -// newTracker creates a new Tracker with the specified runID, key, event sink, config, context, and loggers. -func newTracker( - events EventSink, - runID string, - key string, - variationKey string, - version int, - ctx ldcontext.Context, - config *Config, - loggers interfaces.LDLoggers, -) *Tracker { - return newTrackerWithStopwatch(events, runID, key, variationKey, version, ctx, config, loggers, &defaultStopwatch{}) -} - -// newTrackerWithStopwatch creates a new Tracker with the specified runID, key, event sink, config, context, loggers, -// and stopwatch. This method is used for testing purposes. -func newTrackerWithStopwatch( - events EventSink, - runID string, - key string, - variationKey string, - version int, - ctx ldcontext.Context, - config *Config, - loggers interfaces.LDLoggers, - stopwatch Stopwatch, -) *Tracker { - if config == nil { - panic("LaunchDarkly SDK programmer error: config must never be nil") - } - - builder := ldvalue.ObjectBuild(). - Set("runId", ldvalue.String(runID)). - Set("configKey", ldvalue.String(key)). - Set("version", ldvalue.Int(version)). - Set("providerName", ldvalue.String(config.ProviderName())). - Set("modelName", ldvalue.String(config.ModelName())) - if variationKey != "" { - builder.Set("variationKey", ldvalue.String(variationKey)) - } - trackData := builder.Build() - - return &Tracker{ - key: key, - runID: runID, - variationKey: variationKey, - version: version, - config: config, - trackData: trackData, - events: events, - context: ctx, - logger: loggers, - stopwatch: stopwatch, - } -} - -func (t *Tracker) logWarning(format string, args ...interface{}) { - prefix := "AI Config tracker for '" + t.key + "': " - t.logger.Warnf(prefix+format, args...) -} - -// ResumptionToken returns a URL-safe Base64-encoded token that can be used to reconstruct a tracker -// in a different process (e.g., for deferred feedback). The token contains the runId, configKey, -// variationKey, and version. It does not contain modelName or providerName. -func (t *Tracker) ResumptionToken() string { - payload := resumptionPayload{ - RunID: t.runID, - ConfigKey: t.key, - VariationKey: t.variationKey, - Version: t.version, - } - jsonBytes, _ := json.Marshal(payload) - return base64.RawURLEncoding.EncodeToString(jsonBytes) -} - -// TrackerFromResumptionToken reconstructs a Tracker from a resumption token and the given context. -// This is used for cross-process scenarios (e.g., deferred feedback) where the original tracker -// is no longer available but its runId must be reused. The token is obtained from Tracker.ResumptionToken(). -// The reconstructed tracker will have empty modelName and providerName since these are not included -// in the token. -func TrackerFromResumptionToken(token string, sdk ServerSDK, context ldcontext.Context) (*Tracker, error) { - decoded, err := base64.RawURLEncoding.DecodeString(token) - if err != nil { - return nil, fmt.Errorf("invalid resumption token: %w", err) - } - var payload resumptionPayload - if err := json.Unmarshal(decoded, &payload); err != nil { - return nil, fmt.Errorf("invalid resumption token: %w", err) - } - - builder := ldvalue.ObjectBuild(). - Set("runId", ldvalue.String(payload.RunID)). - Set("configKey", ldvalue.String(payload.ConfigKey)). - Set("version", ldvalue.Int(payload.Version)). - Set("providerName", ldvalue.String("")). - Set("modelName", ldvalue.String("")) - if payload.VariationKey != "" { - builder.Set("variationKey", ldvalue.String(payload.VariationKey)) - } - trackData := builder.Build() - - emptyConfig := Disabled() - - return &Tracker{ - key: payload.ConfigKey, - runID: payload.RunID, - variationKey: payload.VariationKey, - version: payload.Version, - config: &emptyConfig, - trackData: trackData, - events: sdk, - context: context, - logger: sdk.Loggers(), - stopwatch: &defaultStopwatch{}, - }, nil -} - -// TrackDuration tracks the duration of a task. For example, the duration of a model evaluation request may be -// tracked here. See also TrackRequest. -// The duration in milliseconds must fit within a float64. -// -// Records at most once per Tracker; further calls are ignored. -func (t *Tracker) TrackDuration(dur time.Duration) error { - if t.duration.IsSome() { - t.logWarning("Skipping TrackDuration: duration already recorded on this tracker. "+ - "Call CreateTracker on the AI Config for a new run. %s", t.trackData.JSONString()) - return nil - } - t.duration = ldcommon.Some(dur) - return t.events.TrackMetric(duration, t.context, float64(dur.Milliseconds()), t.trackData) -} - -// TrackFeedback tracks the feedback provided by a user for a model evaluation. If the feedback is not -// FeedbackPositive or FeedbackNegative, returns an error and does not track anything. -// -// Records at most once per Tracker; further calls are ignored. -func (t *Tracker) TrackFeedback(feedback Feedback) error { - if t.feedback.IsSome() { - t.logWarning("Skipping TrackFeedback: feedback already recorded on this tracker. "+ - "Call CreateTracker on the AI Config for a new run. %s", t.trackData.JSONString()) - return nil - } - switch feedback { - case FeedbackPositive: - t.feedback = ldcommon.Some(feedback) - return t.events.TrackMetric(feedbackPositive, t.context, 1, t.trackData) - case FeedbackNegative: - t.feedback = ldcommon.Some(feedback) - return t.events.TrackMetric(feedbackNegative, t.context, 1, t.trackData) - default: - return fmt.Errorf("tracker: unexpected feedback value: %v", feedback) - } -} - -// TrackSuccess tracks a successful model evaluation. -// -// Records at most once per Tracker. TrackSuccess and TrackError share state; -// only one of the two can record per Tracker, and subsequent calls are ignored. -func (t *Tracker) TrackSuccess() error { - if t.success.IsSome() { - t.logWarning("Skipping TrackSuccess: success/error already recorded on this tracker. "+ - "Call CreateTracker on the AI Config for a new run. %s", t.trackData.JSONString()) - return nil - } - t.success = ldcommon.Some(true) - - return t.events.TrackMetric(generationSuccess, t.context, 1, t.trackData) -} - -// TrackError tracks an unsuccessful model evaluation. -// -// Records at most once per Tracker. TrackSuccess and TrackError share state; -// only one of the two can record per Tracker, and subsequent calls are ignored. -func (t *Tracker) TrackError() error { - if t.success.IsSome() { - t.logWarning("Skipping TrackError: success/error already recorded on this tracker. "+ - "Call CreateTracker on the AI Config for a new run. %s", t.trackData.JSONString()) - return nil - } - t.success = ldcommon.Some(false) - - return t.events.TrackMetric(generationError, t.context, 1, t.trackData) -} - -// TrackTimeToFirstToken tracks the time to the first token of the streamed response. -// -// Records at most once per Tracker; further calls are ignored. -func (t *Tracker) TrackTimeToFirstToken(dur time.Duration) error { - if t.timeToFirstToken.IsSome() { - t.logWarning("Skipping TrackTimeToFirstToken: time-to-first-token already recorded on this tracker. "+ - "Call CreateTracker on the AI Config for a new run. %s", t.trackData.JSONString()) - return nil - } - t.timeToFirstToken = ldcommon.Some(dur) - return t.events.TrackMetric(timeToFirstToken, t.context, float64(dur.Milliseconds()), t.trackData) -} - -// TrackTokens tracks the token usage for a model evaluation. -// -// Records at most once per Tracker; further calls are ignored. -func (t *Tracker) TrackTokens(usage TokenUsage) error { - if t.tokens.IsSome() { - t.logWarning("Skipping TrackTokens: token usage already recorded on this tracker. "+ - "Call CreateTracker on the AI Config for a new run. %s", t.trackData.JSONString()) - return nil - } - - if usage.Set() { - t.tokens = ldcommon.Some(usage) - } - - var failed bool - - if usage.Total > 0 { - if err1 := t.events.TrackMetric(tokenTotal, t.context, float64(usage.Total), t.trackData); err1 != nil { - t.logWarning("error tracking total token usage: %v", err1) - failed = true - } - } - if usage.Input > 0 { - if err2 := t.events.TrackMetric(tokenInput, t.context, float64(usage.Input), t.trackData); err2 != nil { - t.logWarning("error tracking input token usage: %v", err2) - failed = true - } - } - if usage.Output > 0 { - if err3 := t.events.TrackMetric(tokenOutput, t.context, float64(usage.Output), t.trackData); err3 != nil { - t.logWarning("error tracking output token usage: %v", err3) - failed = true - } - } - - if failed { - return fmt.Errorf("tracker: error tracking token usage, logs contain more information") - } - - return nil -} - -// TrackUsage tracks token usage. -// -// Deprecated: Use TrackTokens instead. -func (t *Tracker) TrackUsage(usage TokenUsage) error { - return t.TrackTokens(usage) -} - -func measureDurationOfTask[T any, A any]( - stopwatch Stopwatch, - arg A, - task func(A) (T, error), -) (T, time.Duration, error) { - stopwatch.Start() - result, err := task(arg) - return result, stopwatch.Stop(), err -} - -// GetSummary returns a summary of all metrics that have been tracked using this tracker. -func (t *Tracker) GetSummary() MetricSummary { - return MetricSummary{ - Duration: t.duration, - Feedback: t.feedback, - Tokens: t.tokens, - Success: t.success, - TimeToFirstToken: t.timeToFirstToken, - } -} - -// TrackRequest tracks metrics for a model evaluation request. The task function should return a ProviderResponse -// which can be used to specify request metrics and token usage. All fields of the returned ProviderResponse are -// optional. -// -// The task function will be passed the current AI Config, which can be used to obtain any parameters or messages -// relevant to the request. -// -// If the task returns an error, then the request is not considered successful and no metrics are tracked. -// Otherwise, the following metrics are tracked: -// 1. Successful model evaluation. -// 2. Any metrics that were that set in the ProviderResponse -// 2a) If Latency was not set in the ProviderResponse's Metrics field, an automatically measured duration. -// 3. Any token usage that was set in the ProviderResponse. -// -// Subsequent calls re-run the task but emit only metrics not already recorded -// on this Tracker. Call CreateTracker on the AI Config to start a new run. -func (t *Tracker) TrackRequest(task func(c *Config) (ProviderResponse, error)) (ProviderResponse, error) { - usage, duration, err := measureDurationOfTask(t.stopwatch, t.config, task) - if err != nil { - if e := t.TrackError(); e != nil { - t.logWarning("error tracking error metric for request: %v", e) - } - - t.logWarning("error executing request: %v", err) - return ProviderResponse{}, err - } - if err := t.TrackSuccess(); err != nil { - t.logWarning("error tracking success metric for request: %v", err) - } - - if usage.Metrics.Latency != 0 { - if err := t.TrackDuration(usage.Metrics.Latency); err != nil { - t.logWarning("error tracking duration metric (user provided) for request: %v", err) - } - } else { - if err := t.TrackDuration(duration); err != nil { - t.logWarning("error tracking duration metric (automatically measured) for request: %v", err) - } - } - - if usage.Metrics.TimeToFirstToken != 0 { - if err := t.TrackTimeToFirstToken(usage.Metrics.TimeToFirstToken); err != nil { - t.logWarning("error tracking time to first token metric for request: %v", err) - } - } - - if usage.Usage.Set() { - // TrackTokens logs errors. - _ = t.TrackTokens(usage.Usage) - } - - return usage, nil -} - -// TrackJudgeResponse tracks the evaluation scores from a judge response. -// -// May be called multiple times per Tracker; each call records the scores from -// the given response. -func (t *Tracker) TrackJudgeResponse(response datamodel.JudgeResponse) error { - if !response.Success { - return nil - } - - // Build the data object once, since it's constant across all iterations - data := t.trackData - if response.JudgeConfigKey != "" { - builder := ldvalue.ObjectBuild() - for _, key := range t.trackData.Keys(nil) { - builder.Set(key, t.trackData.GetByKey(key)) - } - data = builder.Set("judgeConfigKey", ldvalue.String(response.JudgeConfigKey)).Build() - } - - var failed bool - for metricKey, evalScore := range response.Evals { - if err := t.events.TrackMetric(metricKey, t.context, evalScore.Score, data); err != nil { - t.logWarning("error tracking metric %s: %v", metricKey, err) - failed = true - } - } - - if failed { - return fmt.Errorf("error tracking evaluation scores") - } - return nil -} diff --git a/ldai/tracker_test.go b/ldai/tracker_test.go deleted file mode 100644 index ef7eac15..00000000 --- a/ldai/tracker_test.go +++ /dev/null @@ -1,599 +0,0 @@ -package ldai - -import ( - "encoding/base64" - "encoding/json" - "testing" - "time" - - "github.com/launchdarkly/go-server-sdk/ldai/datamodel" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/launchdarkly/go-sdk-common/v3/ldcontext" - "github.com/launchdarkly/go-sdk-common/v3/ldlogtest" - "github.com/launchdarkly/go-sdk-common/v3/ldvalue" -) - -type mockEvents struct { - log *ldlogtest.MockLog - events []trackEvent -} - -type trackEvent struct { - name string - context ldcontext.Context - metricValue float64 - data ldvalue.Value -} - -func newMockEvents() *mockEvents { - return &mockEvents{log: ldlogtest.NewMockLog()} -} - -func (m *mockEvents) TrackMetric(eventName string, context ldcontext.Context, metricValue float64, data ldvalue.Value) error { - m.events = append(m.events, trackEvent{name: eventName, context: context, metricValue: metricValue, data: data}) - return nil -} - -func TestTracker_NewPanicsWithNilConfig(t *testing.T) { - assert.Panics(t, func() { - newTracker(newMockEvents(), newRunID(), "key", "variationKey", 1, ldcontext.New("key"), nil, nil) - }) -} - -func TestTracker_NewDoesNotPanicWithConfig(t *testing.T) { - assert.NotPanics(t, func() { - newTracker(newMockEvents(), newRunID(), "key", "variationKey", 1, ldcontext.New("key"), &Config{}, nil) - }) -} - -func makeTrackData(configKey, variationKey string, version int, config *Config, runId string) ldvalue.Value { - builder := ldvalue.ObjectBuild(). - Set("runId", ldvalue.String(runId)). - Set("configKey", ldvalue.String(configKey)). - Set("version", ldvalue.Int(version)). - Set("providerName", ldvalue.String(config.ProviderName())). - Set("modelName", ldvalue.String(config.ModelName())) - if variationKey != "" { - builder.Set("variationKey", ldvalue.String(variationKey)) - } - return builder.Build() -} - -func extractRunId(t *testing.T, events *mockEvents) string { - t.Helper() - require.NotEmpty(t, events.events, "expected at least one event to extract runId") - runId := events.events[0].data.GetByKey("runId").StringValue() - require.NotEmpty(t, runId, "expected runId to be non-empty") - return runId -} - -func TestTracker_TrackSuccess(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, nil) - assert.NoError(t, tracker.TrackSuccess()) - - runId := extractRunId(t, events) - expectedEvents := []trackEvent{ - { - name: "$ld:ai:generation:success", - context: ldcontext.New("key"), - metricValue: 1.0, - data: makeTrackData("key", "variationKey", 1, config, runId), - }, - } - - assert.ElementsMatch(t, expectedEvents, events.events) -} - -func TestTracker_TrackError(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 2, ldcontext.New("key"), config, nil) - assert.NoError(t, tracker.TrackError()) - - runId := extractRunId(t, events) - expectedEvents := []trackEvent{ - { - name: "$ld:ai:generation:error", - context: ldcontext.New("key"), - metricValue: 1.0, - data: makeTrackData("key", "variationKey", 2, config, runId), - }, - } - - assert.ElementsMatch(t, expectedEvents, events.events) -} - -func TestTracker_TrackRequest(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 3, ldcontext.New("key"), config, nil) - - expectedResponse := ProviderResponse{ - Usage: TokenUsage{ - Total: 1, - }, - Metrics: Metrics{ - Latency: 10 * time.Millisecond, - TimeToFirstToken: 42 * time.Millisecond, - }, - } - - r, err := tracker.TrackRequest(func(c *Config) (ProviderResponse, error) { - return expectedResponse, nil - }) - - assert.NoError(t, err) - assert.Equal(t, expectedResponse, r) - - runId := extractRunId(t, events) - expectedEvents := []trackEvent{ - { - name: "$ld:ai:generation:success", - context: ldcontext.New("key"), - metricValue: 1, - data: makeTrackData("key", "variationKey", 3, config, runId), - }, - { - name: "$ld:ai:duration:total", - context: ldcontext.New("key"), - metricValue: 10.0, - data: makeTrackData("key", "variationKey", 3, config, runId), - }, - { - name: "$ld:ai:tokens:total", - context: ldcontext.New("key"), - metricValue: 1, - data: makeTrackData("key", "variationKey", 3, config, runId), - }, - { - name: "$ld:ai:tokens:ttf", - context: ldcontext.New("key"), - metricValue: 42.0, - data: makeTrackData("key", "variationKey", 3, config, runId), - }, - } - - assert.ElementsMatch(t, expectedEvents, events.events) -} - -func TestTracker_TrackRequestReceivesConfig(t *testing.T) { - events := newMockEvents() - - expectedConfig := NewConfig(). - WithMessage("hello", datamodel.Assistant). - WithModelName("model"). - WithProviderName("provider"). - WithModelParam("param", ldvalue.String("value")). - WithCustomModelParam("custom", ldvalue.String("value")). - Enable(). - Build() - - tracker := newTracker(events, newRunID(), "key", "variationKey", 4, ldcontext.New("key"), &expectedConfig, nil) - - var gotConfig *Config - _, _ = tracker.TrackRequest(func(c *Config) (ProviderResponse, error) { - gotConfig = c - return ProviderResponse{}, nil - }) - - assert.Equal(t, expectedConfig, *gotConfig) -} - -type mockStopwatch time.Duration - -func (m mockStopwatch) Start() {} - -func (m mockStopwatch) Stop() time.Duration { - return time.Duration(m) -} - -func TestTracker_LatencyMeasuredIfNotProvided(t *testing.T) { - events := newMockEvents() - config := &Config{} - - tracker := newTrackerWithStopwatch( - events, newRunID(), "key", "variationKey", 5, ldcontext.New("key"), config, nil, mockStopwatch(42*time.Millisecond)) - - expectedResponse := ProviderResponse{ - Usage: TokenUsage{ - Total: 1, - }, - } - - r, err := tracker.TrackRequest(func(c *Config) (ProviderResponse, error) { - return expectedResponse, nil - }) - - assert.NoError(t, err) - assert.Equal(t, expectedResponse, r) - - require.Equal(t, 3, len(events.events)) - gotEvent := events.events[1] - assert.Equal(t, "$ld:ai:duration:total", gotEvent.name) - assert.Equal(t, 42.0, gotEvent.metricValue) -} - -func TestTracker_TrackDuration(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 6, ldcontext.New("key"), config, nil) - - assert.NoError(t, tracker.TrackDuration(time.Millisecond*10)) - - runId := extractRunId(t, events) - expectedEvent := trackEvent{ - name: "$ld:ai:duration:total", - context: ldcontext.New("key"), - metricValue: 10.0, - data: makeTrackData("key", "variationKey", 6, config, runId), - } - - assert.ElementsMatch(t, []trackEvent{expectedEvent}, events.events) -} - -func TestTracker_TrackFeedback(t *testing.T) { - t.Run("positive feedback", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 7, ldcontext.New("key"), config, nil) - - assert.NoError(t, tracker.TrackFeedback(FeedbackPositive)) - - runId := extractRunId(t, events) - expectedEvent := trackEvent{ - name: "$ld:ai:feedback:user:positive", - context: ldcontext.New("key"), - metricValue: 1.0, - data: makeTrackData("key", "variationKey", 7, config, runId), - } - - assert.ElementsMatch(t, []trackEvent{expectedEvent}, events.events) - }) - - t.Run("negative feedback", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 7, ldcontext.New("key"), config, nil) - - assert.NoError(t, tracker.TrackFeedback(FeedbackNegative)) - - runId := extractRunId(t, events) - expectedEvent := trackEvent{ - name: "$ld:ai:feedback:user:negative", - context: ldcontext.New("key"), - metricValue: 1.0, - data: makeTrackData("key", "variationKey", 7, config, runId), - } - - assert.ElementsMatch(t, []trackEvent{expectedEvent}, events.events) - }) - - t.Run("invalid feedback returns error", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 7, ldcontext.New("key"), config, nil) - - assert.Error(t, tracker.TrackFeedback("not a valid feedback value")) - assert.Empty(t, events.events) - }) -} - -func TestTracker_TrackTokens(t *testing.T) { - t.Run("only one field set, only one event", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 8, ldcontext.New("key"), config, nil) - - assert.NoError(t, tracker.TrackTokens(TokenUsage{ - Total: 42, - })) - - runId := extractRunId(t, events) - expectedEvent := trackEvent{ - name: "$ld:ai:tokens:total", - context: ldcontext.New("key"), - metricValue: 42.0, - data: makeTrackData("key", "variationKey", 8, config, runId), - } - - assert.ElementsMatch(t, []trackEvent{expectedEvent}, events.events) - }) - - t.Run("all fields set, all events", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 9, ldcontext.New("key"), config, nil) - - assert.NoError(t, tracker.TrackTokens(TokenUsage{ - Total: 42, - Input: 20, - Output: 22, - })) - - runId := extractRunId(t, events) - expectedTotal := trackEvent{ - name: "$ld:ai:tokens:total", - context: ldcontext.New("key"), - metricValue: 42.0, - data: makeTrackData("key", "variationKey", 9, config, runId), - } - - expectedInput := trackEvent{ - name: "$ld:ai:tokens:input", - context: ldcontext.New("key"), - metricValue: 20.0, - data: makeTrackData("key", "variationKey", 9, config, runId), - } - - expectedOutput := trackEvent{ - name: "$ld:ai:tokens:output", - context: ldcontext.New("key"), - metricValue: 22.0, - data: makeTrackData("key", "variationKey", 9, config, runId), - } - - assert.ElementsMatch(t, []trackEvent{expectedTotal, expectedInput, expectedOutput}, events.events) - }) -} - -func TestTracker_GetSummary(t *testing.T) { - t.Run("empty summary when nothing tracked", func(t *testing.T) { - events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 10, ldcontext.New("key"), &Config{}, nil) - - summary := tracker.GetSummary() - - assert.True(t, summary.Duration.IsNone()) - assert.True(t, summary.Feedback.IsNone()) - assert.True(t, summary.Tokens.IsNone()) - assert.True(t, summary.Success.IsNone()) - assert.True(t, summary.TimeToFirstToken.IsNone()) - }) - - t.Run("first duration is returned", func(t *testing.T) { - events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 11, ldcontext.New("key"), &Config{}, events.log.Loggers) - - _ = tracker.TrackDuration(time.Millisecond * 10) - _ = tracker.TrackDuration(time.Millisecond * 20) - - summary := tracker.GetSummary() - - assert.True(t, summary.Duration.IsSome()) - assert.Equal(t, time.Millisecond*10, summary.Duration.Unwrap()) - }) - - t.Run("first feedback is returned", func(t *testing.T) { - events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 12, ldcontext.New("key"), &Config{}, events.log.Loggers) - - _ = tracker.TrackFeedback(FeedbackPositive) - _ = tracker.TrackFeedback(FeedbackNegative) - - summary := tracker.GetSummary() - - assert.True(t, summary.Feedback.IsSome()) - assert.Equal(t, FeedbackPositive, summary.Feedback.Unwrap()) - }) - - t.Run("success status tracked correctly", func(t *testing.T) { - events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 13, ldcontext.New("key"), &Config{}, nil) - - _ = tracker.TrackSuccess() - - summary := tracker.GetSummary() - - assert.True(t, summary.Success.IsSome()) - assert.True(t, summary.Success.Unwrap()) - }) - - t.Run("time to first token is returned", func(t *testing.T) { - events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 14, ldcontext.New("key"), &Config{}, nil) - - duration := time.Millisecond * 30 - _ = tracker.TrackTimeToFirstToken(duration) - - summary := tracker.GetSummary() - - assert.True(t, summary.TimeToFirstToken.IsSome()) - assert.Equal(t, duration, summary.TimeToFirstToken.Unwrap()) - }) - - t.Run("token usage is returned", func(t *testing.T) { - events := newMockEvents() - tracker := newTracker(events, newRunID(), "key", "variationKey", 15, ldcontext.New("key"), &Config{}, nil) - - usage := TokenUsage{ - Total: 100, - Input: 40, - Output: 60, - } - _ = tracker.TrackTokens(usage) - - summary := tracker.GetSummary() - - assert.True(t, summary.Tokens.IsSome()) - assert.Equal(t, usage, summary.Tokens.Unwrap()) - }) -} - -func TestTracker_RunIdPresentInTrackData(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, nil) - _ = tracker.TrackSuccess() - - require.NotEmpty(t, events.events) - data := events.events[0].data - runId := data.GetByKey("runId").StringValue() - assert.NotEmpty(t, runId, "runId should be present and non-empty in track data") -} - -func TestTracker_AtMostOnce(t *testing.T) { - t.Run("TrackDuration only tracks once", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) - - assert.NoError(t, tracker.TrackDuration(10*time.Millisecond)) - assert.NoError(t, tracker.TrackDuration(20*time.Millisecond)) - - count := 0 - for _, e := range events.events { - if e.name == "$ld:ai:duration:total" { - count++ - } - } - assert.Equal(t, 1, count, "TrackDuration should only emit one event") - }) - - t.Run("TrackTimeToFirstToken only tracks once", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) - - assert.NoError(t, tracker.TrackTimeToFirstToken(10*time.Millisecond)) - assert.NoError(t, tracker.TrackTimeToFirstToken(20*time.Millisecond)) - - count := 0 - for _, e := range events.events { - if e.name == "$ld:ai:tokens:ttf" { - count++ - } - } - assert.Equal(t, 1, count, "TrackTimeToFirstToken should only emit one event") - }) - - t.Run("TrackTokens only tracks once", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) - - assert.NoError(t, tracker.TrackTokens(TokenUsage{Total: 10})) - assert.NoError(t, tracker.TrackTokens(TokenUsage{Total: 20})) - - count := 0 - for _, e := range events.events { - if e.name == "$ld:ai:tokens:total" { - count++ - } - } - assert.Equal(t, 1, count, "TrackTokens should only emit one event") - }) - - t.Run("TrackFeedback only tracks once", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) - - assert.NoError(t, tracker.TrackFeedback(FeedbackPositive)) - assert.NoError(t, tracker.TrackFeedback(FeedbackNegative)) - - count := 0 - for _, e := range events.events { - if e.name == "$ld:ai:feedback:user:positive" || e.name == "$ld:ai:feedback:user:negative" { - count++ - } - } - assert.Equal(t, 1, count, "TrackFeedback should only emit one event") - }) - - t.Run("TrackSuccess only tracks once", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) - - assert.NoError(t, tracker.TrackSuccess()) - assert.NoError(t, tracker.TrackSuccess()) - - count := 0 - for _, e := range events.events { - if e.name == "$ld:ai:generation:success" { - count++ - } - } - assert.Equal(t, 1, count, "TrackSuccess should only emit one event") - }) - - t.Run("TrackError only tracks once", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) - - assert.NoError(t, tracker.TrackError()) - assert.NoError(t, tracker.TrackError()) - - count := 0 - for _, e := range events.events { - if e.name == "$ld:ai:generation:error" { - count++ - } - } - assert.Equal(t, 1, count, "TrackError should only emit one event") - }) - - t.Run("TrackSuccess then TrackError only tracks success", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "key", "variationKey", 1, ldcontext.New("key"), config, events.log.Loggers) - - assert.NoError(t, tracker.TrackSuccess()) - assert.NoError(t, tracker.TrackError()) - - assert.Equal(t, 1, len(events.events)) - assert.Equal(t, "$ld:ai:generation:success", events.events[0].name) - }) -} - -func TestTracker_ResumptionToken(t *testing.T) { - t.Run("produces valid base64url-encoded token", func(t *testing.T) { - events := newMockEvents() - config := &Config{} - tracker := newTracker(events, newRunID(), "my-config", "var-1", 3, ldcontext.New("key"), config, nil) - - token := tracker.ResumptionToken() - assert.NotEmpty(t, token) - - // Decode and verify - decoded, err := base64.RawURLEncoding.DecodeString(token) - require.NoError(t, err) - - var payload struct { - RunID string `json:"runId"` - ConfigKey string `json:"configKey"` - VariationKey string `json:"variationKey"` - Version int `json:"version"` - } - require.NoError(t, json.Unmarshal(decoded, &payload)) - - assert.NotEmpty(t, payload.RunID) - assert.Equal(t, "my-config", payload.ConfigKey) - assert.Equal(t, "var-1", payload.VariationKey) - assert.Equal(t, 3, payload.Version) - }) - - t.Run("does not include modelName or providerName", func(t *testing.T) { - events := newMockEvents() - config := NewConfig().WithModelName("gpt-4").WithProviderName("openai").Build() - tracker := newTracker(events, newRunID(), "key", "var", 1, ldcontext.New("key"), &config, nil) - - token := tracker.ResumptionToken() - decoded, err := base64.RawURLEncoding.DecodeString(token) - require.NoError(t, err) - - var raw map[string]interface{} - require.NoError(t, json.Unmarshal(decoded, &raw)) - - _, hasModel := raw["modelName"] - _, hasProvider := raw["providerName"] - assert.False(t, hasModel, "token should not contain modelName") - assert.False(t, hasProvider, "token should not contain providerName") - }) -} diff --git a/release-please-config.json b/release-please-config.json index bf986039..b13cecb6 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -27,16 +27,6 @@ "package_info.go" ] }, - "ldai" : { - "package-name": "ldai", - "release-type" : "go", - "bump-minor-pre-major" : true, - "tag-separator": "/", - "versioning" : "default", - "extra-files" : [ - "package_info.go" - ] - }, "ldmiddleware" : { "package-name": "ldmiddleware", "release-type" : "go",