From 16102a9d13dd6839adf209dada6b7e15f5e70b1c Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 15 Apr 2026 09:27:35 -0500 Subject: [PATCH 01/19] feat!: Add per-execution runId and at-most-once event tracking - Each tracker now carries a runId (UUIDv4) included in all emitted events, scoping every metric to a single execution - At-most-once semantics: duplicate calls to TrackDuration, TrackTokens, TrackSuccess/TrackError, TrackFeedback, and TrackTimeToFirstToken on the same tracker are dropped with a warning Co-Authored-By: Claude Sonnet 4.6 --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 54 ++- .../server-ai/test/LdAiConfigTrackerTest.cs | 311 ++++++++++++------ 2 files changed, 263 insertions(+), 102 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 93bfd372..ba7def72 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -17,6 +17,14 @@ public class LdAiConfigTracker : ILdAiConfigTracker private readonly ILaunchDarklyClient _client; private readonly Context _context; private readonly LdValue _trackData; + private readonly string _runId = Guid.NewGuid().ToString(); + private readonly ILogger _logger; + + private bool _trackedDuration; + private bool _trackedTimeToFirstToken; + private bool _trackedTokens; + private bool _trackedFeedback; + private bool? _trackedSuccess; private const string Duration = "$ld:ai:duration:total"; private const string FeedbackPositive = "$ld:ai:feedback:user:positive"; @@ -42,8 +50,10 @@ public LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfi Config = config ?? throw new ArgumentNullException(nameof(config)); _client = client ?? throw new ArgumentNullException(nameof(client)); _context = context; + _logger = client.GetLogger(); _trackData = LdValue.ObjectFrom(new Dictionary { + { "runId", LdValue.Of(_runId) }, { "variationKey", LdValue.Of(config.VariationKey)}, { "version", LdValue.Of(config.Version)}, { "configKey" , LdValue.Of(configKey ?? throw new ArgumentNullException(nameof(configKey))) }, @@ -57,8 +67,16 @@ public LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfi public LdAiConfig Config { get; } /// - public void TrackDuration(float durationMs) => + public void TrackDuration(float durationMs) + { + if (_trackedDuration) + { + _logger?.Warn("Duration has already been tracked for this operation."); + return; + } + _trackedDuration = true; _client.Track(Duration, _context, _trackData, durationMs); + } /// @@ -74,12 +92,26 @@ public async Task TrackDurationOfTask(Task task) } /// - public void TrackTimeToFirstToken(float timeToFirstTokenMs) => + public void TrackTimeToFirstToken(float timeToFirstTokenMs) + { + if (_trackedTimeToFirstToken) + { + _logger?.Warn("Time to first token has already been tracked for this operation."); + return; + } + _trackedTimeToFirstToken = true; _client.Track(TimeToFirstToken, _context, _trackData, timeToFirstTokenMs); + } /// public void TrackFeedback(Feedback feedback) { + if (_trackedFeedback) + { + _logger?.Warn("Feedback has already been tracked for this operation."); + return; + } + _trackedFeedback = true; switch (feedback) { case Feedback.Positive: @@ -96,12 +128,24 @@ public void TrackFeedback(Feedback feedback) /// public void TrackSuccess() { + if (_trackedSuccess.HasValue) + { + _logger?.Warn("Generation result has already been tracked for this operation."); + return; + } + _trackedSuccess = true; _client.Track(GenerationSuccess, _context, _trackData, 1); } /// public void TrackError() { + if (_trackedSuccess.HasValue) + { + _logger?.Warn("Generation result has already been tracked for this operation."); + return; + } + _trackedSuccess = false; _client.Track(GenerationError, _context, _trackData, 1); } @@ -137,6 +181,12 @@ public async Task TrackRequest(Task request) /// public void TrackTokens(Usage usage) { + if (_trackedTokens) + { + _logger?.Warn("Tokens have already been tracked for this operation."); + return; + } + _trackedTokens = true; if (usage.Total is > 0) { _client.Track(TokenTotal, _context, _trackData, usage.Total.Value); diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 70c397dd..baad45f3 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -10,6 +10,17 @@ namespace LaunchDarkly.Sdk.Server.Ai { public class LdAiTrackerTest { + private static bool MatchesTrackData(LdValue actual, string flagKey, LdAiConfig config) + { + return actual.Get("variationKey").Equals(LdValue.Of(config.VariationKey)) && + actual.Get("version").Equals(LdValue.Of(config.Version)) && + actual.Get("configKey").Equals(LdValue.Of(flagKey)) && + actual.Get("modelName").Equals(LdValue.Of(config.Model.Name)) && + actual.Get("providerName").Equals(LdValue.Of(config.Provider.Name)) && + actual.Get("runId").Type == LdValueType.String && + actual.Get("runId").AsString.Length > 0; + } + [Fact] public void ThrowsIfClientIsNull() { @@ -33,6 +44,24 @@ public void ThrowsIfKeyIsNull() new LdAiConfigTracker(mockClient.Object, null, LdAiConfig.Disabled, Context.New("key"))); } + [Fact] + public void TrackDataIncludesRunId() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + tracker.TrackDuration(1.0f); + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.Is(d => + d.Get("runId").Type == LdValueType.String && + d.Get("runId").AsString.Length > 0), + 1.0f), Times.Once); + } + [Fact] public void CanTrackDuration() { @@ -40,18 +69,11 @@ public void CanTrackDuration() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); tracker.TrackDuration(1.0f); - mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, data, 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); } [Fact] @@ -61,18 +83,11 @@ public void CanTrackTimeToFirstToken() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); tracker.TrackTimeToFirstToken(1.0f); - mockClient.Verify(x => x.Track("$ld:ai:tokens:ttf", context, data, 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:ttf", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); } [Fact] @@ -82,18 +97,11 @@ public void CanTrackSuccess() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); tracker.TrackSuccess(); - mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, data, 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); } @@ -104,18 +112,11 @@ public void CanTrackError() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); tracker.TrackError(); - mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, data, 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); } @@ -126,14 +127,6 @@ public async void CanTrackDurationOfTask() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); @@ -151,33 +144,40 @@ public async void CanTrackDurationOfTask() // error so this isn't flaky. If this proves to be really flaky, we can at least constrain it to be // between 0 and some large number. mockClient.Verify( - x => x.Track("$ld:ai:duration:total", context, data, + x => x.Track("$ld:ai:duration:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), It.IsInRange(0, 500, Range.Inclusive)), Times.Once); } [Fact] - public void CanTrackFeedback() + public void CanTrackPositiveFeedback() { var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); tracker.TrackFeedback(Feedback.Positive); + + mockClient.Verify(x => x.Track("$ld:ai:feedback:user:positive", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + } + + [Fact] + public void CanTrackNegativeFeedback() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); tracker.TrackFeedback(Feedback.Negative); - mockClient.Verify(x => x.Track("$ld:ai:feedback:user:positive", context, data, 1.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:feedback:user:negative", context, data, 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:feedback:user:negative", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); } [Fact] @@ -187,14 +187,6 @@ public void CanTrackTokens() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); @@ -206,9 +198,12 @@ public void CanTrackTokens() }; tracker.TrackTokens(givenUsage); - mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, data, 1.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, data, 2.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, data, 3.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 2.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 3.0f), Times.Once); } [Fact] @@ -218,14 +213,6 @@ public void CanTrackResponseWithSpecificLatency() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); @@ -249,11 +236,16 @@ public void CanTrackResponseWithSpecificLatency() var result = tracker.TrackRequest(Task.Run(() => givenResponse)); Assert.Equal(givenResponse, result.Result); - mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, data, 1.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, data, 1.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, data, 2.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, data, 3.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, data, 500.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 2.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 3.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 500.0f), Times.Once); } [Fact] @@ -263,14 +255,6 @@ public void CanTrackResponseWithPartialData() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); @@ -287,10 +271,12 @@ public void CanTrackResponseWithPartialData() var result = tracker.TrackRequest(Task.Run(() => givenResponse)); Assert.Equal(givenResponse, result.Result); - mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, data, 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); // if latency isn't provided via Statistics, then it is automatically measured. - mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, data, It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), It.IsAny()), Times.Once); } [Fact] @@ -300,23 +286,148 @@ public async Task CanTrackExceptionFromResponse() var context = Context.New("key"); const string flagKey = "key"; var config = LdAiConfig.Disabled; - var data = LdValue.ObjectFrom(new Dictionary - { - { "variationKey", LdValue.Of(config.VariationKey) }, - { "version", LdValue.Of(config.Version) }, - { "configKey", LdValue.Of(flagKey) }, - { "modelName", LdValue.Of(config.Model.Name) }, - { "providerName", LdValue.Of(config.Provider.Name) } - }); var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); await Assert.ThrowsAsync(() => tracker.TrackRequest(Task.FromException(new System.Exception("I am an exception")))); - mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, data, 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); // if latency isn't provided via Statistics, then it is automatically measured. - mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, data, It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.Is(d => MatchesTrackData(d, flagKey, config)), It.IsAny()), Times.Once); + } + + [Fact] + public void DuplicateTrackDurationIsIgnored() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + tracker.TrackDuration(1.0f); + tracker.TrackDuration(2.0f); + + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public void DuplicateTrackTimeToFirstTokenIsIgnored() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + tracker.TrackTimeToFirstToken(1.0f); + tracker.TrackTimeToFirstToken(2.0f); + + mockClient.Verify(x => x.Track("$ld:ai:tokens:ttf", context, + It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public void DuplicateTrackTokensIsIgnored() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + var usage = new Usage { Total = 1, Input = 2, Output = 3 }; + + tracker.TrackTokens(usage); + tracker.TrackTokens(usage); + + mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, + It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public void DuplicateTrackFeedbackIsIgnored() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + tracker.TrackFeedback(Feedback.Positive); + tracker.TrackFeedback(Feedback.Negative); + + mockClient.Verify(x => x.Track("$ld:ai:feedback:user:positive", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:feedback:user:negative", context, + It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public void DuplicateTrackSuccessIsIgnored() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + tracker.TrackSuccess(); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.IsAny(), It.IsAny()), Times.Once); + } + + [Fact] + public void TrackErrorAfterSuccessIsIgnored() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + tracker.TrackSuccess(); + tracker.TrackError(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, + It.IsAny(), It.IsAny()), Times.Never); + } + + [Fact] + public void TrackSuccessAfterErrorIsIgnored() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + tracker.TrackError(); + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.IsAny(), It.IsAny()), Times.Never); } } } From 4dee48f4abbd79335814c0bc9088271cb149c828 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 15 Apr 2026 12:36:25 -0500 Subject: [PATCH 02/19] feat!: Add per-execution runId, at-most-once tracking, and cross-process tracker resumption Co-Authored-By: Claude Opus 4.6 --- .../server-ai/src/Interfaces/ILdAiClient.cs | 13 ++ .../src/Interfaces/ILdAiConfigTracker.cs | 18 ++ pkgs/sdk/server-ai/src/LdAiClient.cs | 58 ++++++ pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 45 ++++- pkgs/sdk/server-ai/test/LdAiClientTest.cs | 152 +++++++++++++++ .../server-ai/test/LdAiConfigTrackerTest.cs | 184 +++++++++++++++++- 6 files changed, 467 insertions(+), 3 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs index 28fd3dfb..9e56bc26 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs @@ -40,4 +40,17 @@ public ILdAiConfigTracker CompletionConfig(string key, Context context, LdAiConf [Obsolete("Use CompletionConfig instead.")] public ILdAiConfigTracker Config(string key, Context context, LdAiConfig defaultValue = null, IReadOnlyDictionary variables = null); + + /// + /// Reconstructs a tracker from a resumption token. This enables cross-process scenarios + /// such as deferred feedback, where a tracker's runId needs to be reused in a different + /// process or at a later time. + /// + /// The reconstructed tracker will have empty model and provider names, as these are not + /// included in the resumption token. + /// + /// the resumption token obtained from + /// the context to use for track events + /// a tracker associated with the original runId + public ILdAiConfigTracker CreateTracker(string resumptionToken, Context context); } diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index 422bb261..f99f3cd1 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -16,6 +16,14 @@ public interface ILdAiConfigTracker /// public LdAiConfig Config { get; } + /// + /// A URL-safe Base64-encoded token that can be used to reconstruct this tracker in a different + /// process or at a later time. The token contains the runId, configKey, variationKey, and version. + /// + /// Use to reconstruct a tracker from this token. + /// + public string ResumptionToken { get; } + /// /// Tracks a duration metric related to this config. For example, if a particular operation /// related to usage of the AI model takes 100ms, this can be tracked and made available in @@ -101,4 +109,14 @@ public interface ILdAiConfigTracker /// /// the token usage public void TrackTokens(Usage usage); + + /// + /// Creates a new tracker with a fresh runId for tracking a new execution. + /// Each call returns a tracker with independent at-most-once tracking state, + /// allowing multiple executions to be tracked separately against the same config. + /// + /// Returns null if the config is disabled. + /// + /// a new tracker, or null if the config is disabled + public ILdAiConfigTracker CreateTracker(); } diff --git a/pkgs/sdk/server-ai/src/LdAiClient.cs b/pkgs/sdk/server-ai/src/LdAiClient.cs index 803b3d5a..d5374878 100644 --- a/pkgs/sdk/server-ai/src/LdAiClient.cs +++ b/pkgs/sdk/server-ai/src/LdAiClient.cs @@ -2,7 +2,9 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; +using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using LaunchDarkly.Sdk.Server.Ai.Adapters; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.DataModel; @@ -137,6 +139,62 @@ public ILdAiConfigTracker Config(string key, Context context, LdAiConfig default } + /// + public ILdAiConfigTracker CreateTracker(string resumptionToken, Context context) + { + if (resumptionToken == null) throw new ArgumentNullException(nameof(resumptionToken)); + + var base64 = resumptionToken.Replace('-', '+').Replace('_', '/'); + switch (base64.Length % 4) + { + case 2: base64 += "=="; break; + case 3: base64 += "="; break; + } + + ResumptionPayload payload; + try + { + var json = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + payload = JsonSerializer.Deserialize(json); + } + catch (Exception e) when (e is FormatException || e is JsonException) + { + throw new ArgumentException("Invalid resumption token", nameof(resumptionToken), e); + } + + if (string.IsNullOrEmpty(payload.RunId) || string.IsNullOrEmpty(payload.ConfigKey)) + { + throw new ArgumentException("Resumption token is missing required fields (runId, configKey)", + nameof(resumptionToken)); + } + + var config = new LdAiConfig( + true, + null, + new Meta { VariationKey = payload.VariationKey ?? "", Version = payload.Version }, + null, + null + ); + + return new LdAiConfigTracker(_client, payload.ConfigKey, config, context, payload.RunId); + } + + private class ResumptionPayload + { + [JsonPropertyName("runId")] + public string RunId { get; set; } + + [JsonPropertyName("configKey")] + public string ConfigKey { get; set; } + + [JsonPropertyName("variationKey")] + public string VariationKey { get; set; } + + [JsonPropertyName("version")] + public int Version { get; set; } + } + + private static IDictionary AddSingleKindContextAttributes(Context context) { var attributes = new Dictionary diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index ba7def72..66fda808 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.Interfaces; @@ -16,8 +18,9 @@ public class LdAiConfigTracker : ILdAiConfigTracker { private readonly ILaunchDarklyClient _client; private readonly Context _context; + private readonly string _configKey; private readonly LdValue _trackData; - private readonly string _runId = Guid.NewGuid().ToString(); + private readonly string _runId; private readonly ILogger _logger; private bool _trackedDuration; @@ -46,17 +49,25 @@ public class LdAiConfigTracker : ILdAiConfigTracker /// the context /// public LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfig config, Context context) + : this(client, configKey, config, context, GenerateRunId()) + { + } + + internal LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfig config, Context context, + string runId) { Config = config ?? throw new ArgumentNullException(nameof(config)); _client = client ?? throw new ArgumentNullException(nameof(client)); _context = context; + _configKey = configKey ?? throw new ArgumentNullException(nameof(configKey)); + _runId = runId ?? throw new ArgumentNullException(nameof(runId)); _logger = client.GetLogger(); _trackData = LdValue.ObjectFrom(new Dictionary { { "runId", LdValue.Of(_runId) }, { "variationKey", LdValue.Of(config.VariationKey)}, { "version", LdValue.Of(config.Version)}, - { "configKey" , LdValue.Of(configKey ?? throw new ArgumentNullException(nameof(configKey))) }, + { "configKey" , LdValue.Of(_configKey) }, { "modelName", LdValue.Of(config.Model?.Name) }, { "providerName", LdValue.Of(config.Provider?.Name) }, }); @@ -66,6 +77,23 @@ public LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfi /// public LdAiConfig Config { get; } + /// + public string ResumptionToken + { + get + { + var json = JsonSerializer.Serialize(new + { + runId = _runId, + configKey = _configKey, + variationKey = Config.VariationKey, + version = Config.Version, + }); + var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(json)); + return base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); + } + } + /// public void TrackDuration(float durationMs) { @@ -200,4 +228,17 @@ public void TrackTokens(Usage usage) _client.Track(TokenOutput, _context, _trackData, usage.Output.Value); } } + + /// + public ILdAiConfigTracker CreateTracker() + { + if (!Config.Enabled) + { + return null; + } + var runId = GenerateRunId(); + return new LdAiConfigTracker(_client, _configKey, Config, _context, runId); + } + + private static string GenerateRunId() => Guid.NewGuid().ToString(); } diff --git a/pkgs/sdk/server-ai/test/LdAiClientTest.cs b/pkgs/sdk/server-ai/test/LdAiClientTest.cs index f28e9c9d..0e52116e 100644 --- a/pkgs/sdk/server-ai/test/LdAiClientTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiClientTest.cs @@ -1,4 +1,7 @@ +using System; using System.Collections.Generic; +using System.Text; +using System.Text.Json; using LaunchDarkly.Logging; using LaunchDarkly.Sdk.Server.Ai.Adapters; using LaunchDarkly.Sdk.Server.Ai.Config; @@ -355,4 +358,153 @@ public void DisabledMethodReturnsNewInstanceEachCall() var second = LdAiConfig.Disabled; Assert.NotSame(first, second); } + + [Fact] + public void CreateTrackerFromResumptionTokenRoundTrips() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var context = Context.New("user-key"); + const string configKey = "my-config"; + + mockClient.Setup(x => + x.JsonVariation(configKey, It.IsAny(), It.IsAny())).Returns( + LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("var-1"), + ["version"] = LdValue.Of(3) + }), + ["model"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("gpt-4") + }), + ["provider"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("openai") + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var originalTracker = client.CompletionConfig(configKey, context); + var token = originalTracker.ResumptionToken; + + // Reconstruct in a different context + var newContext = Context.New("other-user"); + var resumedTracker = client.CreateTracker(token, newContext); + + Assert.NotNull(resumedTracker); + + // Track on both and verify the resumed tracker uses the same runId + originalTracker.TrackDuration(100); + resumedTracker.TrackDuration(200); + + string originalRunId = null; + string resumedRunId = null; + + foreach (var call in mockClient.Invocations) + { + if (call.Method.Name == "Track" && (string)call.Arguments[0] == "$ld:ai:duration:total") + { + var data = (LdValue)call.Arguments[2]; + var runId = data.Get("runId").AsString; + if (originalRunId == null) originalRunId = runId; + else resumedRunId = runId; + } + } + + Assert.NotNull(originalRunId); + Assert.NotNull(resumedRunId); + Assert.Equal(originalRunId, resumedRunId); + } + + [Fact] + public void CreateTrackerFromResumptionTokenSetsEmptyModelAndProvider() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var context = Context.New("user-key"); + + // Build a token manually with known values + var payload = JsonSerializer.Serialize(new + { + runId = "test-run-id", + configKey = "test-key", + variationKey = "var-1", + version = 2, + }); + var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload)); + var token = base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); + + var client = new LdAiClient(mockClient.Object); + var tracker = client.CreateTracker(token, context); + + // Track something and verify the track data has empty model/provider + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("runId").AsString == "test-run-id" && + d.Get("configKey").AsString == "test-key" && + d.Get("variationKey").AsString == "var-1" && + d.Get("version").AsInt == 2 && + d.Get("modelName").AsString == "" && + d.Get("providerName").AsString == ""), + 1.0f), Times.Once); + } + + [Fact] + public void CreateTrackerFromInvalidTokenThrows() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var client = new LdAiClient(mockClient.Object); + var context = Context.New("user-key"); + + Assert.Throws(() => client.CreateTracker("not-valid-base64!!!", context)); + } + + [Fact] + public void CreateTrackerFromNullTokenThrows() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var client = new LdAiClient(mockClient.Object); + var context = Context.New("user-key"); + + Assert.Throws(() => client.CreateTracker(null, context)); + } + + [Fact] + public void CreateTrackerFromTokenMissingRunIdThrows() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var client = new LdAiClient(mockClient.Object); + var context = Context.New("user-key"); + + var payload = JsonSerializer.Serialize(new + { + configKey = "test-key", + variationKey = "var-1", + version = 1, + }); + var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload)); + var token = base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); + + Assert.Throws(() => client.CreateTracker(token, context)); + } } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index baad45f3..84a7e2d3 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -1,4 +1,7 @@ +using System; using System.Collections.Generic; +using System.Text; +using System.Text.Json; using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.Interfaces; @@ -146,7 +149,7 @@ public async void CanTrackDurationOfTask() mockClient.Verify( x => x.Track("$ld:ai:duration:total", context, It.Is(d => MatchesTrackData(d, flagKey, config)), - It.IsInRange(0, 500, Range.Inclusive)), Times.Once); + It.IsInRange(0, 500, Moq.Range.Inclusive)), Times.Once); } @@ -429,5 +432,184 @@ public void TrackSuccessAfterErrorIsIgnored() mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, It.IsAny(), It.IsAny()), Times.Never); } + + [Fact] + public void CreateTrackerReturnsNullWhenDisabled() + { + var mockClient = new Mock(); + var context = Context.New("key"); + var config = LdAiConfig.Disabled; + + var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); + + Assert.Null(tracker.CreateTracker()); + } + + [Fact] + public void CreateTrackerReturnsNewTrackerWhenEnabled() + { + var mockClient = new Mock(); + var context = Context.New("key"); + var config = LdAiConfig.New().Enable().Build(); + + var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); + var newTracker = tracker.CreateTracker(); + + Assert.NotNull(newTracker); + Assert.NotSame(tracker, newTracker); + Assert.Equal(tracker.Config, newTracker.Config); + } + + [Fact] + public void CreateTrackerReturnsTrackerWithFreshRunId() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.New().Enable().Build(); + + var tracker1 = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker2 = tracker1.CreateTracker(); + + tracker1.TrackDuration(10.0f); + tracker2.TrackDuration(20.0f); + + string runId1 = null; + string runId2 = null; + + foreach (var call in mockClient.Invocations) + { + if (call.Method.Name == "Track" && (string)call.Arguments[0] == "$ld:ai:duration:total") + { + var data = (LdValue)call.Arguments[2]; + var runId = data.Get("runId").AsString; + if (runId1 == null) runId1 = runId; + else runId2 = runId; + } + } + + Assert.NotNull(runId1); + Assert.NotNull(runId2); + Assert.NotEqual(runId1, runId2); + } + + [Fact] + public void CreateTrackerReturnsTrackerWithIndependentTrackingState() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.New().Enable().Build(); + + var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + // Track duration on the original tracker (exhausts at-most-once) + tracker.TrackDuration(1.0f); + + // Create a new tracker - it should have fresh tracking state + var newTracker = tracker.CreateTracker(); + newTracker.TrackDuration(2.0f); + + // Both calls should have gone through + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.IsAny(), 1.0f), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, + It.IsAny(), 2.0f), Times.Once); + } + + [Fact] + public void CreateTrackerCanBeCalledMultipleTimes() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + var config = LdAiConfig.New().Enable().Build(); + + var original = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + + var tracker1 = original.CreateTracker(); + var tracker2 = original.CreateTracker(); + var tracker3 = original.CreateTracker(); + + Assert.NotNull(tracker1); + Assert.NotNull(tracker2); + Assert.NotNull(tracker3); + + // Each tracker should be able to independently track success + tracker1.TrackSuccess(); + tracker2.TrackSuccess(); + tracker3.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.IsAny(), It.IsAny()), Times.Exactly(3)); + } + + [Fact] + public void ResumptionTokenContainsExpectedFields() + { + var mockClient = new Mock(); + var context = Context.New("key"); + var config = LdAiConfig.New() + .Enable() + .SetModelName("test-model") + .SetModelProviderName("test-provider") + .Build(); + + var tracker = new LdAiConfigTracker(mockClient.Object, "my-config-key", config, context); + var token = tracker.ResumptionToken; + + Assert.NotNull(token); + Assert.NotEmpty(token); + + // Decode and verify the payload + var base64 = token.Replace('-', '+').Replace('_', '/'); + switch (base64.Length % 4) + { + case 2: base64 += "=="; break; + case 3: base64 += "="; break; + } + var json = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + var doc = JsonDocument.Parse(json); + + Assert.Equal("my-config-key", doc.RootElement.GetProperty("configKey").GetString()); + Assert.Equal(config.VariationKey, doc.RootElement.GetProperty("variationKey").GetString()); + Assert.Equal(config.Version, doc.RootElement.GetProperty("version").GetInt32()); + Assert.True(doc.RootElement.GetProperty("runId").GetString().Length > 0); + + // modelName and providerName should NOT be in the token + Assert.False(doc.RootElement.TryGetProperty("modelName", out _)); + Assert.False(doc.RootElement.TryGetProperty("providerName", out _)); + } + + [Fact] + public void ResumptionTokenIsUrlSafeBase64() + { + var mockClient = new Mock(); + var context = Context.New("key"); + var config = LdAiConfig.New().Enable().Build(); + + var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); + var token = tracker.ResumptionToken; + + // URL-safe base64 should not contain +, /, or = + Assert.DoesNotContain("+", token); + Assert.DoesNotContain("/", token); + Assert.DoesNotContain("=", token); + } + + [Fact] + public void ResumptionTokenIsConsistentAcrossCalls() + { + var mockClient = new Mock(); + var context = Context.New("key"); + var config = LdAiConfig.New().Enable().Build(); + + var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); + + var token1 = tracker.ResumptionToken; + var token2 = tracker.ResumptionToken; + + Assert.Equal(token1, token2); + } } } From c60b42e3581f2618b82f15df7a0dba3634893b65 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 16:30:53 -0500 Subject: [PATCH 03/19] refactor: Replace bool tracking guards with nullable metric values Co-Authored-By: Claude Opus 4.6 --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 66fda808..9584e814 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -23,10 +23,10 @@ public class LdAiConfigTracker : ILdAiConfigTracker private readonly string _runId; private readonly ILogger _logger; - private bool _trackedDuration; - private bool _trackedTimeToFirstToken; - private bool _trackedTokens; - private bool _trackedFeedback; + private double? _durationMs; + private double? _timeToFirstTokenMs; + private Usage? _tokens; + private Feedback? _feedback; private bool? _trackedSuccess; private const string Duration = "$ld:ai:duration:total"; @@ -97,12 +97,12 @@ public string ResumptionToken /// public void TrackDuration(float durationMs) { - if (_trackedDuration) + if (_durationMs.HasValue) { _logger?.Warn("Duration has already been tracked for this operation."); return; } - _trackedDuration = true; + _durationMs = durationMs; _client.Track(Duration, _context, _trackData, durationMs); } @@ -122,24 +122,24 @@ public async Task TrackDurationOfTask(Task task) /// public void TrackTimeToFirstToken(float timeToFirstTokenMs) { - if (_trackedTimeToFirstToken) + if (_timeToFirstTokenMs.HasValue) { _logger?.Warn("Time to first token has already been tracked for this operation."); return; } - _trackedTimeToFirstToken = true; + _timeToFirstTokenMs = timeToFirstTokenMs; _client.Track(TimeToFirstToken, _context, _trackData, timeToFirstTokenMs); } /// public void TrackFeedback(Feedback feedback) { - if (_trackedFeedback) + if (_feedback.HasValue) { _logger?.Warn("Feedback has already been tracked for this operation."); return; } - _trackedFeedback = true; + _feedback = feedback; switch (feedback) { case Feedback.Positive: @@ -209,12 +209,12 @@ public async Task TrackRequest(Task request) /// public void TrackTokens(Usage usage) { - if (_trackedTokens) + if (_tokens.HasValue) { _logger?.Warn("Tokens have already been tracked for this operation."); return; } - _trackedTokens = true; + _tokens = usage; if (usage.Total is > 0) { _client.Track(TokenTotal, _context, _trackData, usage.Total.Value); From 4b43399674b30720ecc54d2872de9cd8f94f63fa Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 16:58:53 -0500 Subject: [PATCH 04/19] fix: Align constructor param order with spec and omit empty variationKey from resumption token Co-Authored-By: Claude Opus 4.6 --- pkgs/sdk/server-ai/src/LdAiClient.cs | 11 +--- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 65 +++++++++++++------ pkgs/sdk/server-ai/test/LdAiClientTest.cs | 36 ++++++++++ .../server-ai/test/LdAiConfigTrackerTest.cs | 43 +++++++++++- 4 files changed, 124 insertions(+), 31 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiClient.cs b/pkgs/sdk/server-ai/src/LdAiClient.cs index d5374878..1eb62907 100644 --- a/pkgs/sdk/server-ai/src/LdAiClient.cs +++ b/pkgs/sdk/server-ai/src/LdAiClient.cs @@ -168,15 +168,8 @@ public ILdAiConfigTracker CreateTracker(string resumptionToken, Context context) nameof(resumptionToken)); } - var config = new LdAiConfig( - true, - null, - new Meta { VariationKey = payload.VariationKey ?? "", Version = payload.Version }, - null, - null - ); - - return new LdAiConfigTracker(_client, payload.ConfigKey, config, context, payload.RunId); + return new LdAiConfigTracker(_client, payload.RunId, payload.ConfigKey, + payload.VariationKey, payload.Version, context, "", ""); } private class ResumptionPayload diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 9584e814..9aa34a16 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -6,6 +6,7 @@ using System.Text.Json; using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; +using LaunchDarkly.Sdk.Server.Ai.DataModel; using LaunchDarkly.Sdk.Server.Ai.Interfaces; using LaunchDarkly.Sdk.Server.Ai.Tracking; @@ -17,10 +18,14 @@ namespace LaunchDarkly.Sdk.Server.Ai; public class LdAiConfigTracker : ILdAiConfigTracker { private readonly ILaunchDarklyClient _client; - private readonly Context _context; + private readonly string _runId; private readonly string _configKey; + private readonly string _variationKey; + private readonly int _version; + private readonly Context _context; + private readonly string _modelName; + private readonly string _providerName; private readonly LdValue _trackData; - private readonly string _runId; private readonly ILogger _logger; private double? _durationMs; @@ -49,27 +54,39 @@ public class LdAiConfigTracker : ILdAiConfigTracker /// the context /// public LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfig config, Context context) - : this(client, configKey, config, context, GenerateRunId()) + : this(client, GenerateRunId(), configKey, + (config ?? throw new ArgumentNullException(nameof(config))).VariationKey, + config.Version, context, config.Model?.Name ?? "", config.Provider?.Name ?? "", config) { } - internal LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfig config, Context context, - string runId) + internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string configKey, + string variationKey, int version, Context context, string modelName, string providerName, + LdAiConfig config = null) { - Config = config ?? throw new ArgumentNullException(nameof(config)); _client = client ?? throw new ArgumentNullException(nameof(client)); - _context = context; - _configKey = configKey ?? throw new ArgumentNullException(nameof(configKey)); _runId = runId ?? throw new ArgumentNullException(nameof(runId)); + _configKey = configKey ?? throw new ArgumentNullException(nameof(configKey)); + _variationKey = variationKey; + _version = version; + _context = context; + _modelName = modelName ?? ""; + _providerName = providerName ?? ""; _logger = client.GetLogger(); - _trackData = LdValue.ObjectFrom(new Dictionary + + Config = config ?? new LdAiConfig(true, null, + new Meta { VariationKey = _variationKey ?? "", Version = _version }, + new Model { Name = _modelName }, + new Provider { Name = _providerName }); + + _trackData = LdValue.ObjectFrom(new Dictionary { { "runId", LdValue.Of(_runId) }, - { "variationKey", LdValue.Of(config.VariationKey)}, - { "version", LdValue.Of(config.Version)}, - { "configKey" , LdValue.Of(_configKey) }, - { "modelName", LdValue.Of(config.Model?.Name) }, - { "providerName", LdValue.Of(config.Provider?.Name) }, + { "variationKey", LdValue.Of(_variationKey) }, + { "version", LdValue.Of(_version) }, + { "configKey", LdValue.Of(_configKey) }, + { "modelName", LdValue.Of(_modelName) }, + { "providerName", LdValue.Of(_providerName) }, }); } @@ -82,13 +99,18 @@ public string ResumptionToken { get { - var json = JsonSerializer.Serialize(new + var dict = new Dictionary { - runId = _runId, - configKey = _configKey, - variationKey = Config.VariationKey, - version = Config.Version, - }); + { "runId", _runId }, + { "configKey", _configKey }, + }; + if (!string.IsNullOrEmpty(_variationKey)) + { + dict.Add("variationKey", _variationKey); + } + dict.Add("version", _version); + + var json = JsonSerializer.Serialize(dict); var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(json)); return base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); } @@ -237,7 +259,8 @@ public ILdAiConfigTracker CreateTracker() return null; } var runId = GenerateRunId(); - return new LdAiConfigTracker(_client, _configKey, Config, _context, runId); + return new LdAiConfigTracker(_client, runId, _configKey, + _variationKey, _version, _context, _modelName, _providerName, Config); } private static string GenerateRunId() => Guid.NewGuid().ToString(); diff --git a/pkgs/sdk/server-ai/test/LdAiClientTest.cs b/pkgs/sdk/server-ai/test/LdAiClientTest.cs index 0e52116e..8b663bf1 100644 --- a/pkgs/sdk/server-ai/test/LdAiClientTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiClientTest.cs @@ -507,4 +507,40 @@ public void CreateTrackerFromTokenMissingRunIdThrows() Assert.Throws(() => client.CreateTracker(token, context)); } + + [Fact] + public void CreateTrackerFromTokenWithoutVariationKeyHandlesAbsence() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var context = Context.New("user-key"); + + // Token without variationKey + var payload = JsonSerializer.Serialize(new + { + runId = "test-run-id", + configKey = "test-key", + version = 3, + }); + var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(payload)); + var token = base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); + + var client = new LdAiClient(mockClient.Object); + var tracker = client.CreateTracker(token, context); + + Assert.NotNull(tracker); + + // Track and verify variationKey is null in the track data + tracker.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.Is(d => + d.Get("runId").AsString == "test-run-id" && + d.Get("configKey").AsString == "test-key" && + d.Get("variationKey").IsNull && + d.Get("version").AsInt == 3), + 1.0f), Times.Once); + } } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 84a7e2d3..00e0d2a5 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -572,10 +572,12 @@ public void ResumptionTokenContainsExpectedFields() var doc = JsonDocument.Parse(json); Assert.Equal("my-config-key", doc.RootElement.GetProperty("configKey").GetString()); - Assert.Equal(config.VariationKey, doc.RootElement.GetProperty("variationKey").GetString()); Assert.Equal(config.Version, doc.RootElement.GetProperty("version").GetInt32()); Assert.True(doc.RootElement.GetProperty("runId").GetString().Length > 0); + // variationKey is empty for builder-created configs, so it should be omitted + Assert.False(doc.RootElement.TryGetProperty("variationKey", out _)); + // modelName and providerName should NOT be in the token Assert.False(doc.RootElement.TryGetProperty("modelName", out _)); Assert.False(doc.RootElement.TryGetProperty("providerName", out _)); @@ -611,5 +613,44 @@ public void ResumptionTokenIsConsistentAcrossCalls() Assert.Equal(token1, token2); } + + [Fact] + public void ResumptionTokenIncludesVariationKeyWhenPresent() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + var context = Context.New("key"); + + // Use the LdAiClient to get a config with a real variationKey from flag evaluation + mockClient.Setup(x => + x.JsonVariation("key", It.IsAny(), It.IsAny())).Returns( + LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("my-variation"), + ["version"] = LdValue.Of(5) + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var tracker = client.CompletionConfig("key", context); + var token = tracker.ResumptionToken; + + // Decode and verify variationKey is present + var base64 = token.Replace('-', '+').Replace('_', '/'); + switch (base64.Length % 4) + { + case 2: base64 += "=="; break; + case 3: base64 += "="; break; + } + var json = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + var doc = JsonDocument.Parse(json); + + Assert.Equal("my-variation", doc.RootElement.GetProperty("variationKey").GetString()); + } } } From bd2aa398bda3a0254bec61cbfe8ae475dc377bfb Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 17:06:28 -0500 Subject: [PATCH 05/19] fix: Include track data in at-most-once warning log messages Co-Authored-By: Claude Opus 4.6 --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 9aa34a16..3262815b 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -121,7 +121,7 @@ public void TrackDuration(float durationMs) { if (_durationMs.HasValue) { - _logger?.Warn("Duration has already been tracked for this operation."); + _logger?.Warn("Duration has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); return; } _durationMs = durationMs; @@ -146,7 +146,7 @@ public void TrackTimeToFirstToken(float timeToFirstTokenMs) { if (_timeToFirstTokenMs.HasValue) { - _logger?.Warn("Time to first token has already been tracked for this operation."); + _logger?.Warn("Time to first token has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); return; } _timeToFirstTokenMs = timeToFirstTokenMs; @@ -158,7 +158,7 @@ public void TrackFeedback(Feedback feedback) { if (_feedback.HasValue) { - _logger?.Warn("Feedback has already been tracked for this operation."); + _logger?.Warn("Feedback has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); return; } _feedback = feedback; @@ -180,7 +180,7 @@ public void TrackSuccess() { if (_trackedSuccess.HasValue) { - _logger?.Warn("Generation result has already been tracked for this operation."); + _logger?.Warn("Generation result has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); return; } _trackedSuccess = true; @@ -192,7 +192,7 @@ public void TrackError() { if (_trackedSuccess.HasValue) { - _logger?.Warn("Generation result has already been tracked for this operation."); + _logger?.Warn("Generation result has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); return; } _trackedSuccess = false; @@ -233,7 +233,7 @@ public void TrackTokens(Usage usage) { if (_tokens.HasValue) { - _logger?.Warn("Tokens have already been tracked for this operation."); + _logger?.Warn("Tokens have already been tracked for this operation. [{0}]", _trackData.ToJsonString()); return; } _tokens = usage; From 25477849eaa95cfd8dffc95852229b357a6de2ad Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 17:10:02 -0500 Subject: [PATCH 06/19] refactor: Move resumption token decoding to LdAiConfigTracker.FromResumptionToken static method Co-Authored-By: Claude Opus 4.6 --- pkgs/sdk/server-ai/src/LdAiClient.cs | 46 +------------- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 63 +++++++++++++++++++ .../server-ai/test/LdAiConfigTrackerTest.cs | 49 +++++++++++++++ 3 files changed, 113 insertions(+), 45 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiClient.cs b/pkgs/sdk/server-ai/src/LdAiClient.cs index 1eb62907..72393226 100644 --- a/pkgs/sdk/server-ai/src/LdAiClient.cs +++ b/pkgs/sdk/server-ai/src/LdAiClient.cs @@ -2,9 +2,7 @@ using System.Collections.Generic; using System.Collections.Immutable; using System.Linq; -using System.Text; using System.Text.Json; -using System.Text.Json.Serialization; using LaunchDarkly.Sdk.Server.Ai.Adapters; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.DataModel; @@ -142,49 +140,7 @@ public ILdAiConfigTracker Config(string key, Context context, LdAiConfig default /// public ILdAiConfigTracker CreateTracker(string resumptionToken, Context context) { - if (resumptionToken == null) throw new ArgumentNullException(nameof(resumptionToken)); - - var base64 = resumptionToken.Replace('-', '+').Replace('_', '/'); - switch (base64.Length % 4) - { - case 2: base64 += "=="; break; - case 3: base64 += "="; break; - } - - ResumptionPayload payload; - try - { - var json = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); - payload = JsonSerializer.Deserialize(json); - } - catch (Exception e) when (e is FormatException || e is JsonException) - { - throw new ArgumentException("Invalid resumption token", nameof(resumptionToken), e); - } - - if (string.IsNullOrEmpty(payload.RunId) || string.IsNullOrEmpty(payload.ConfigKey)) - { - throw new ArgumentException("Resumption token is missing required fields (runId, configKey)", - nameof(resumptionToken)); - } - - return new LdAiConfigTracker(_client, payload.RunId, payload.ConfigKey, - payload.VariationKey, payload.Version, context, "", ""); - } - - private class ResumptionPayload - { - [JsonPropertyName("runId")] - public string RunId { get; set; } - - [JsonPropertyName("configKey")] - public string ConfigKey { get; set; } - - [JsonPropertyName("variationKey")] - public string VariationKey { get; set; } - - [JsonPropertyName("version")] - public int Version { get; set; } + return LdAiConfigTracker.FromResumptionToken(resumptionToken, _client, context); } diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 3262815b..da287032 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -4,6 +4,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.DataModel; @@ -263,5 +264,67 @@ public ILdAiConfigTracker CreateTracker() _variationKey, _version, _context, _modelName, _providerName, Config); } + /// + /// Reconstructs a tracker from a resumption token. This enables cross-process scenarios + /// such as deferred feedback, where a tracker's runId needs to be reused in a different + /// process or at a later time. + /// + /// The reconstructed tracker will have empty model and provider names, as these are not + /// included in the resumption token. + /// + /// the resumption token obtained from + /// the LaunchDarkly client + /// the context to use for track events + /// a new tracker associated with the original run + /// thrown if token or client is null + /// thrown if the token is malformed or missing required fields + public static LdAiConfigTracker FromResumptionToken(string token, ILaunchDarklyClient client, Context context) + { + if (token == null) throw new ArgumentNullException(nameof(token)); + if (client == null) throw new ArgumentNullException(nameof(client)); + + var base64 = token.Replace('-', '+').Replace('_', '/'); + switch (base64.Length % 4) + { + case 2: base64 += "=="; break; + case 3: base64 += "="; break; + } + + ResumptionPayload payload; + try + { + var json = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + payload = JsonSerializer.Deserialize(json); + } + catch (Exception e) when (e is FormatException || e is JsonException) + { + throw new ArgumentException("Invalid resumption token", nameof(token), e); + } + + if (string.IsNullOrEmpty(payload.RunId) || string.IsNullOrEmpty(payload.ConfigKey)) + { + throw new ArgumentException("Resumption token is missing required fields (runId, configKey)", + nameof(token)); + } + + return new LdAiConfigTracker(client, payload.RunId, payload.ConfigKey, + payload.VariationKey, payload.Version, context, "", ""); + } + private static string GenerateRunId() => Guid.NewGuid().ToString(); + + private class ResumptionPayload + { + [JsonPropertyName("runId")] + public string RunId { get; set; } + + [JsonPropertyName("configKey")] + public string ConfigKey { get; set; } + + [JsonPropertyName("variationKey")] + public string VariationKey { get; set; } + + [JsonPropertyName("version")] + public int Version { get; set; } + } } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 00e0d2a5..95641769 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -652,5 +652,54 @@ public void ResumptionTokenIncludesVariationKeyWhenPresent() Assert.Equal("my-variation", doc.RootElement.GetProperty("variationKey").GetString()); } + + [Fact] + public void FromResumptionTokenReconstructsTrackerWithOriginalRunId() + { + var mockClient = new Mock(); + var context = Context.New("key"); + var config = LdAiConfig.New().Enable().Build(); + + var original = new LdAiConfigTracker(mockClient.Object, "my-key", config, context); + var token = original.ResumptionToken; + + var newContext = Context.New("other-key"); + var resumed = LdAiConfigTracker.FromResumptionToken(token, mockClient.Object, newContext); + + // Both should track with the same runId + original.TrackDuration(10); + resumed.TrackDuration(20); + + string originalRunId = null; + string resumedRunId = null; + foreach (var call in mockClient.Invocations) + { + if (call.Method.Name == "Track" && (string)call.Arguments[0] == "$ld:ai:duration:total") + { + var data = (LdValue)call.Arguments[2]; + if (originalRunId == null) originalRunId = data.Get("runId").AsString; + else resumedRunId = data.Get("runId").AsString; + } + } + + Assert.NotNull(originalRunId); + Assert.Equal(originalRunId, resumedRunId); + } + + [Fact] + public void FromResumptionTokenThrowsOnNullToken() + { + var mockClient = new Mock(); + Assert.Throws(() => + LdAiConfigTracker.FromResumptionToken(null, mockClient.Object, Context.New("key"))); + } + + [Fact] + public void FromResumptionTokenThrowsOnInvalidToken() + { + var mockClient = new Mock(); + Assert.Throws(() => + LdAiConfigTracker.FromResumptionToken("not-valid!!!", mockClient.Object, Context.New("key"))); + } } } From 23f93d8e33aef02b329d488b1384faba955f5509 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 17:20:37 -0500 Subject: [PATCH 07/19] fix: Omit variationKey from track data when null or empty Matches the existing behavior in ResumptionToken where variationKey is conditionally included. Updates MatchesTrackData test helper to handle absence of variationKey for configs with empty variation keys. Co-Authored-By: Claude Opus 4.6 --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 12 ++++++++---- pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index da287032..ffd0ca52 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -80,15 +80,19 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf new Model { Name = _modelName }, new Provider { Name = _providerName }); - _trackData = LdValue.ObjectFrom(new Dictionary + var trackDataBuilder = new Dictionary { { "runId", LdValue.Of(_runId) }, - { "variationKey", LdValue.Of(_variationKey) }, - { "version", LdValue.Of(_version) }, { "configKey", LdValue.Of(_configKey) }, + { "version", LdValue.Of(_version) }, { "modelName", LdValue.Of(_modelName) }, { "providerName", LdValue.Of(_providerName) }, - }); + }; + if (!string.IsNullOrEmpty(_variationKey)) + { + trackDataBuilder.Add("variationKey", LdValue.Of(_variationKey)); + } + _trackData = LdValue.ObjectFrom(trackDataBuilder); } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 95641769..4e37118a 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -15,7 +15,11 @@ public class LdAiTrackerTest { private static bool MatchesTrackData(LdValue actual, string flagKey, LdAiConfig config) { - return actual.Get("variationKey").Equals(LdValue.Of(config.VariationKey)) && + var variationKeyMatch = string.IsNullOrEmpty(config.VariationKey) + ? actual.Get("variationKey").IsNull + : actual.Get("variationKey").Equals(LdValue.Of(config.VariationKey)); + + return variationKeyMatch && actual.Get("version").Equals(LdValue.Of(config.Version)) && actual.Get("configKey").Equals(LdValue.Of(flagKey)) && actual.Get("modelName").Equals(LdValue.Of(config.Model.Name)) && From 6a17dd5fed6e9383ca83286d13393b564642897e Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 09:30:45 -0500 Subject: [PATCH 08/19] fix: CreateTracker always returns a tracker, even for disabled configs Per AICONF spec 1.2.7.1, CreateTracker() must always return a new tracker instance. Removed the null-return guard for disabled configs. Co-Authored-By: Claude Opus 4.6 --- pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs | 4 +--- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 4 ---- pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs | 7 +++++-- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index f99f3cd1..8e60d5f4 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -114,9 +114,7 @@ public interface ILdAiConfigTracker /// Creates a new tracker with a fresh runId for tracking a new execution. /// Each call returns a tracker with independent at-most-once tracking state, /// allowing multiple executions to be tracked separately against the same config. - /// - /// Returns null if the config is disabled. /// - /// a new tracker, or null if the config is disabled + /// a new tracker public ILdAiConfigTracker CreateTracker(); } diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index ffd0ca52..7b0dbe21 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -259,10 +259,6 @@ public void TrackTokens(Usage usage) /// public ILdAiConfigTracker CreateTracker() { - if (!Config.Enabled) - { - return null; - } var runId = GenerateRunId(); return new LdAiConfigTracker(_client, runId, _configKey, _variationKey, _version, _context, _modelName, _providerName, Config); diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 4e37118a..1f04ab33 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -438,15 +438,18 @@ public void TrackSuccessAfterErrorIsIgnored() } [Fact] - public void CreateTrackerReturnsNullWhenDisabled() + public void CreateTrackerReturnsTrackerWhenDisabled() { var mockClient = new Mock(); var context = Context.New("key"); var config = LdAiConfig.Disabled; var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); + var newTracker = tracker.CreateTracker(); - Assert.Null(tracker.CreateTracker()); + Assert.NotNull(newTracker); + Assert.NotSame(tracker, newTracker); + Assert.Equal(tracker.Config, newTracker.Config); } [Fact] From 465b3b184da5abb53ff3b3cf8b874d46f81f3114 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 14:46:40 -0500 Subject: [PATCH 09/19] refactor!: CompletionConfig returns LdAiConfig with CreateTracker factory Breaking change: CompletionConfig/Config now return LdAiConfig instead of ILdAiConfigTracker. The tracker factory is available as a CreateTracker property on the config, each invocation generates a fresh runId. - Move CreateTracker from ILdAiConfigTracker to LdAiConfig as a Func - Remove Config property from ILdAiConfigTracker/LdAiConfigTracker - Collapse tracker constructors to single public ctor requiring runId - Replace null-guard exceptions with safe defaults - Add MetricSummary record and Summary property on tracker - Fix ResumptionToken key ordering using Utf8JsonWriter Co-Authored-By: Claude Opus 4.6 --- pkgs/sdk/server-ai/src/Config/LdAiConfig.cs | 25 +- .../server-ai/src/Interfaces/ILdAiClient.cs | 13 +- .../src/Interfaces/ILdAiConfigTracker.cs | 20 +- pkgs/sdk/server-ai/src/LdAiClient.cs | 31 +- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 85 ++--- .../server-ai/src/Tracking/MetricSummary.cs | 17 + pkgs/sdk/server-ai/test/InterpolationTests.cs | 8 +- pkgs/sdk/server-ai/test/LdAiClientTest.cs | 213 +++++++++-- .../server-ai/test/LdAiConfigTrackerTest.cs | 351 +++++++----------- 9 files changed, 431 insertions(+), 332 deletions(-) create mode 100644 pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs index 7fe18702..4d98e460 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs @@ -1,6 +1,8 @@ +using System; using System.Collections.Generic; using System.Linq; using LaunchDarkly.Sdk.Server.Ai.DataModel; +using LaunchDarkly.Sdk.Server.Ai.Interfaces; namespace LaunchDarkly.Sdk.Server.Ai.Config; @@ -217,7 +219,8 @@ public LdAiConfig Build() /// public readonly ModelProvider Provider; - internal LdAiConfig(bool enabled, IEnumerable messages, Meta meta, Model model, Provider provider) + internal LdAiConfig(bool enabled, IEnumerable messages, Meta meta, Model model, Provider provider, + Func createTracker = null) { Model = new ModelConfiguration(model?.Name ?? "", model?.Parameters ?? new Dictionary(), model?.Custom ?? new Dictionary()); @@ -226,6 +229,18 @@ internal LdAiConfig(bool enabled, IEnumerable messages, Meta meta, Mode Version = meta?.Version ?? 1; Enabled = enabled; Provider = new ModelProvider(provider?.Name ?? ""); + CreateTracker = createTracker; + } + + internal LdAiConfig(LdAiConfig source, Func createTracker) + { + Model = source.Model; + Messages = source.Messages; + VariationKey = source.VariationKey; + Version = source.Version; + Enabled = source.Enabled; + Provider = source.Provider; + CreateTracker = createTracker; } internal LdValue ToLdValue() { @@ -278,6 +293,14 @@ internal LdValue ToLdValue() /// public int Version { get; } + /// + /// A factory that creates a new with a fresh runId. + /// Each invocation returns a tracker with independent at-most-once tracking state. + /// This property is set when the config is returned by . + /// It will be null for configs created directly via the builder. + /// + public Func CreateTracker { get; } + /// /// Convenient helper that returns a disabled LdAiConfig. /// diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs index 9e56bc26..526c7cbd 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiClient.cs @@ -12,8 +12,9 @@ public interface ILdAiClient /// /// Retrieves a LaunchDarkly AI Completion Config identified by the given key. The return value - /// is an , which makes the configuration available and - /// provides convenience methods for generating events related to model usage. + /// is an , which makes the configuration available and provides + /// a factory for generating trackers capable of + /// emitting events related to model usage. /// /// Any variables provided will be interpolated into the prompt's messages. /// Additionally, the current LaunchDarkly context will be available as 'ldctx' within @@ -25,8 +26,8 @@ public interface ILdAiClient /// the default config, if unable to retrieve from LaunchDarkly. When not provided, /// a disabled config is used as the fallback. /// the list of variables used when interpolating the prompt - /// an AI Completion Config tracker - public ILdAiConfigTracker CompletionConfig(string key, Context context, LdAiConfig defaultValue = null, + /// an AI Completion Config + public LdAiConfig CompletionConfig(string key, Context context, LdAiConfig defaultValue = null, IReadOnlyDictionary variables = null); /// @@ -36,9 +37,9 @@ public ILdAiConfigTracker CompletionConfig(string key, Context context, LdAiConf /// the context /// the default config, if unable to retrieve from LaunchDarkly /// the list of variables used when interpolating the prompt - /// an AI Completion Config tracker + /// an AI Completion Config [Obsolete("Use CompletionConfig instead.")] - public ILdAiConfigTracker Config(string key, Context context, LdAiConfig defaultValue = null, + public LdAiConfig Config(string key, Context context, LdAiConfig defaultValue = null, IReadOnlyDictionary variables = null); /// diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index 8e60d5f4..a4c6cb63 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -11,11 +11,6 @@ namespace LaunchDarkly.Sdk.Server.Ai.Interfaces; /// public interface ILdAiConfigTracker { - /// - /// The AI model configuration retrieved from LaunchDarkly, or a default value if unable to retrieve. - /// - public LdAiConfig Config { get; } - /// /// A URL-safe Base64-encoded token that can be used to reconstruct this tracker in a different /// process or at a later time. The token contains the runId, configKey, variationKey, and version. @@ -24,6 +19,11 @@ public interface ILdAiConfigTracker /// public string ResumptionToken { get; } + /// + /// A summary of the metrics tracked by this tracker. + /// + public MetricSummary Summary { get; } + /// /// Tracks a duration metric related to this config. For example, if a particular operation /// related to usage of the AI model takes 100ms, this can be tracked and made available in @@ -44,7 +44,7 @@ public interface ILdAiConfigTracker /// type of the task's result /// the task public Task TrackDurationOfTask(Task task); - + /// /// Tracks the time it takes for the first token to be generated. /// @@ -109,12 +109,4 @@ public interface ILdAiConfigTracker /// /// the token usage public void TrackTokens(Usage usage); - - /// - /// Creates a new tracker with a fresh runId for tracking a new execution. - /// Each call returns a tracker with independent at-most-once tracking state, - /// allowing multiple executions to be tracked separately against the same config. - /// - /// a new tracker - public ILdAiConfigTracker CreateTracker(); } diff --git a/pkgs/sdk/server-ai/src/LdAiClient.cs b/pkgs/sdk/server-ai/src/LdAiClient.cs index 72393226..18e70457 100644 --- a/pkgs/sdk/server-ai/src/LdAiClient.cs +++ b/pkgs/sdk/server-ai/src/LdAiClient.cs @@ -58,7 +58,7 @@ public LdAiClient(ILaunchDarklyClient client) private const string LdContextVariable = "ldctx"; /// - public ILdAiConfigTracker CompletionConfig(string key, Context context, LdAiConfig defaultValue = null, + public LdAiConfig CompletionConfig(string key, Context context, LdAiConfig defaultValue = null, IReadOnlyDictionary variables = null) { _client.Track(TrackUsageCompletionConfig, context, LdValue.Of(key), 1); @@ -72,7 +72,7 @@ public ILdAiConfigTracker CompletionConfig(string key, Context context, LdAiConf /// This allows higher-level SDK entry methods to track their own usage events without /// double-counting. /// - private ILdAiConfigTracker Evaluate(string key, Context context, LdAiConfig defaultValue, + private LdAiConfig Evaluate(string key, Context context, LdAiConfig defaultValue, IReadOnlyDictionary variables = null) { var result = _client.JsonVariation(key, context, defaultValue.ToLdValue()); @@ -80,8 +80,7 @@ private ILdAiConfigTracker Evaluate(string key, Context context, LdAiConfig defa var parsed = ParseConfig(result, key); if (parsed == null) { - // ParseConfig already does logging. - return new LdAiConfigTracker(_client, key, defaultValue, context); + return WithTrackerFactory(defaultValue, key, context); } var mergedVariables = new Dictionary { { LdContextVariable, GetAllAttributes(context) } }; @@ -113,12 +112,28 @@ private ILdAiConfigTracker Evaluate(string key, Context context, LdAiConfig defa { _logger.Error( $"AI model config prompt has malformed message at index {i}: {ex.Message} (returning default config, which will not contain interpolated prompt messages)"); - return new LdAiConfigTracker(_client, key, defaultValue, context); + return WithTrackerFactory(defaultValue, key, context); } } } - return new LdAiConfigTracker(_client, key, new LdAiConfig(parsed.Meta?.Enabled ?? false, prompt, parsed.Meta, parsed.Model, parsed.Provider), context); + var config = new LdAiConfig(parsed.Meta?.Enabled ?? false, prompt, parsed.Meta, parsed.Model, parsed.Provider); + return WithTrackerFactory(config, key, context); + } + + private LdAiConfig WithTrackerFactory(LdAiConfig config, string configKey, Context context) + { + var variationKey = config.VariationKey; + var version = config.Version; + var modelName = config.Model?.Name ?? ""; + var providerName = config.Provider?.Name ?? ""; + var client = _client; + + return new LdAiConfig(config, () => + { + var runId = Guid.NewGuid().ToString(); + return new LdAiConfigTracker(client, runId, configKey, variationKey, version, context, modelName, providerName); + }); } /// @@ -128,9 +143,9 @@ private ILdAiConfigTracker Evaluate(string key, Context context, LdAiConfig defa /// the context /// the default config, if unable to retrieve from LaunchDarkly /// the list of variables used when interpolating the prompt - /// an AI Completion Config tracker + /// an AI Completion Config [Obsolete("Use CompletionConfig instead.")] - public ILdAiConfigTracker Config(string key, Context context, LdAiConfig defaultValue = null, + public LdAiConfig Config(string key, Context context, LdAiConfig defaultValue = null, IReadOnlyDictionary variables = null) { return CompletionConfig(key, context, defaultValue, variables); diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 7b0dbe21..265627ac 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -1,13 +1,12 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Runtime.CompilerServices; +using System.IO; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; -using LaunchDarkly.Sdk.Server.Ai.DataModel; using LaunchDarkly.Sdk.Server.Ai.Interfaces; using LaunchDarkly.Sdk.Server.Ai.Tracking; @@ -46,39 +45,29 @@ public class LdAiConfigTracker : ILdAiConfigTracker private const string TimeToFirstToken = "$ld:ai:tokens:ttf"; /// - /// Constructs a new AI Config tracker. The tracker is associated with a configuration, - /// a context, and a key which identifies the configuration. + /// Constructs a new AI Config tracker. The tracker is associated with a run, a config key, + /// and a context. The runId should be a unique identifier (UUID v4) for each execution. /// /// the LaunchDarkly client + /// the unique run identifier /// key of the AI Config - /// the AI Config + /// the variation key + /// the config version /// the context - /// - public LdAiConfigTracker(ILaunchDarklyClient client, string configKey, LdAiConfig config, Context context) - : this(client, GenerateRunId(), configKey, - (config ?? throw new ArgumentNullException(nameof(config))).VariationKey, - config.Version, context, config.Model?.Name ?? "", config.Provider?.Name ?? "", config) + /// the model name + /// the provider name + public LdAiConfigTracker(ILaunchDarklyClient client, string runId, string configKey, + string variationKey, int version, Context context, string modelName, string providerName) { - } - - internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string configKey, - string variationKey, int version, Context context, string modelName, string providerName, - LdAiConfig config = null) - { - _client = client ?? throw new ArgumentNullException(nameof(client)); - _runId = runId ?? throw new ArgumentNullException(nameof(runId)); - _configKey = configKey ?? throw new ArgumentNullException(nameof(configKey)); + _client = client; + _runId = runId ?? ""; + _configKey = configKey ?? ""; _variationKey = variationKey; _version = version; _context = context; _modelName = modelName ?? ""; _providerName = providerName ?? ""; - _logger = client.GetLogger(); - - Config = config ?? new LdAiConfig(true, null, - new Meta { VariationKey = _variationKey ?? "", Version = _version }, - new Model { Name = _modelName }, - new Provider { Name = _providerName }); + _logger = client?.GetLogger(); var trackDataBuilder = new Dictionary { @@ -95,32 +84,38 @@ internal LdAiConfigTracker(ILaunchDarklyClient client, string runId, string conf _trackData = LdValue.ObjectFrom(trackDataBuilder); } - - /// - public LdAiConfig Config { get; } - /// public string ResumptionToken { get { - var dict = new Dictionary - { - { "runId", _runId }, - { "configKey", _configKey }, - }; - if (!string.IsNullOrEmpty(_variationKey)) + using var stream = new MemoryStream(); + using (var writer = new Utf8JsonWriter(stream)) { - dict.Add("variationKey", _variationKey); + writer.WriteStartObject(); + writer.WriteString("runId", _runId); + writer.WriteString("configKey", _configKey); + if (!string.IsNullOrEmpty(_variationKey)) + { + writer.WriteString("variationKey", _variationKey); + } + writer.WriteNumber("version", _version); + writer.WriteEndObject(); } - dict.Add("version", _version); - - var json = JsonSerializer.Serialize(dict); - var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(json)); + var base64 = Convert.ToBase64String(stream.ToArray()); return base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); } } + /// + public MetricSummary Summary => new MetricSummary( + _durationMs, + _feedback, + _tokens, + _trackedSuccess, + _timeToFirstTokenMs + ); + /// public void TrackDuration(float durationMs) { @@ -256,14 +251,6 @@ public void TrackTokens(Usage usage) } } - /// - public ILdAiConfigTracker CreateTracker() - { - var runId = GenerateRunId(); - return new LdAiConfigTracker(_client, runId, _configKey, - _variationKey, _version, _context, _modelName, _providerName, Config); - } - /// /// Reconstructs a tracker from a resumption token. This enables cross-process scenarios /// such as deferred feedback, where a tracker's runId needs to be reused in a different @@ -311,8 +298,6 @@ public static LdAiConfigTracker FromResumptionToken(string token, ILaunchDarklyC payload.VariationKey, payload.Version, context, "", ""); } - private static string GenerateRunId() => Guid.NewGuid().ToString(); - private class ResumptionPayload { [JsonPropertyName("runId")] diff --git a/pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs b/pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs new file mode 100644 index 00000000..bfe79cf2 --- /dev/null +++ b/pkgs/sdk/server-ai/src/Tracking/MetricSummary.cs @@ -0,0 +1,17 @@ +namespace LaunchDarkly.Sdk.Server.Ai.Tracking; + +/// +/// A summary of the metrics tracked by a tracker. +/// +/// the duration in milliseconds +/// the feedback sentiment +/// the token usage +/// whether the generation was successful +/// the time to first token in milliseconds +public record struct MetricSummary( + double? DurationMs, + Feedback? Feedback, + Usage? Tokens, + bool? Success, + double? TimeToFirstTokenMs +); diff --git a/pkgs/sdk/server-ai/test/InterpolationTests.cs b/pkgs/sdk/server-ai/test/InterpolationTests.cs index 366f093e..a9f724d5 100644 --- a/pkgs/sdk/server-ai/test/InterpolationTests.cs +++ b/pkgs/sdk/server-ai/test/InterpolationTests.cs @@ -39,9 +39,9 @@ private string Eval(string prompt, Context context, IReadOnlyDictionary x.GetLogger()).Returns(mockLogger.Object); var client = new LdAiClient(mockClient.Object); - var tracker = client.Config("foo", context, LdAiConfig.Disabled, variables); + var config = client.Config("foo", context, LdAiConfig.Disabled, variables); - return tracker.Config.Messages[0].Content; + return config.Messages[0].Content; } [Theory] @@ -146,8 +146,8 @@ public void TestInterpolationMalformed() mockLogger.Setup(x => x.Error(It.IsAny())); var client = new LdAiClient(mockClient.Object); - var tracker = client.Config("foo", Context.New("key"), LdAiConfig.Disabled); - Assert.False(tracker.Config.Enabled); + var config = client.Config("foo", Context.New("key"), LdAiConfig.Disabled); + Assert.False(config.Enabled); } [Fact] diff --git a/pkgs/sdk/server-ai/test/LdAiClientTest.cs b/pkgs/sdk/server-ai/test/LdAiClientTest.cs index 8b663bf1..e08b28c1 100644 --- a/pkgs/sdk/server-ai/test/LdAiClientTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiClientTest.cs @@ -19,8 +19,8 @@ public void CanInstantiateWithServerSideClient() { var client = new LdClientAdapter(new LdClient(Configuration.Builder("key").Offline(true).Build())); var aiClient = new LdAiClient(client); - var result= aiClient.CompletionConfig("foo", Context.New("key"), LdAiConfig.Disabled); - Assert.False(result.Config.Enabled); + var result = aiClient.CompletionConfig("foo", Context.New("key"), LdAiConfig.Disabled); + Assert.False(result.Enabled); } [Fact] @@ -47,9 +47,10 @@ public void ReturnsDefaultConfigWhenGivenInvalidVariation() var defaultConfig = LdAiConfig.New().AddMessage("Hello").Build(); - var tracker = client.CompletionConfig("foo", Context.New(ContextKind.Default, "key"), defaultConfig); + var config = client.CompletionConfig("foo", Context.New(ContextKind.Default, "key"), defaultConfig); - Assert.Equal(defaultConfig, tracker.Config); + Assert.Equal(defaultConfig.Enabled, config.Enabled); + Assert.Equal(defaultConfig.Messages, config.Messages); } [Fact] @@ -88,7 +89,7 @@ public void CompletionConfigMethodCallsTrackWithCorrectParameters() var client = new LdAiClient(mockClient.Object); var defaultConfig = LdAiConfig.New().Build(); - var tracker = client.CompletionConfig(configKey, context, defaultConfig); + var config = client.CompletionConfig(configKey, context, defaultConfig); mockClient.Verify(c => c.Track( "$ld:ai:usage:completion-config", @@ -96,7 +97,7 @@ public void CompletionConfigMethodCallsTrackWithCorrectParameters() LdValue.Of(configKey), 1), Times.Once); - Assert.NotNull(tracker); + Assert.NotNull(config); } [Fact] @@ -168,10 +169,10 @@ public void ConfigNotEnabledReturnsDisabledInstance(string json) // All the JSON inputs here are considered disabled, either due to lack of the 'enabled' property, // or if present, it is set to false. Therefore, if the default was returned, we'd see the assertion fail // (since calling LdAiConfig.New() constructs an enabled config by default.) - var tracker = client.CompletionConfig("foo", Context.New(ContextKind.Default, "key"), + var config = client.CompletionConfig("foo", Context.New(ContextKind.Default, "key"), LdAiConfig.New().AddMessage("foo").Build()); - Assert.False(tracker.Config.Enabled); + Assert.False(config.Enabled); } [Fact] @@ -188,7 +189,7 @@ public void CanSetAllDefaultValueFields() var client = new LdAiClient(mockClient.Object); - var tracker = client.CompletionConfig("foo", Context.New(ContextKind.Default, "key"), + var config = client.CompletionConfig("foo", Context.New(ContextKind.Default, "key"), LdAiConfig.New(). AddMessage("foo"). SetModelParam("foo", LdValue.Of("bar")). @@ -197,17 +198,17 @@ public void CanSetAllDefaultValueFields() SetModelProviderName("amazing-provider"). SetEnabled(true).Build()); - Assert.True(tracker.Config.Enabled); - Assert.Collection(tracker.Config.Messages, + Assert.True(config.Enabled); + Assert.Collection(config.Messages, message => { Assert.Equal("foo", message.Content); Assert.Equal(Role.User, message.Role); }); - Assert.Equal("amazing-provider", tracker.Config.Provider.Name); - Assert.Equal("bar", tracker.Config.Model.Parameters["foo"].AsString); - Assert.Equal("baz", tracker.Config.Model.Custom["foo"].AsString); - Assert.Equal("awesome-model", tracker.Config.Model.Name); + Assert.Equal("amazing-provider", config.Provider.Name); + Assert.Equal("bar", config.Model.Parameters["foo"].AsString); + Assert.Equal("baz", config.Model.Custom["foo"].AsString); + Assert.Equal("awesome-model", config.Model.Name); } [Fact] @@ -233,20 +234,20 @@ public void ConfigEnabledReturnsInstance() var client = new LdAiClient(mockClient.Object); // We shouldn't get this default. - var tracker = client.CompletionConfig("foo", context, + var config = client.CompletionConfig("foo", context, LdAiConfig.New().AddMessage("Goodbye!").Build()); - Assert.Collection(tracker.Config.Messages, + Assert.Collection(config.Messages, message => { Assert.Equal("Hello!", message.Content); Assert.Equal(Role.System, message.Role); }); - Assert.Equal("", tracker.Config.Provider.Name); - Assert.Equal("", tracker.Config.Model.Name); - Assert.Empty(tracker.Config.Model.Custom); - Assert.Empty(tracker.Config.Model.Parameters); + Assert.Equal("", config.Provider.Name); + Assert.Equal("", config.Model.Name); + Assert.Empty(config.Model.Custom); + Assert.Empty(config.Model.Parameters); } @@ -285,14 +286,14 @@ public void ModelParametersAreParsed() var client = new LdAiClient(mockClient.Object); // We shouldn't get this default. - var tracker = client.CompletionConfig("foo", context, + var config = client.CompletionConfig("foo", context, LdAiConfig.New().AddMessage("Goodbye!").Build()); - Assert.Equal("model-foo", tracker.Config.Model.Name); - Assert.Equal("bar", tracker.Config.Model.Parameters["foo"].AsString); - Assert.Equal(42, tracker.Config.Model.Parameters["baz"].AsInt); - Assert.Equal("baz", tracker.Config.Model.Custom["foo"].AsString); - Assert.Equal(43, tracker.Config.Model.Custom["baz"].AsInt); + Assert.Equal("model-foo", config.Model.Name); + Assert.Equal("bar", config.Model.Parameters["foo"].AsString); + Assert.Equal(42, config.Model.Parameters["baz"].AsInt); + Assert.Equal("baz", config.Model.Custom["foo"].AsString); + Assert.Equal(43, config.Model.Custom["baz"].AsInt); } [Fact] @@ -322,10 +323,10 @@ public void ProviderConfigIsParsed() var client = new LdAiClient(mockClient.Object); // We shouldn't get this default. - var tracker = client.CompletionConfig("foo", context, + var config = client.CompletionConfig("foo", context, LdAiConfig.New().AddMessage("Goodbye!").Build()); - Assert.Equal("amazing-provider", tracker.Config.Provider.Name); + Assert.Equal("amazing-provider", config.Provider.Name); } [Fact] @@ -339,9 +340,9 @@ public void ConfigWithoutDefaultValueUsesDisabledConfig() mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); var client = new LdAiClient(mockClient.Object); - var tracker = client.Config("foo", Context.New(ContextKind.Default, "key")); + var config = client.Config("foo", Context.New(ContextKind.Default, "key")); - Assert.False(tracker.Config.Enabled); + Assert.False(config.Enabled); } [Fact] @@ -359,6 +360,153 @@ public void DisabledMethodReturnsNewInstanceEachCall() Assert.NotSame(first, second); } + [Fact] + public void CompletionConfigReturnsConfigWithCreateTrackerFactory() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var context = Context.New("user-key"); + const string configKey = "my-config"; + + mockClient.Setup(x => + x.JsonVariation(configKey, It.IsAny(), It.IsAny())).Returns( + LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("var-1"), + ["version"] = LdValue.Of(3) + }), + ["model"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("gpt-4") + }), + ["provider"] = LdValue.ObjectFrom(new Dictionary + { + ["name"] = LdValue.Of("openai") + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var config = client.CompletionConfig(configKey, context); + + Assert.NotNull(config.CreateTracker); + + var tracker = config.CreateTracker(); + Assert.NotNull(tracker); + } + + [Fact] + public void CreateTrackerFactoryReturnsTrackersWithFreshRunIds() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var context = Context.New("user-key"); + const string configKey = "my-config"; + + mockClient.Setup(x => + x.JsonVariation(configKey, It.IsAny(), It.IsAny())).Returns( + LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("var-1"), + ["version"] = LdValue.Of(3) + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var config = client.CompletionConfig(configKey, context); + + var tracker1 = config.CreateTracker(); + var tracker2 = config.CreateTracker(); + + tracker1.TrackDuration(10.0f); + tracker2.TrackDuration(20.0f); + + string runId1 = null; + string runId2 = null; + + foreach (var call in mockClient.Invocations) + { + if (call.Method.Name == "Track" && (string)call.Arguments[0] == "$ld:ai:duration:total") + { + var data = (LdValue)call.Arguments[2]; + var runId = data.Get("runId").AsString; + if (runId1 == null) runId1 = runId; + else runId2 = runId; + } + } + + Assert.NotNull(runId1); + Assert.NotNull(runId2); + Assert.NotEqual(runId1, runId2); + } + + [Fact] + public void CreateTrackerFactoryReturnsTrackersWithIndependentState() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var context = Context.New("user-key"); + const string configKey = "my-config"; + + mockClient.Setup(x => + x.JsonVariation(configKey, It.IsAny(), It.IsAny())).Returns( + LdValue.ObjectFrom(new Dictionary + { + ["_ldMeta"] = LdValue.ObjectFrom(new Dictionary + { + ["enabled"] = LdValue.Of(true), + ["variationKey"] = LdValue.Of("var-1"), + ["version"] = LdValue.Of(3) + }), + ["messages"] = LdValue.ArrayOf() + })); + + var client = new LdAiClient(mockClient.Object); + var config = client.CompletionConfig(configKey, context); + + var tracker1 = config.CreateTracker(); + var tracker2 = config.CreateTracker(); + + // Both should be able to track independently + tracker1.TrackSuccess(); + tracker2.TrackSuccess(); + + mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, + It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + [Fact] + public void DefaultConfigGetsCreateTrackerFactory() + { + var mockClient = new Mock(); + var mockLogger = new Mock(); + + mockClient.Setup(x => + x.JsonVariation("foo", It.IsAny(), It.IsAny())).Returns(LdValue.Null); + mockClient.Setup(x => x.GetLogger()).Returns(mockLogger.Object); + + var client = new LdAiClient(mockClient.Object); + var config = client.CompletionConfig("foo", Context.New("key")); + + Assert.NotNull(config.CreateTracker); + + var tracker = config.CreateTracker(); + Assert.NotNull(tracker); + } + [Fact] public void CreateTrackerFromResumptionTokenRoundTrips() { @@ -391,7 +539,8 @@ public void CreateTrackerFromResumptionTokenRoundTrips() })); var client = new LdAiClient(mockClient.Object); - var originalTracker = client.CompletionConfig(configKey, context); + var config = client.CompletionConfig(configKey, context); + var originalTracker = config.CreateTracker(); var token = originalTracker.ResumptionToken; // Reconstruct in a different context diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 1f04ab33..51ba0c7f 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -13,42 +13,19 @@ namespace LaunchDarkly.Sdk.Server.Ai { public class LdAiTrackerTest { - private static bool MatchesTrackData(LdValue actual, string flagKey, LdAiConfig config) + private static LdAiConfigTracker MakeTracker(Mock mockClient, string flagKey, + Context context, string variationKey = "", int version = 1, string modelName = "", + string providerName = "") { - var variationKeyMatch = string.IsNullOrEmpty(config.VariationKey) - ? actual.Get("variationKey").IsNull - : actual.Get("variationKey").Equals(LdValue.Of(config.VariationKey)); - - return variationKeyMatch && - actual.Get("version").Equals(LdValue.Of(config.Version)) && - actual.Get("configKey").Equals(LdValue.Of(flagKey)) && - actual.Get("modelName").Equals(LdValue.Of(config.Model.Name)) && - actual.Get("providerName").Equals(LdValue.Of(config.Provider.Name)) && - actual.Get("runId").Type == LdValueType.String && - actual.Get("runId").AsString.Length > 0; - } - - [Fact] - public void ThrowsIfClientIsNull() - { - Assert.Throws(() => - new LdAiConfigTracker(null, "key", LdAiConfig.Disabled, Context.New("key"))); + return new LdAiConfigTracker(mockClient.Object, Guid.NewGuid().ToString(), flagKey, + variationKey, version, context, modelName, providerName); } - [Fact] - public void ThrowsIfConfigIsNull() + private static bool MatchesTrackData(LdValue actual, string flagKey) { - var mockClient = new Mock(); - Assert.Throws(() => - new LdAiConfigTracker(mockClient.Object, "key", null, Context.New("key"))); - } - - [Fact] - public void ThrowsIfKeyIsNull() - { - var mockClient = new Mock(); - Assert.Throws(() => - new LdAiConfigTracker(mockClient.Object, null, LdAiConfig.Disabled, Context.New("key"))); + return actual.Get("configKey").Equals(LdValue.Of(flagKey)) && + actual.Get("runId").Type == LdValueType.String && + actual.Get("runId").AsString.Length > 0; } [Fact] @@ -57,9 +34,8 @@ public void TrackDataIncludesRunId() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackDuration(1.0f); mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, @@ -75,12 +51,11 @@ public void CanTrackDuration() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackDuration(1.0f); mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); } [Fact] @@ -89,12 +64,11 @@ public void CanTrackTimeToFirstToken() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackTimeToFirstToken(1.0f); mockClient.Verify(x => x.Track("$ld:ai:tokens:ttf", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); } [Fact] @@ -103,12 +77,11 @@ public void CanTrackSuccess() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackSuccess(); mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); } @@ -118,12 +91,11 @@ public void CanTrackError() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackError(); mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); } @@ -133,9 +105,8 @@ public async void CanTrackDurationOfTask() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); const int waitMs = 10; @@ -152,7 +123,7 @@ public async void CanTrackDurationOfTask() // between 0 and some large number. mockClient.Verify( x => x.Track("$ld:ai:duration:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), + It.Is(d => MatchesTrackData(d, flagKey)), It.IsInRange(0, 500, Moq.Range.Inclusive)), Times.Once); } @@ -163,13 +134,12 @@ public void CanTrackPositiveFeedback() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackFeedback(Feedback.Positive); mockClient.Verify(x => x.Track("$ld:ai:feedback:user:positive", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); } [Fact] @@ -178,13 +148,12 @@ public void CanTrackNegativeFeedback() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackFeedback(Feedback.Negative); mockClient.Verify(x => x.Track("$ld:ai:feedback:user:negative", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); } [Fact] @@ -193,9 +162,8 @@ public void CanTrackTokens() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); var givenUsage = new Usage { @@ -206,11 +174,11 @@ public void CanTrackTokens() tracker.TrackTokens(givenUsage); mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 2.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 2.0f), Times.Once); mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 3.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 3.0f), Times.Once); } [Fact] @@ -219,9 +187,8 @@ public void CanTrackResponseWithSpecificLatency() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); var givenUsage = new Usage { @@ -244,15 +211,15 @@ public void CanTrackResponseWithSpecificLatency() var result = tracker.TrackRequest(Task.Run(() => givenResponse)); Assert.Equal(givenResponse, result.Result); mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 2.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 2.0f), Times.Once); mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 3.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 3.0f), Times.Once); mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 500.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 500.0f), Times.Once); } [Fact] @@ -261,9 +228,8 @@ public void CanTrackResponseWithPartialData() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); var givenUsage = new Usage { @@ -279,11 +245,11 @@ public void CanTrackResponseWithPartialData() var result = tracker.TrackRequest(Task.Run(() => givenResponse)); Assert.Equal(givenResponse, result.Result); mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); // if latency isn't provided via Statistics, then it is automatically measured. mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), It.IsAny()), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), It.IsAny()), Times.Once); } [Fact] @@ -292,18 +258,17 @@ public async Task CanTrackExceptionFromResponse() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); await Assert.ThrowsAsync(() => tracker.TrackRequest(Task.FromException(new System.Exception("I am an exception")))); mockClient.Verify(x => x.Track("$ld:ai:generation:error", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), 1.0f), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), 1.0f), Times.Once); // if latency isn't provided via Statistics, then it is automatically measured. mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, - It.Is(d => MatchesTrackData(d, flagKey, config)), It.IsAny()), Times.Once); + It.Is(d => MatchesTrackData(d, flagKey)), It.IsAny()), Times.Once); } [Fact] @@ -312,9 +277,8 @@ public void DuplicateTrackDurationIsIgnored() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackDuration(1.0f); tracker.TrackDuration(2.0f); @@ -329,9 +293,8 @@ public void DuplicateTrackTimeToFirstTokenIsIgnored() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackTimeToFirstToken(1.0f); tracker.TrackTimeToFirstToken(2.0f); @@ -346,9 +309,8 @@ public void DuplicateTrackTokensIsIgnored() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); var usage = new Usage { Total = 1, Input = 2, Output = 3 }; @@ -369,9 +331,8 @@ public void DuplicateTrackFeedbackIsIgnored() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackFeedback(Feedback.Positive); tracker.TrackFeedback(Feedback.Negative); @@ -388,9 +349,8 @@ public void DuplicateTrackSuccessIsIgnored() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackSuccess(); tracker.TrackSuccess(); @@ -405,9 +365,8 @@ public void TrackErrorAfterSuccessIsIgnored() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackSuccess(); tracker.TrackError(); @@ -424,9 +383,8 @@ public void TrackSuccessAfterErrorIsIgnored() var mockClient = new Mock(); var context = Context.New("key"); const string flagKey = "key"; - var config = LdAiConfig.Disabled; - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); + var tracker = MakeTracker(mockClient, flagKey, context); tracker.TrackError(); tracker.TrackSuccess(); @@ -438,137 +396,50 @@ public void TrackSuccessAfterErrorIsIgnored() } [Fact] - public void CreateTrackerReturnsTrackerWhenDisabled() - { - var mockClient = new Mock(); - var context = Context.New("key"); - var config = LdAiConfig.Disabled; - - var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); - var newTracker = tracker.CreateTracker(); - - Assert.NotNull(newTracker); - Assert.NotSame(tracker, newTracker); - Assert.Equal(tracker.Config, newTracker.Config); - } - - [Fact] - public void CreateTrackerReturnsNewTrackerWhenEnabled() - { - var mockClient = new Mock(); - var context = Context.New("key"); - var config = LdAiConfig.New().Enable().Build(); - - var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); - var newTracker = tracker.CreateTracker(); - - Assert.NotNull(newTracker); - Assert.NotSame(tracker, newTracker); - Assert.Equal(tracker.Config, newTracker.Config); - } - - [Fact] - public void CreateTrackerReturnsTrackerWithFreshRunId() + public void ResumptionTokenContainsExpectedFields() { var mockClient = new Mock(); var context = Context.New("key"); - const string flagKey = "key"; - var config = LdAiConfig.New().Enable().Build(); - var tracker1 = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); - var tracker2 = tracker1.CreateTracker(); - - tracker1.TrackDuration(10.0f); - tracker2.TrackDuration(20.0f); + var tracker = new LdAiConfigTracker(mockClient.Object, Guid.NewGuid().ToString(), + "my-config-key", "", 1, context, "test-model", "test-provider"); + var token = tracker.ResumptionToken; - string runId1 = null; - string runId2 = null; + Assert.NotNull(token); + Assert.NotEmpty(token); - foreach (var call in mockClient.Invocations) + // Decode and verify the payload + var base64 = token.Replace('-', '+').Replace('_', '/'); + switch (base64.Length % 4) { - if (call.Method.Name == "Track" && (string)call.Arguments[0] == "$ld:ai:duration:total") - { - var data = (LdValue)call.Arguments[2]; - var runId = data.Get("runId").AsString; - if (runId1 == null) runId1 = runId; - else runId2 = runId; - } + case 2: base64 += "=="; break; + case 3: base64 += "="; break; } + var json = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); + var doc = JsonDocument.Parse(json); - Assert.NotNull(runId1); - Assert.NotNull(runId2); - Assert.NotEqual(runId1, runId2); - } - - [Fact] - public void CreateTrackerReturnsTrackerWithIndependentTrackingState() - { - var mockClient = new Mock(); - var context = Context.New("key"); - const string flagKey = "key"; - var config = LdAiConfig.New().Enable().Build(); - - var tracker = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); - - // Track duration on the original tracker (exhausts at-most-once) - tracker.TrackDuration(1.0f); - - // Create a new tracker - it should have fresh tracking state - var newTracker = tracker.CreateTracker(); - newTracker.TrackDuration(2.0f); - - // Both calls should have gone through - mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, - It.IsAny(), 1.0f), Times.Once); - mockClient.Verify(x => x.Track("$ld:ai:duration:total", context, - It.IsAny(), 2.0f), Times.Once); - } - - [Fact] - public void CreateTrackerCanBeCalledMultipleTimes() - { - var mockClient = new Mock(); - var context = Context.New("key"); - const string flagKey = "key"; - var config = LdAiConfig.New().Enable().Build(); - - var original = new LdAiConfigTracker(mockClient.Object, flagKey, config, context); - - var tracker1 = original.CreateTracker(); - var tracker2 = original.CreateTracker(); - var tracker3 = original.CreateTracker(); - - Assert.NotNull(tracker1); - Assert.NotNull(tracker2); - Assert.NotNull(tracker3); + Assert.Equal("my-config-key", doc.RootElement.GetProperty("configKey").GetString()); + Assert.Equal(1, doc.RootElement.GetProperty("version").GetInt32()); + Assert.True(doc.RootElement.GetProperty("runId").GetString().Length > 0); - // Each tracker should be able to independently track success - tracker1.TrackSuccess(); - tracker2.TrackSuccess(); - tracker3.TrackSuccess(); + // variationKey is empty, so it should be omitted + Assert.False(doc.RootElement.TryGetProperty("variationKey", out _)); - mockClient.Verify(x => x.Track("$ld:ai:generation:success", context, - It.IsAny(), It.IsAny()), Times.Exactly(3)); + // modelName and providerName should NOT be in the token + Assert.False(doc.RootElement.TryGetProperty("modelName", out _)); + Assert.False(doc.RootElement.TryGetProperty("providerName", out _)); } [Fact] - public void ResumptionTokenContainsExpectedFields() + public void ResumptionTokenHasCanonicalKeyOrder() { var mockClient = new Mock(); var context = Context.New("key"); - var config = LdAiConfig.New() - .Enable() - .SetModelName("test-model") - .SetModelProviderName("test-provider") - .Build(); - var tracker = new LdAiConfigTracker(mockClient.Object, "my-config-key", config, context); + var tracker = new LdAiConfigTracker(mockClient.Object, "test-run-id", + "my-config-key", "my-variation", 5, context, "", ""); var token = tracker.ResumptionToken; - Assert.NotNull(token); - Assert.NotEmpty(token); - - // Decode and verify the payload var base64 = token.Replace('-', '+').Replace('_', '/'); switch (base64.Length % 4) { @@ -576,18 +447,16 @@ public void ResumptionTokenContainsExpectedFields() case 3: base64 += "="; break; } var json = Encoding.UTF8.GetString(Convert.FromBase64String(base64)); - var doc = JsonDocument.Parse(json); - Assert.Equal("my-config-key", doc.RootElement.GetProperty("configKey").GetString()); - Assert.Equal(config.Version, doc.RootElement.GetProperty("version").GetInt32()); - Assert.True(doc.RootElement.GetProperty("runId").GetString().Length > 0); - - // variationKey is empty for builder-created configs, so it should be omitted - Assert.False(doc.RootElement.TryGetProperty("variationKey", out _)); + // Verify canonical order: runId → configKey → variationKey → version + var runIdIdx = json.IndexOf("\"runId\"", StringComparison.Ordinal); + var configKeyIdx = json.IndexOf("\"configKey\"", StringComparison.Ordinal); + var variationKeyIdx = json.IndexOf("\"variationKey\"", StringComparison.Ordinal); + var versionIdx = json.IndexOf("\"version\"", StringComparison.Ordinal); - // modelName and providerName should NOT be in the token - Assert.False(doc.RootElement.TryGetProperty("modelName", out _)); - Assert.False(doc.RootElement.TryGetProperty("providerName", out _)); + Assert.True(runIdIdx < configKeyIdx); + Assert.True(configKeyIdx < variationKeyIdx); + Assert.True(variationKeyIdx < versionIdx); } [Fact] @@ -595,9 +464,9 @@ public void ResumptionTokenIsUrlSafeBase64() { var mockClient = new Mock(); var context = Context.New("key"); - var config = LdAiConfig.New().Enable().Build(); - var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); + var tracker = new LdAiConfigTracker(mockClient.Object, Guid.NewGuid().ToString(), + "key", "", 1, context, "", ""); var token = tracker.ResumptionToken; // URL-safe base64 should not contain +, /, or = @@ -611,9 +480,9 @@ public void ResumptionTokenIsConsistentAcrossCalls() { var mockClient = new Mock(); var context = Context.New("key"); - var config = LdAiConfig.New().Enable().Build(); - var tracker = new LdAiConfigTracker(mockClient.Object, "key", config, context); + var tracker = new LdAiConfigTracker(mockClient.Object, Guid.NewGuid().ToString(), + "key", "", 1, context, "", ""); var token1 = tracker.ResumptionToken; var token2 = tracker.ResumptionToken; @@ -644,7 +513,8 @@ public void ResumptionTokenIncludesVariationKeyWhenPresent() })); var client = new LdAiClient(mockClient.Object); - var tracker = client.CompletionConfig("key", context); + var config = client.CompletionConfig("key", context); + var tracker = config.CreateTracker(); var token = tracker.ResumptionToken; // Decode and verify variationKey is present @@ -665,9 +535,9 @@ public void FromResumptionTokenReconstructsTrackerWithOriginalRunId() { var mockClient = new Mock(); var context = Context.New("key"); - var config = LdAiConfig.New().Enable().Build(); - var original = new LdAiConfigTracker(mockClient.Object, "my-key", config, context); + var original = new LdAiConfigTracker(mockClient.Object, Guid.NewGuid().ToString(), + "my-key", "", 1, context, "", ""); var token = original.ResumptionToken; var newContext = Context.New("other-key"); @@ -708,5 +578,52 @@ public void FromResumptionTokenThrowsOnInvalidToken() Assert.Throws(() => LdAiConfigTracker.FromResumptionToken("not-valid!!!", mockClient.Object, Context.New("key"))); } + + [Fact] + public void SummaryReflectsTrackedMetrics() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = MakeTracker(mockClient, "key", context); + + // Initially all null + var summary = tracker.Summary; + Assert.Null(summary.DurationMs); + Assert.Null(summary.Feedback); + Assert.Null(summary.Tokens); + Assert.Null(summary.Success); + Assert.Null(summary.TimeToFirstTokenMs); + + // Track some metrics + tracker.TrackDuration(100.5f); + tracker.TrackTimeToFirstToken(10.2f); + tracker.TrackFeedback(Feedback.Positive); + tracker.TrackSuccess(); + tracker.TrackTokens(new Usage { Total = 10, Input = 3, Output = 7 }); + + summary = tracker.Summary; + Assert.Equal(100.5, summary.DurationMs.Value, 1); + Assert.Equal(Feedback.Positive, summary.Feedback); + Assert.NotNull(summary.Tokens); + Assert.Equal(10, summary.Tokens.Value.Total); + Assert.Equal(3, summary.Tokens.Value.Input); + Assert.Equal(7, summary.Tokens.Value.Output); + Assert.True(summary.Success); + Assert.Equal(10.2, summary.TimeToFirstTokenMs.Value, 1); + } + + [Fact] + public void SummaryReflectsErrorState() + { + var mockClient = new Mock(); + var context = Context.New("key"); + + var tracker = MakeTracker(mockClient, "key", context); + tracker.TrackError(); + + var summary = tracker.Summary; + Assert.False(summary.Success); + } } } From f477ee46b5108ebe207bde14cc5ae4dbe6934185 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 13 May 2026 11:26:27 -0500 Subject: [PATCH 10/19] refactor: Clarify already-tracked warnings as skips with remedy Reword the six at-most-once warning messages to lead with "Skipping :" and direct the user to call CreateTracker on the AI Config to start a new run. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 265627ac..0eab2077 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -121,7 +121,7 @@ public void TrackDuration(float durationMs) { if (_durationMs.HasValue) { - _logger?.Warn("Duration has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); + _logger?.Warn("Skipping TrackDuration: duration already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } _durationMs = durationMs; @@ -146,7 +146,7 @@ public void TrackTimeToFirstToken(float timeToFirstTokenMs) { if (_timeToFirstTokenMs.HasValue) { - _logger?.Warn("Time to first token has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); + _logger?.Warn("Skipping TrackTimeToFirstToken: time-to-first-token already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } _timeToFirstTokenMs = timeToFirstTokenMs; @@ -158,7 +158,7 @@ public void TrackFeedback(Feedback feedback) { if (_feedback.HasValue) { - _logger?.Warn("Feedback has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); + _logger?.Warn("Skipping TrackFeedback: feedback already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } _feedback = feedback; @@ -180,7 +180,7 @@ public void TrackSuccess() { if (_trackedSuccess.HasValue) { - _logger?.Warn("Generation result has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); + _logger?.Warn("Skipping TrackSuccess: success/error already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } _trackedSuccess = true; @@ -192,7 +192,7 @@ public void TrackError() { if (_trackedSuccess.HasValue) { - _logger?.Warn("Generation result has already been tracked for this operation. [{0}]", _trackData.ToJsonString()); + _logger?.Warn("Skipping TrackError: success/error already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } _trackedSuccess = false; @@ -233,7 +233,7 @@ public void TrackTokens(Usage usage) { if (_tokens.HasValue) { - _logger?.Warn("Tokens have already been tracked for this operation. [{0}]", _trackData.ToJsonString()); + _logger?.Warn("Skipping TrackTokens: tokens already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } _tokens = usage; From 2991347127b1a96f4aa0343c5b2f923799c4a100 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 13 May 2026 11:28:07 -0500 Subject: [PATCH 11/19] docs: Clarify runId purpose and per-method tracker semantics Expand the tracker class and interface summaries to explain that all events share a runId so LaunchDarkly can correlate them in metrics views, and that a ResumptionToken preserves the runId across processes. Reframe LdAiConfig.CreateTracker as the entry point for a fresh AI run, calling out the UUIDv4 runId and the rule that metrics from different runIds cannot be combined. Document the at-most-once semantics on TrackDuration, TrackTimeToFirstToken, TrackFeedback, TrackTokens, the shared-state semantics on TrackSuccess/TrackError, and note that TrackRequest's inner metrics are themselves at-most-once. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/Config/LdAiConfig.cs | 9 ++++--- .../src/Interfaces/ILdAiConfigTracker.cs | 26 +++++++++++++++++-- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 11 ++++++-- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs index 4d98e460..b11854f1 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs @@ -294,11 +294,14 @@ internal LdValue ToLdValue() public int Version { get; } /// - /// A factory that creates a new with a fresh runId. - /// Each invocation returns a tracker with independent at-most-once tracking state. + /// Returns a new tracker for a fresh AI run. Each call mints a new runId (a UUIDv4) + /// that LaunchDarkly uses to correlate the run's events in metrics views. Call this + /// once per AI run; metrics from different runIds cannot be combined. + /// + /// /// This property is set when the config is returned by . /// It will be null for configs created directly via the builder. - /// + /// public Func CreateTracker { get; } /// diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index a4c6cb63..7da59764 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -6,9 +6,15 @@ namespace LaunchDarkly.Sdk.Server.Ai.Interfaces; /// -/// A utility capable of generating events related to a specific AI model -/// configuration. +/// Records metrics for a single AI run. /// +/// +/// All events a tracker emits share a runId (a UUIDv4) so LaunchDarkly can correlate +/// them in metrics views. See individual track methods for their specific semantics. +/// Call CreateTracker on the AI Config to start a new run. A +/// preserves the runId, so events emitted by a tracker +/// reconstructed in another process correlate with the original run. +/// public interface ILdAiConfigTracker { /// @@ -29,6 +35,7 @@ public interface ILdAiConfigTracker /// related to usage of the AI model takes 100ms, this can be tracked and made available in /// LaunchDarkly. /// + /// Records at most once per Tracker; further calls are ignored. /// the duration in milliseconds public void TrackDuration(float durationMs); @@ -48,12 +55,14 @@ public interface ILdAiConfigTracker /// /// Tracks the time it takes for the first token to be generated. /// + /// Records at most once per Tracker; further calls are ignored. /// the duration in milliseconds public void TrackTimeToFirstToken(float timeToFirstTokenMs); /// /// Tracks feedback (positive or negative) related to the output of the model. /// + /// Records at most once per Tracker; further calls are ignored. /// the feedback /// thrown if the feedback value is not or public void TrackFeedback(Feedback feedback); @@ -61,11 +70,19 @@ public interface ILdAiConfigTracker /// /// Tracks a generation event related to this config. /// + /// + /// Records at most once per Tracker. TrackSuccess and TrackError share state; only + /// one of the two can record per Tracker, and subsequent calls are ignored. + /// public void TrackSuccess(); /// /// Tracks an unsuccessful generation event related to this config. /// + /// + /// Records at most once per Tracker. TrackSuccess and TrackError share state; only + /// one of the two can record per Tracker, and subsequent calls are ignored. + /// public void TrackError(); /// @@ -100,6 +117,10 @@ public interface ILdAiConfigTracker /// Task is automatically measured and recorded as the latency metric associated with this request. /// /// + /// + /// Because each inner metric is at-most-once per Tracker, calling TrackRequest twice + /// on the same Tracker will run the task again but produce no additional metric events. + /// /// a task representing the request /// the task public Task TrackRequest(Task request); @@ -107,6 +128,7 @@ public interface ILdAiConfigTracker /// /// Tracks token usage related to this config. /// + /// Records at most once per Tracker; further calls are ignored. /// the token usage public void TrackTokens(Usage usage); } diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 0eab2077..4f5aa609 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -13,8 +13,15 @@ namespace LaunchDarkly.Sdk.Server.Ai; /// -/// A tracker capable of reporting events related to a particular AI Config. +/// Records metrics for a single AI run. /// +/// +/// All events a tracker emits share a runId (a UUIDv4) so LaunchDarkly can correlate +/// them in metrics views. See individual track methods for their specific semantics. +/// Call CreateTracker on the AI Config to start a new run. A +/// preserves the runId, so events emitted by a tracker +/// reconstructed in another process correlate with the original run. +/// public class LdAiConfigTracker : ILdAiConfigTracker { private readonly ILaunchDarklyClient _client; @@ -46,7 +53,7 @@ public class LdAiConfigTracker : ILdAiConfigTracker /// /// Constructs a new AI Config tracker. The tracker is associated with a run, a config key, - /// and a context. The runId should be a unique identifier (UUID v4) for each execution. + /// and a context. The runId should be a unique identifier (UUID v4) for each AI run. /// /// the LaunchDarkly client /// the unique run identifier From 279a38e7bff993fcb0bb8d473633da33488da5a5 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 13 May 2026 11:58:17 -0500 Subject: [PATCH 12/19] docs: Avoid "run...runId" duplicate phrasing in resumption docs Replace "correlate with the original run" and "associated with the original run" with "the original tracker's runId" so the surrounding runId references no longer read awkwardly. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs | 2 +- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index 7da59764..6fe2adb9 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -13,7 +13,7 @@ namespace LaunchDarkly.Sdk.Server.Ai.Interfaces; /// them in metrics views. See individual track methods for their specific semantics. /// Call CreateTracker on the AI Config to start a new run. A /// preserves the runId, so events emitted by a tracker -/// reconstructed in another process correlate with the original run. +/// reconstructed in another process correlate with the original tracker's runId. /// public interface ILdAiConfigTracker { diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 4f5aa609..bbf658f9 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -20,7 +20,7 @@ namespace LaunchDarkly.Sdk.Server.Ai; /// them in metrics views. See individual track methods for their specific semantics. /// Call CreateTracker on the AI Config to start a new run. A /// preserves the runId, so events emitted by a tracker -/// reconstructed in another process correlate with the original run. +/// reconstructed in another process correlate with the original tracker's runId. /// public class LdAiConfigTracker : ILdAiConfigTracker { @@ -269,7 +269,7 @@ public void TrackTokens(Usage usage) /// the resumption token obtained from /// the LaunchDarkly client /// the context to use for track events - /// a new tracker associated with the original run + /// a new tracker associated with the original tracker's runId /// thrown if token or client is null /// thrown if the token is malformed or missing required fields public static LdAiConfigTracker FromResumptionToken(string token, ILaunchDarklyClient client, Context context) From 2e0e593151921baf3842a8657fb6f28ae4360a72 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 14 May 2026 11:33:56 -0500 Subject: [PATCH 13/19] docs: Clarify composite-tracker re-call wording A retried TrackRequest can still emit metrics that were not recorded on the first call (e.g. tokens after a previous failure), so the previous "produce no additional metric events" wording was wrong in the partial-failure case. Reframe around the actual behavior and direct callers to CreateTracker for a fresh run. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index 6fe2adb9..bd38cdb8 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -118,8 +118,8 @@ public interface ILdAiConfigTracker /// /// /// - /// Because each inner metric is at-most-once per Tracker, calling TrackRequest twice - /// on the same Tracker will run the task again but produce no additional metric events. + /// Subsequent calls re-run the task but emit only metrics not already recorded on this Tracker. + /// Call CreateTracker on the AI Config to start a new run. /// /// a task representing the request /// the task From 958464ad3887b823b518bec539245e8b55c2c49c Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 14 May 2026 12:02:36 -0500 Subject: [PATCH 14/19] fix: Validate feedback before consuming at-most-once slot in TrackFeedback Move the _feedback assignment to after the switch so an invalid Feedback value (e.g. (Feedback)42) throws ArgumentOutOfRangeException without permanently consuming the at-most-once slot. A subsequent valid TrackFeedback call now records normally instead of being silently dropped by the skip guard. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 2 +- .../server-ai/test/LdAiConfigTrackerTest.cs | 26 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index bbf658f9..592c1a76 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -168,7 +168,6 @@ public void TrackFeedback(Feedback feedback) _logger?.Warn("Skipping TrackFeedback: feedback already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } - _feedback = feedback; switch (feedback) { case Feedback.Positive: @@ -180,6 +179,7 @@ public void TrackFeedback(Feedback feedback) default: throw new ArgumentOutOfRangeException(nameof(feedback), feedback, null); } + _feedback = feedback; } /// diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 51ba0c7f..9f49306e 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -343,6 +343,32 @@ public void DuplicateTrackFeedbackIsIgnored() It.IsAny(), It.IsAny()), Times.Never); } + [Fact] + public void TrackFeedbackWithInvalidValueDoesNotConsumeAtMostOnceSlot() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + + var tracker = MakeTracker(mockClient, flagKey, context); + + Assert.Throws(() => tracker.TrackFeedback((Feedback)42)); + + // No event emitted by the invalid call. + mockClient.Verify(x => x.Track("$ld:ai:feedback:user:positive", context, + It.IsAny(), It.IsAny()), Times.Never); + mockClient.Verify(x => x.Track("$ld:ai:feedback:user:negative", context, + It.IsAny(), It.IsAny()), Times.Never); + + // A subsequent valid call should record normally. + tracker.TrackFeedback(Feedback.Positive); + mockClient.Verify(x => x.Track("$ld:ai:feedback:user:positive", context, + It.IsAny(), It.IsAny()), Times.Once); + + // And the slot is now consumed. + Assert.Equal(Feedback.Positive, tracker.Summary.Feedback); + } + [Fact] public void DuplicateTrackSuccessIsIgnored() { From 6ad7497c95c5d36dfcc1204278efef45c016b880 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 15 May 2026 09:05:30 -0500 Subject: [PATCH 15/19] refactor: Convert LdAiConfig to a class and expose CreateTracker as a method LdAiConfig was declared as a record, which made the compiler synthesize value equality over every instance field. Adding the tracker factory broke that contract because two configs returned by separate CompletionConfig calls hold different delegate instances even when they represent the same AI Config. Since LdAiConfig carries a delegate and exposes a CreateTracker method, it is not really a pure data value object; reference equality is the honest contract. Convert it to a plain class so no synthesized equality is generated, drop the public Func<> property in favor of a method CreateTracker() that invokes the private factory, and reword the remarks accordingly. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/Config/LdAiConfig.cs | 15 +++++++++------ pkgs/sdk/server-ai/test/LdAiClientTest.cs | 4 ---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs index b11854f1..37221a5b 100644 --- a/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs +++ b/pkgs/sdk/server-ai/src/Config/LdAiConfig.cs @@ -9,7 +9,7 @@ namespace LaunchDarkly.Sdk.Server.Ai.Config; /// /// Represents an AI Config, which contains model parameters and prompt messages. /// -public record LdAiConfig +public class LdAiConfig { /// @@ -219,6 +219,8 @@ public LdAiConfig Build() /// public readonly ModelProvider Provider; + private readonly Func _trackerFactory; + internal LdAiConfig(bool enabled, IEnumerable messages, Meta meta, Model model, Provider provider, Func createTracker = null) { @@ -229,7 +231,7 @@ internal LdAiConfig(bool enabled, IEnumerable messages, Meta meta, Mode Version = meta?.Version ?? 1; Enabled = enabled; Provider = new ModelProvider(provider?.Name ?? ""); - CreateTracker = createTracker; + _trackerFactory = createTracker; } internal LdAiConfig(LdAiConfig source, Func createTracker) @@ -240,8 +242,9 @@ internal LdAiConfig(LdAiConfig source, Func createTracker) Version = source.Version; Enabled = source.Enabled; Provider = source.Provider; - CreateTracker = createTracker; + _trackerFactory = createTracker; } + internal LdValue ToLdValue() { return LdValue.ObjectFrom(new Dictionary @@ -299,10 +302,10 @@ internal LdValue ToLdValue() /// once per AI run; metrics from different runIds cannot be combined. /// /// - /// This property is set when the config is returned by . - /// It will be null for configs created directly via the builder. + /// Returns null for configs created directly via the builder; only configs returned by + /// have a factory wired in. /// - public Func CreateTracker { get; } + public ILdAiConfigTracker CreateTracker() => _trackerFactory?.Invoke(); /// /// Convenient helper that returns a disabled LdAiConfig. diff --git a/pkgs/sdk/server-ai/test/LdAiClientTest.cs b/pkgs/sdk/server-ai/test/LdAiClientTest.cs index e08b28c1..f204718c 100644 --- a/pkgs/sdk/server-ai/test/LdAiClientTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiClientTest.cs @@ -394,8 +394,6 @@ public void CompletionConfigReturnsConfigWithCreateTrackerFactory() var client = new LdAiClient(mockClient.Object); var config = client.CompletionConfig(configKey, context); - Assert.NotNull(config.CreateTracker); - var tracker = config.CreateTracker(); Assert.NotNull(tracker); } @@ -501,8 +499,6 @@ public void DefaultConfigGetsCreateTrackerFactory() var client = new LdAiClient(mockClient.Object); var config = client.CompletionConfig("foo", Context.New("key")); - Assert.NotNull(config.CreateTracker); - var tracker = config.CreateTracker(); Assert.NotNull(tracker); } From 31c1293d3f0f49807bdfe7ff9ab8a43876fb63b9 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 15 May 2026 09:05:41 -0500 Subject: [PATCH 16/19] fix: Validate usage before consuming at-most-once slot in TrackTokens A TrackTokens call with all-zero or all-null Usage emits no events because each $ld:ai:tokens:* event is gated on the corresponding counter being positive. The previous implementation still assigned _tokens = usage unconditionally, which consumed the at-most-once slot and caused a subsequent valid TrackTokens call to be silently dropped by the skip guard. Track whether any event was emitted and only assign _tokens when at least one was, so empty calls do not consume the slot. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 9 ++++- .../server-ai/test/LdAiConfigTrackerTest.cs | 35 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 592c1a76..856b5438 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -243,18 +243,25 @@ public void TrackTokens(Usage usage) _logger?.Warn("Skipping TrackTokens: tokens already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } - _tokens = usage; + var emitted = false; if (usage.Total is > 0) { _client.Track(TokenTotal, _context, _trackData, usage.Total.Value); + emitted = true; } if (usage.Input is > 0) { _client.Track(TokenInput, _context, _trackData, usage.Input.Value); + emitted = true; } if (usage.Output is > 0) { _client.Track(TokenOutput, _context, _trackData, usage.Output.Value); + emitted = true; + } + if (emitted) + { + _tokens = usage; } } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 9f49306e..f1f3c0f9 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -325,6 +325,41 @@ public void DuplicateTrackTokensIsIgnored() It.IsAny(), It.IsAny()), Times.Once); } + [Fact] + public void TrackTokensWithEmptyUsageDoesNotConsumeAtMostOnceSlot() + { + var mockClient = new Mock(); + var context = Context.New("key"); + const string flagKey = "key"; + + var tracker = MakeTracker(mockClient, flagKey, context); + + // Empty usage emits no events. + tracker.TrackTokens(default(Usage)); + + mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, + It.IsAny(), It.IsAny()), Times.Never); + mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, + It.IsAny(), It.IsAny()), Times.Never); + mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, + It.IsAny(), It.IsAny()), Times.Never); + Assert.Null(tracker.Summary.Tokens); + + // A subsequent call with positive values should still record. + var realUsage = new Usage { Total = 100, Input = 40, Output = 60 }; + tracker.TrackTokens(realUsage); + + mockClient.Verify(x => x.Track("$ld:ai:tokens:total", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:input", context, + It.IsAny(), It.IsAny()), Times.Once); + mockClient.Verify(x => x.Track("$ld:ai:tokens:output", context, + It.IsAny(), It.IsAny()), Times.Once); + + // And the slot is now consumed. + Assert.Equal(realUsage, tracker.Summary.Tokens); + } + [Fact] public void DuplicateTrackFeedbackIsIgnored() { From 864333442fa3d0c0b79447ff57ddff185996ec8f Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 15 May 2026 09:21:46 -0500 Subject: [PATCH 17/19] fix: Reject null payload in FromResumptionToken with ArgumentException A resumption token whose base64-decoded contents are the JSON literal "null" parses successfully but deserializes to a null payload. The existing check that follows then NRE'd on payload.RunId instead of throwing the documented ArgumentException. Include payload == null in the missing-fields condition so the same descriptive error fires. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 2 +- pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 856b5438..9bb1a323 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -302,7 +302,7 @@ public static LdAiConfigTracker FromResumptionToken(string token, ILaunchDarklyC throw new ArgumentException("Invalid resumption token", nameof(token), e); } - if (string.IsNullOrEmpty(payload.RunId) || string.IsNullOrEmpty(payload.ConfigKey)) + if (payload == null || string.IsNullOrEmpty(payload.RunId) || string.IsNullOrEmpty(payload.ConfigKey)) { throw new ArgumentException("Resumption token is missing required fields (runId, configKey)", nameof(token)); diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index f1f3c0f9..6dba51af 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -640,6 +640,19 @@ public void FromResumptionTokenThrowsOnInvalidToken() LdAiConfigTracker.FromResumptionToken("not-valid!!!", mockClient.Object, Context.New("key"))); } + [Fact] + public void FromResumptionTokenRejectsNullJsonPayload() + { + var mockClient = new Mock(); + + // Base64-encode the JSON literal "null" so deserialization succeeds and yields a null payload. + var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes("null")); + var token = base64.Replace('+', '-').Replace('/', '_').TrimEnd('='); + + Assert.Throws(() => + LdAiConfigTracker.FromResumptionToken(token, mockClient.Object, Context.New("key"))); + } + [Fact] public void SummaryReflectsTrackedMetrics() { From 6b098260452f5e85b78cfa3bf50f56089f2acafb Mon Sep 17 00:00:00 2001 From: Jason Bailey Date: Fri, 15 May 2026 15:16:05 -0500 Subject: [PATCH 18/19] Apply suggestions from code review Co-authored-by: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> --- pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs index bd38cdb8..b89e20fa 100644 --- a/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/Interfaces/ILdAiConfigTracker.cs @@ -9,8 +9,8 @@ namespace LaunchDarkly.Sdk.Server.Ai.Interfaces; /// Records metrics for a single AI run. /// /// -/// All events a tracker emits share a runId (a UUIDv4) so LaunchDarkly can correlate -/// them in metrics views. See individual track methods for their specific semantics. +/// All events emitted by a tracker share a runId (a UUIDv4) so LaunchDarkly can correlate +/// them. See individual track methods for their specific semantics. /// Call CreateTracker on the AI Config to start a new run. A /// preserves the runId, so events emitted by a tracker /// reconstructed in another process correlate with the original tracker's runId. From 64e0a90503f03b8b2a06890b7fc57e7ba10ff909 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Wed, 27 May 2026 10:36:52 -0500 Subject: [PATCH 19/19] fix: Use Interlocked.CompareExchange to make tracker slot writes atomic Each at-most-once Track method previously performed a check-then-set on a nullable backing field, which is not atomic across threads. Two threads racing into the same method could both observe HasValue == false and both emit the same metric event, and for TrackSuccess / TrackError racing on the shared _trackedSuccess slot, both a success and an error event could be emitted for the same runId. Replace each nullable backing field with a StrongBox reference slot and claim the slot with Interlocked.CompareExchange before emitting the metric. The CAS becomes the atomic check-then-set, so only the thread that successfully wins the slot emits its event. For TrackSuccess and TrackError this also enforces the documented mutual-exclusion contract on _trackedSuccess. TrackFeedback validates the enum value before the CAS so an invalid value still throws without consuming the slot. TrackTokens keeps the empty-usage gate so an all-zero Usage does not burn the slot. Add three concurrency tests using Barrier-aligned threads across 500 iterations: TrackSuccessIsAtMostOnceUnderConcurrency, TrackSuccessAndTrackErrorAreMutuallyExclusiveUnderConcurrency, and TrackDurationIsAtMostOnceUnderConcurrency. Co-Authored-By: Claude Opus 4.7 (1M context) --- pkgs/sdk/server-ai/src/LdAiConfigTracker.cs | 74 ++++++++------- .../server-ai/test/LdAiConfigTrackerTest.cs | 95 +++++++++++++++++++ 2 files changed, 134 insertions(+), 35 deletions(-) diff --git a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs index 9bb1a323..266ef581 100644 --- a/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs +++ b/pkgs/sdk/server-ai/src/LdAiConfigTracker.cs @@ -2,9 +2,11 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading; using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.Interfaces; @@ -35,11 +37,11 @@ public class LdAiConfigTracker : ILdAiConfigTracker private readonly LdValue _trackData; private readonly ILogger _logger; - private double? _durationMs; - private double? _timeToFirstTokenMs; - private Usage? _tokens; - private Feedback? _feedback; - private bool? _trackedSuccess; + private StrongBox _durationMs; + private StrongBox _timeToFirstTokenMs; + private StrongBox _tokens; + private StrongBox _feedback; + private StrongBox _trackedSuccess; // true = success, false = error private const string Duration = "$ld:ai:duration:total"; private const string FeedbackPositive = "$ld:ai:feedback:user:positive"; @@ -116,22 +118,22 @@ public string ResumptionToken /// public MetricSummary Summary => new MetricSummary( - _durationMs, - _feedback, - _tokens, - _trackedSuccess, - _timeToFirstTokenMs + _durationMs?.Value, + _feedback?.Value, + _tokens?.Value, + _trackedSuccess?.Value, + _timeToFirstTokenMs?.Value ); /// public void TrackDuration(float durationMs) { - if (_durationMs.HasValue) + if (Interlocked.CompareExchange(ref _durationMs, + new StrongBox(durationMs), null) != null) { _logger?.Warn("Skipping TrackDuration: duration already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } - _durationMs = durationMs; _client.Track(Duration, _context, _trackData, durationMs); } @@ -151,58 +153,61 @@ public async Task TrackDurationOfTask(Task task) /// public void TrackTimeToFirstToken(float timeToFirstTokenMs) { - if (_timeToFirstTokenMs.HasValue) + if (Interlocked.CompareExchange(ref _timeToFirstTokenMs, + new StrongBox(timeToFirstTokenMs), null) != null) { _logger?.Warn("Skipping TrackTimeToFirstToken: time-to-first-token already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } - _timeToFirstTokenMs = timeToFirstTokenMs; _client.Track(TimeToFirstToken, _context, _trackData, timeToFirstTokenMs); } /// public void TrackFeedback(Feedback feedback) { - if (_feedback.HasValue) - { - _logger?.Warn("Skipping TrackFeedback: feedback already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); - return; - } + // Validate the enum value first so invalid input throws without consuming the slot. + string eventName; switch (feedback) { case Feedback.Positive: - _client.Track(FeedbackPositive, _context, _trackData, 1); + eventName = FeedbackPositive; break; case Feedback.Negative: - _client.Track(FeedbackNegative, _context, _trackData, 1); + eventName = FeedbackNegative; break; default: throw new ArgumentOutOfRangeException(nameof(feedback), feedback, null); } - _feedback = feedback; + if (Interlocked.CompareExchange(ref _feedback, + new StrongBox(feedback), null) != null) + { + _logger?.Warn("Skipping TrackFeedback: feedback already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); + return; + } + _client.Track(eventName, _context, _trackData, 1); } /// public void TrackSuccess() { - if (_trackedSuccess.HasValue) + if (Interlocked.CompareExchange(ref _trackedSuccess, + new StrongBox(true), null) != null) { _logger?.Warn("Skipping TrackSuccess: success/error already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } - _trackedSuccess = true; _client.Track(GenerationSuccess, _context, _trackData, 1); } /// public void TrackError() { - if (_trackedSuccess.HasValue) + if (Interlocked.CompareExchange(ref _trackedSuccess, + new StrongBox(false), null) != null) { _logger?.Warn("Skipping TrackError: success/error already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } - _trackedSuccess = false; _client.Track(GenerationError, _context, _trackData, 1); } @@ -238,30 +243,29 @@ public async Task TrackRequest(Task request) /// public void TrackTokens(Usage usage) { - if (_tokens.HasValue) + // Empty usage doesn't burn the slot. + if ((usage.Total ?? 0) <= 0 && (usage.Input ?? 0) <= 0 && (usage.Output ?? 0) <= 0) + { + return; + } + // Atomic claim. + if (Interlocked.CompareExchange(ref _tokens, + new StrongBox(usage), null) != null) { _logger?.Warn("Skipping TrackTokens: tokens already recorded on this tracker. Call CreateTracker on the AI Config for a new run. {0}", _trackData.ToJsonString()); return; } - var emitted = false; if (usage.Total is > 0) { _client.Track(TokenTotal, _context, _trackData, usage.Total.Value); - emitted = true; } if (usage.Input is > 0) { _client.Track(TokenInput, _context, _trackData, usage.Input.Value); - emitted = true; } if (usage.Output is > 0) { _client.Track(TokenOutput, _context, _trackData, usage.Output.Value); - emitted = true; - } - if (emitted) - { - _tokens = usage; } } diff --git a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs index 6dba51af..28116a9b 100644 --- a/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs +++ b/pkgs/sdk/server-ai/test/LdAiConfigTrackerTest.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Text.Json; +using System.Threading; using System.Threading.Tasks; using LaunchDarkly.Sdk.Server.Ai.Config; using LaunchDarkly.Sdk.Server.Ai.Interfaces; @@ -699,5 +701,98 @@ public void SummaryReflectsErrorState() var summary = tracker.Summary; Assert.False(summary.Success); } + + [Fact] + public void TrackSuccessIsAtMostOnceUnderConcurrency() + { + const int iterations = 500; + var maxObservedInvocations = 0; + var iterationWithMax = -1; + + for (var i = 0; i < iterations; i++) + { + var mockClient = new Mock(); + var context = Context.New("key"); + var tracker = MakeTracker(mockClient, "key", context); + + var barrier = new Barrier(2); + var t1 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackSuccess(); }); + var t2 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackSuccess(); }); + Task.WaitAll(t1, t2); + + var calls = mockClient.Invocations.Count(inv => + inv.Method.Name == "Track" && + (string)inv.Arguments[0] == "$ld:ai:generation:success"); + + if (calls > maxObservedInvocations) { maxObservedInvocations = calls; iterationWithMax = i; } + } + + Assert.True(maxObservedInvocations == 1, + $"Expected at-most-once emission of $ld:ai:generation:success across {iterations} " + + $"iterations, but observed {maxObservedInvocations} emissions on iteration {iterationWithMax}. " + + "TrackSuccess's check-then-set on _trackedSuccess is not atomic."); + } + + [Fact] + public void TrackSuccessAndTrackErrorAreMutuallyExclusiveUnderConcurrency() + { + const int iterations = 500; + var iterationsWithBoth = 0; + + for (var i = 0; i < iterations; i++) + { + var mockClient = new Mock(); + var context = Context.New("key"); + var tracker = MakeTracker(mockClient, "key", context); + + var barrier = new Barrier(2); + var t1 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackSuccess(); }); + var t2 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackError(); }); + Task.WaitAll(t1, t2); + + var successCalls = mockClient.Invocations.Count(inv => + inv.Method.Name == "Track" && (string)inv.Arguments[0] == "$ld:ai:generation:success"); + var errorCalls = mockClient.Invocations.Count(inv => + inv.Method.Name == "Track" && (string)inv.Arguments[0] == "$ld:ai:generation:error"); + + if (successCalls > 0 && errorCalls > 0) iterationsWithBoth++; + } + + Assert.True(iterationsWithBoth == 0, + $"Expected TrackSuccess and TrackError to be mutually exclusive across {iterations} " + + $"iterations, but {iterationsWithBoth} iterations emitted both success AND error " + + "events for the same runId. The shared _trackedSuccess slot is not atomically guarded."); + } + + [Fact] + public void TrackDurationIsAtMostOnceUnderConcurrency() + { + const int iterations = 500; + var maxObservedInvocations = 0; + var iterationWithMax = -1; + + for (var i = 0; i < iterations; i++) + { + var mockClient = new Mock(); + var context = Context.New("key"); + var tracker = MakeTracker(mockClient, "key", context); + + var barrier = new Barrier(2); + var t1 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackDuration(123.0f); }); + var t2 = Task.Run(() => { barrier.SignalAndWait(); tracker.TrackDuration(123.0f); }); + Task.WaitAll(t1, t2); + + var calls = mockClient.Invocations.Count(inv => + inv.Method.Name == "Track" && + (string)inv.Arguments[0] == "$ld:ai:duration:total"); + + if (calls > maxObservedInvocations) { maxObservedInvocations = calls; iterationWithMax = i; } + } + + Assert.True(maxObservedInvocations == 1, + $"Expected at-most-once emission of $ld:ai:duration:total across {iterations} " + + $"iterations, but observed {maxObservedInvocations} emissions on iteration {iterationWithMax}. " + + "TrackDuration's check-then-set on _durationMs is not atomic."); + } } }