Skip to content

feat!: Add AI online evaluations (Judge, Evaluator, IRunner) #301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mattrmc1 wants to merge 14 commits into
base: main
Choose a base branch
from
Open

feat!: Add AI online evaluations (Judge, Evaluator, IRunner) #301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
a625933
[AIC-2660] Implement AIEVALS (first pass)
mattrmc1 Jun 15, 2026
db62ad3
Merge branch 'main' of github.com:launchdarkly/dotnet-core into mmcca…
mattrmc1 Jun 30, 2026
a680fb4
fix: AIEVALS spec compliance — JudgeResult defaults, null safety, sam…
mattrmc1 Jun 30, 2026
f920e9d
fix: set errorMessage on out-of-range score for JS/Python parity
mattrmc1 Jun 30, 2026
aa959ce
Merge branch 'main' into mmccarthy/AIC-2660/aievals-core-judge
mattrmc1 Jun 30, 2026
b814097
Merge branch 'main' into mmccarthy/AIC-2660/aievals-core-judge
mattrmc1 Jun 30, 2026
66e2c31
fix: robust judge score parsing, fallback evaluator wiring, filtered …
mattrmc1 Jun 30, 2026
f5f064e
fix: forward caller variables to embedded judge message interpolation
mattrmc1 Jul 1, 2026
3bea03d
Merge branch 'main' into mmccarthy/AIC-2660/aievals-core-judge
mattrmc1 Jul 1, 2026
b3bad64
fix: NaN/Infinity score rejection, nullable judge samplingRate, graph…
mattrmc1 Jul 1, 2026
c06c4b9
Merge branch 'main' of github.com:launchdarkly/dotnet-core into mmcca…
mattrmc1 Jul 1, 2026
6f7d88d
Merge branch 'mmccarthy/AIC-2660/aievals-core-judge' of github.com:la…
mattrmc1 Jul 1, 2026
fae8b52
Merge branch 'main' into mmccarthy/AIC-2660/aievals-core-judge
mattrmc1 Jul 6, 2026
2562911
fix: use shared Random instance in Evaluator to prevent correlated ju…
mattrmc1 Jul 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 78 additions & 13 deletions pkgs/sdk/server-ai/src/Config/ConfigFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using LaunchDarkly.Sdk.Server.Ai.Evals;
using LaunchDarkly.Sdk.Server.Ai.Interfaces;
using Mustache;

Expand All @@ -21,11 +22,14 @@ internal sealed class ConfigFactory

private readonly ILaunchDarklyClient _client;
private readonly ILogger _logger;
private readonly Func<LdAiJudgeConfig, IRunner> _runnerFactory;

public ConfigFactory(ILaunchDarklyClient client, ILogger logger)
public ConfigFactory(ILaunchDarklyClient client, ILogger logger,
Func<LdAiJudgeConfig, IRunner> runnerFactory = null)
{
_client = client;
_logger = logger;
_runnerFactory = runnerFactory;
}

public LdAiCompletionConfig BuildCompletionConfig(
Expand All @@ -44,7 +48,7 @@ public LdAiCompletionConfig BuildCompletionConfig(
_logger.Error(
"AI Config '{0}': variation result is not an object (got {1}); using caller's default.",
key, ldValue.Type);
return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate);
return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate);
}

var (enabled, variationKey, version, mode) = ParseMeta(ldValue);
Expand All @@ -54,7 +58,7 @@ public LdAiCompletionConfig BuildCompletionConfig(
_logger.Warn(
"AI Config mode mismatch for {0}: expected {1}, got {2}. Returning caller's default.",
key, LdAiCompletionConfig.Mode, mode);
return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate);
return BuildCompletionFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate);
}

var model = ParseModel(ldValue.Get("model"));
Expand All @@ -64,6 +68,7 @@ public LdAiCompletionConfig BuildCompletionConfig(
: ParseMessages(ldValue.Get("messages"));
var tools = ParseTools(ldValue.Get("tools"));
var judgeConfiguration = ParseJudgeConfiguration(ldValue.Get("judgeConfiguration"));
var evaluator = BuildEvaluator(judgeConfiguration, context, variables);

return new LdAiCompletionConfig(
key,
Expand All @@ -75,21 +80,25 @@ public LdAiCompletionConfig BuildCompletionConfig(
judgeConfiguration,
model,
provider,
trackerFactory);
trackerFactory,
evaluator);
}

private LdAiCompletionConfig BuildCompletionFromDefault(
string key,
LdAiCompletionConfigDefault defaultValue,
IReadOnlyDictionary<string, object> mergedVars,
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory,
Context context,
IReadOnlyDictionary<string, object> variables = null,
bool interpolate = true)
{
// Caller-supplied default messages can contain Mustache templates too; interpolate
// with the same per-message fallback as server-returned configs.
var messages = interpolate
? InterpolateMessages(defaultValue.Messages, mergedVars, key)
: (defaultValue.Messages ?? new List<LdAiConfigTypes.Message>());
var evaluator = BuildEvaluator(defaultValue.JudgeConfiguration, context, variables);
return new LdAiCompletionConfig(
key,
defaultValue.Enabled ?? true,
Expand All @@ -100,7 +109,8 @@ private LdAiCompletionConfig BuildCompletionFromDefault(
defaultValue.JudgeConfiguration,
defaultValue.Model,
defaultValue.Provider,
trackerFactory);
trackerFactory,
evaluator);
}

public LdAiAgentConfig BuildAgentConfig(
Expand All @@ -120,7 +130,7 @@ public LdAiAgentConfig BuildAgentConfig(
_logger.Error(
"AI Config '{0}': variation result is not an object (got {1}); using caller's default.",
key, ldValue.Type);
return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate);
return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate, graphKey);
}

var (enabled, variationKey, version, mode) = ParseMeta(ldValue);
Expand All @@ -130,7 +140,7 @@ public LdAiAgentConfig BuildAgentConfig(
_logger.Warn(
"AI Config mode mismatch for {0}: expected {1}, got {2}. Returning caller's default.",
key, LdAiAgentConfig.Mode, mode);
return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, interpolate);
return BuildAgentFromDefault(key, defaultValue, mergedVars, trackerFactory, context, variables, interpolate, graphKey);
}

var model = ParseModel(ldValue.Get("model"));
Expand All @@ -140,6 +150,7 @@ public LdAiAgentConfig BuildAgentConfig(
? InterpolateInstructions(ParseInstructions(ldValue.Get("instructions")), mergedVars, key)
: ParseInstructions(ldValue.Get("instructions"));
var judgeConfiguration = ParseJudgeConfiguration(ldValue.Get("judgeConfiguration"));
var evaluator = BuildEvaluator(judgeConfiguration, context, variables, graphKey);

return new LdAiAgentConfig(
key,
Expand All @@ -151,19 +162,24 @@ public LdAiAgentConfig BuildAgentConfig(
model,
provider,
judgeConfiguration,
trackerFactory);
trackerFactory,
evaluator);
}

internal LdAiAgentConfig BuildAgentFromDefault(
string key,
LdAiAgentConfigDefault defaultValue,
IReadOnlyDictionary<string, object> mergedVars,
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory,
bool interpolate = true)
Context context,
IReadOnlyDictionary<string, object> variables = null,
bool interpolate = true,
string graphKey = null)
{
var instructions = interpolate
? InterpolateInstructions(defaultValue.Instructions, mergedVars, key)
: defaultValue.Instructions;
var evaluator = BuildEvaluator(defaultValue.JudgeConfiguration, context, variables, graphKey);
return new LdAiAgentConfig(
key,
defaultValue.Enabled ?? true,
Expand All @@ -174,7 +190,8 @@ internal LdAiAgentConfig BuildAgentFromDefault(
defaultValue.Model,
defaultValue.Provider,
defaultValue.JudgeConfiguration,
trackerFactory);
trackerFactory,
evaluator);
}

public LdAiJudgeConfig BuildJudgeConfig(
Expand All @@ -183,10 +200,11 @@ public LdAiJudgeConfig BuildJudgeConfig(
Context context,
LdAiJudgeConfigDefault defaultValue,
IReadOnlyDictionary<string, object> variables,
bool interpolate = true)
bool interpolate = true,
string graphKey = null)
{
var mergedVars = interpolate ? MergeVariables(variables, context) : null;
var trackerFactory = TrackerFactoryFor(context);
var trackerFactory = TrackerFactoryFor(context, graphKey);

if (ldValue.Type != LdValueType.Object)
{
Expand Down Expand Up @@ -247,6 +265,49 @@ private LdAiJudgeConfig BuildJudgeFromDefault(
trackerFactory);
}

private Evaluator BuildEvaluator(LdAiConfigTypes.JudgeConfiguration judgeConfiguration, Context context,
IReadOnlyDictionary<string, object> variables = null, string graphKey = null)
{
if (_runnerFactory == null || judgeConfiguration == null || judgeConfiguration.Judges.Count == 0)
{
return Evaluator.Noop();
}

var judges = new Dictionary<string, Judge>();
foreach (var judgeEntry in judgeConfiguration.Judges)
{
try
{
var defaultValue = LdAiJudgeConfigDefault.Disabled;
var ldValue = _client.JsonVariation(judgeEntry.Key, context, defaultValue.ToLdValue());
var judgeConfig = BuildJudgeConfig(judgeEntry.Key, ldValue, context, defaultValue, variables, graphKey: graphKey);

@cursor cursor Bot Jul 1, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Judge sampling ignores interpolate flag

Medium Severity

BuildEvaluator always calls BuildJudgeConfig with the default interpolate: true, even when the parent config was built with interpolate: false (template APIs). Nested judge configs then run Mustache interpolation (including ldctx) while completion/agent instructions and messages stay as raw templates, so template configs and runnerFactory output can disagree with the documented template behavior.

Additional Locations (2)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 6f7d88d. Configure here.

@mattrmc1 mattrmc1 Jul 6, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not a bug. Attached judges are operational -- they execute against a model provider via IRunner.RunAsync and require interpolated prompts to function. A judge receiving raw {{variable}} tokens would produce nonsense. When fetching a judge directly with interpolation: false, the raw {{variable}} tokens will return correctly

@mattrmc1 mattrmc1 Jul 6, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Messages when fetching toxicity judge as template:

[
    {
        "content": "You are a business information accuracy and safety expert. Evaluate....",
        "role": "system"
    },
    {
        "content": "MESSAGE HISTORY:\n{{message_history}}\n",
        "role": "assistant"
    },
    {
        "content": "RESPONSE TO EVALUATE:\n{{response_to_evaluate}}\n",
        "role": "user"
    }
]


if (!judgeConfig.Enabled)
{
_logger?.Debug("Judge '{0}' is disabled; skipping.", judgeEntry.Key);
continue;
Comment thread
cursor[bot] marked this conversation as resolved.
}

var runner = _runnerFactory(judgeConfig);
if (runner == null)
{
_logger?.Warn("Runner factory returned null for judge '{0}'; skipping.", judgeEntry.Key);
continue;
}

judges[judgeEntry.Key] = new Judge(judgeConfig, runner, _logger);
}
catch (Exception ex)
{
_logger?.Warn("Failed to initialize judge '{0}': {1}", judgeEntry.Key, ex.Message);
}
}

var filteredConfig = new LdAiConfigTypes.JudgeConfiguration(
judgeConfiguration.Judges.Where(j => judges.ContainsKey(j.Key)).ToList());
return new Evaluator(judges, filteredConfig, _logger);
}

private string InterpolateInstructions(
string instructions,
IReadOnlyDictionary<string, object> mergedVars,
Expand Down Expand Up @@ -401,9 +462,13 @@ internal static LdAiConfigTypes.JudgeConfiguration ParseJudgeConfiguration(LdVal
{
var j = judgesArray.Get(i);
if (j.Type != LdValueType.Object) continue;
var samplingRateValue = j.Get("samplingRate");
double? samplingRate = samplingRateValue.Type == LdValueType.Number
? samplingRateValue.AsDouble
: (double?)null;
entries.Add(new LdAiConfigTypes.JudgeConfiguration.Judge(
j.Get("key").AsString ?? "",
j.Get("samplingRate").AsDouble));
samplingRate));
}
return new LdAiConfigTypes.JudgeConfiguration(entries);
}
Expand Down
6 changes: 4 additions & 2 deletions pkgs/sdk/server-ai/src/Config/LdAiAgentConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using LaunchDarkly.Sdk.Server.Ai.Evals;
using LaunchDarkly.Sdk.Server.Ai.Interfaces;

namespace LaunchDarkly.Sdk.Server.Ai.Config;
Expand Down Expand Up @@ -46,8 +47,9 @@ internal LdAiAgentConfig(
LdAiConfigTypes.ModelConfig model,
LdAiConfigTypes.ProviderConfig provider,
LdAiConfigTypes.JudgeConfiguration judgeConfiguration,
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory)
: base(key, enabled, variationKey, version, model, provider, trackerFactory)
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory,
Evaluator evaluator = null)
: base(key, enabled, variationKey, version, model, provider, trackerFactory, evaluator)
{
Instructions = instructions;
Tools = tools?.ToImmutableDictionary() ?? ImmutableDictionary<string, LdAiConfigTypes.Tool>.Empty;
Expand Down
12 changes: 8 additions & 4 deletions pkgs/sdk/server-ai/src/Config/LdAiAgentConfigDefault.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,11 +192,15 @@ internal LdValue ToLdValue()
root["judgeConfiguration"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
{ "judges", LdValue.ArrayFrom(JudgeConfiguration.Judges.Select(j =>
LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
{ "key", LdValue.Of(j.Key) },
{ "samplingRate", LdValue.Of(j.SamplingRate) }
})))
var judgeFields = new Dictionary<string, LdValue>
{
{ "key", LdValue.Of(j.Key) }
};
if (j.SamplingRate.HasValue)
judgeFields["samplingRate"] = LdValue.Of(j.SamplingRate.Value);
return LdValue.ObjectFrom(judgeFields);
}))
}
});
}
Expand Down
6 changes: 4 additions & 2 deletions pkgs/sdk/server-ai/src/Config/LdAiCompletionConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using LaunchDarkly.Sdk.Server.Ai.Evals;
using LaunchDarkly.Sdk.Server.Ai.Interfaces;

namespace LaunchDarkly.Sdk.Server.Ai.Config;
Expand Down Expand Up @@ -42,8 +43,9 @@ internal LdAiCompletionConfig(string key, bool enabled, string variationKey, int
IEnumerable<LdAiConfigTypes.Message> messages, IReadOnlyDictionary<string, LdAiConfigTypes.Tool> tools,
LdAiConfigTypes.JudgeConfiguration judgeConfiguration,
LdAiConfigTypes.ModelConfig model, LdAiConfigTypes.ProviderConfig provider,
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory)
: base(key, enabled, variationKey, version, model, provider, trackerFactory)
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory,
Evaluator evaluator = null)
: base(key, enabled, variationKey, version, model, provider, trackerFactory, evaluator)
{
Messages = messages?.ToList() ?? new List<LdAiConfigTypes.Message>();
Tools = tools ?? ImmutableDictionary<string, LdAiConfigTypes.Tool>.Empty;
Expand Down
12 changes: 8 additions & 4 deletions pkgs/sdk/server-ai/src/Config/LdAiCompletionConfigDefault.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,11 +198,15 @@ internal LdValue ToLdValue()
root["judgeConfiguration"] = LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
{ "judges", LdValue.ArrayFrom(JudgeConfiguration.Judges.Select(j =>
LdValue.ObjectFrom(new Dictionary<string, LdValue>
{
{ "key", LdValue.Of(j.Key) },
{ "samplingRate", LdValue.Of(j.SamplingRate) }
})))
var judgeFields = new Dictionary<string, LdValue>
{
{ "key", LdValue.Of(j.Key) }
};
if (j.SamplingRate.HasValue)
judgeFields["samplingRate"] = LdValue.Of(j.SamplingRate.Value);
return LdValue.ObjectFrom(judgeFields);
}))
}
});
}
Expand Down
11 changes: 10 additions & 1 deletion pkgs/sdk/server-ai/src/Config/LdAiConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using LaunchDarkly.Sdk.Server.Ai.Evals;
using LaunchDarkly.Sdk.Server.Ai.Interfaces;

namespace LaunchDarkly.Sdk.Server.Ai.Config;
Expand Down Expand Up @@ -40,6 +41,12 @@ public abstract class LdAiConfig
/// </summary>
internal int Version { get; }

/// <summary>
/// The evaluator attached to this config. Always non-null; use <see cref="Evaluator.Noop"/>
/// when no real evaluation is configured.
/// </summary>
public Evaluator Evaluator { get; }

/// <summary>
/// Factory that produces a tracker for the config. The factory is mode-agnostic — it
/// operates only on the shared fields (<see cref="Key"/>, <see cref="Model"/>,
Expand All @@ -66,7 +73,8 @@ private protected LdAiConfig(
int version,
LdAiConfigTypes.ModelConfig model,
LdAiConfigTypes.ProviderConfig provider,
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory)
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory,
Evaluator evaluator = null)
{
Key = key;
Enabled = enabled;
Expand All @@ -75,5 +83,6 @@ private protected LdAiConfig(
Model = model ?? new LdAiConfigTypes.ModelConfig("", new Dictionary<string, LdValue>(), new Dictionary<string, LdValue>());
Provider = provider ?? new LdAiConfigTypes.ProviderConfig("");
_trackerFactory = trackerFactory ?? throw new ArgumentNullException(nameof(trackerFactory));
Evaluator = evaluator ?? Evals.Evaluator.Noop();
}
}
5 changes: 3 additions & 2 deletions pkgs/sdk/server-ai/src/Config/LdAiConfigTypes.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,10 +161,11 @@ public sealed class Judge

/// <summary>
/// The fraction of requests that this judge evaluates, in the range [0, 1].
/// When null, the judge evaluates 100% of requests.
/// </summary>
public double SamplingRate { get; }
public double? SamplingRate { get; }

internal Judge(string key, double samplingRate)
internal Judge(string key, double? samplingRate)
{
Key = key;
SamplingRate = samplingRate;
Expand Down
3 changes: 2 additions & 1 deletion pkgs/sdk/server-ai/src/Config/LdAiJudgeConfig.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using LaunchDarkly.Sdk.Server.Ai.Evals;
using LaunchDarkly.Sdk.Server.Ai.Interfaces;

namespace LaunchDarkly.Sdk.Server.Ai.Config;
Expand Down Expand Up @@ -41,7 +42,7 @@ internal LdAiJudgeConfig(
LdAiConfigTypes.ModelConfig model,
LdAiConfigTypes.ProviderConfig provider,
Func<LdAiConfig, ILdAiConfigTracker> trackerFactory)
: base(key, enabled, variationKey, version, model, provider, trackerFactory)
: base(key, enabled, variationKey, version, model, provider, trackerFactory, Evaluator.Noop())
{
Messages = messages?.ToList() ?? new List<LdAiConfigTypes.Message>();
EvaluationMetricKey = evaluationMetricKey;
Expand Down
Loading
Loading