From ea9f2d663752599dae5e4de440f1c5555ccd888b Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 13:20:34 -0500 Subject: [PATCH 1/6] feat: simplify evaluation schema to flat score/reasoning shape Co-Authored-By: Claude Opus 4.6 --- .../sdk/server-ai/__tests__/Judge.test.ts | 224 +++++------------- .../src/api/judge/EvaluationSchemaBuilder.ts | 36 +-- packages/sdk/server-ai/src/api/judge/Judge.ts | 44 +--- 3 files changed, 77 insertions(+), 227 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/Judge.test.ts b/packages/sdk/server-ai/__tests__/Judge.test.ts index ee9ff0351f..deec6b03ac 100644 --- a/packages/sdk/server-ai/__tests__/Judge.test.ts +++ b/packages/sdk/server-ai/__tests__/Judge.test.ts @@ -70,14 +70,12 @@ describe('Judge', () => { it('evaluates AI response successfully', async () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant to the question' }, - }, + score: 0.8, + reasoning: 'The response is relevant to the question', }, rawResponse: JSON.stringify({ - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant to the question' }, - }, + score: 0.8, + reasoning: 'The response is relevant to the question', }), metrics: { success: true, @@ -125,14 +123,12 @@ describe('Judge', () => { it('returns evaluation result with correct evaluationMetricKey for tracker integration', async () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - relevance: { score: 0.85, reasoning: 'Highly relevant response' }, - }, + score: 0.85, + reasoning: 'Highly relevant response', }, rawResponse: JSON.stringify({ - evaluations: { - relevance: { score: 0.85, reasoning: 'Highly relevant response' }, - }, + score: 0.85, + reasoning: 'Highly relevant response', }), metrics: { success: true, @@ -159,14 +155,12 @@ describe('Judge', () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - relevance: { score: 0.8, reasoning: 'Good' }, - }, + score: 0.8, + reasoning: 'Good', }, rawResponse: JSON.stringify({ - evaluations: { - relevance: { score: 0.8, reasoning: 'Good' }, - }, + score: 0.8, + reasoning: 'Good', }), metrics: { success: true, @@ -237,14 +231,12 @@ describe('Judge', () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant' }, - }, + score: 0.8, + reasoning: 'The response is relevant', }, rawResponse: JSON.stringify({ - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant' }, - }, + score: 0.8, + reasoning: 'The response is relevant', }), metrics: { success: true, @@ -277,14 +269,12 @@ describe('Judge', () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant' }, - }, + score: 0.8, + reasoning: 'The response is relevant', }, rawResponse: JSON.stringify({ - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant' }, - }, + score: 0.8, + reasoning: 'The response is relevant', }), metrics: { success: true, @@ -317,14 +307,12 @@ describe('Judge', () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant' }, - }, + score: 0.8, + reasoning: 'The response is relevant', }, rawResponse: JSON.stringify({ - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant' }, - }, + score: 0.8, + reasoning: 'The response is relevant', }), metrics: { success: true, @@ -358,14 +346,12 @@ describe('Judge', () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - helpfulness: { score: 0.7, reasoning: 'The response is helpful' }, - }, + score: 0.7, + reasoning: 'The response is helpful', }, rawResponse: JSON.stringify({ - evaluations: { - helpfulness: { score: 0.7, reasoning: 'The response is helpful' }, - }, + score: 0.7, + reasoning: 'The response is helpful', }), metrics: { success: true, @@ -409,18 +395,10 @@ describe('Judge', () => { ); }); - it('returns result with success false when expected metric is missing', async () => { + it('returns result with success false when response has no score or reasoning', async () => { const mockStructuredResponse: StructuredResponse = { - data: { - evaluations: { - accuracy: { score: 0.9, reasoning: 'Accurate' }, - }, - }, - rawResponse: JSON.stringify({ - evaluations: { - accuracy: { score: 0.9, reasoning: 'Accurate' }, - }, - }), + data: {}, + rawResponse: '{}', metrics: { success: true, usage: { total: 100, input: 50, output: 50 }, @@ -442,14 +420,14 @@ describe('Judge', () => { it('returns result with success false when response structure is malformed', async () => { const mockStructuredResponse: StructuredResponse = { data: { - relevance: { score: 0.8, reasoning: 'Good' }, - accuracy: { score: 0.9, reasoning: 'Accurate' }, - helpfulness: { score: 0.7, reasoning: 'Helpful' }, + evaluations: { + relevance: { score: 0.8, reasoning: 'Good' }, + }, }, rawResponse: JSON.stringify({ - relevance: { score: 0.8, reasoning: 'Good' }, - accuracy: { score: 0.9, reasoning: 'Accurate' }, - helpfulness: { score: 0.7, reasoning: 'Helpful' }, + evaluations: { + relevance: { score: 0.8, reasoning: 'Good' }, + }, }), metrics: { success: true, @@ -517,14 +495,12 @@ describe('Judge', () => { const mockStructuredResponse: StructuredResponse = { data: { - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant to the question' }, - }, + score: 0.8, + reasoning: 'The response is relevant to the question', }, rawResponse: JSON.stringify({ - evaluations: { - relevance: { score: 0.8, reasoning: 'The response is relevant to the question' }, - }, + score: 0.8, + reasoning: 'The response is relevant to the question', }), metrics: { success: true, @@ -620,13 +596,9 @@ describe('Judge', () => { it('parses valid evaluation response correctly', () => { // eslint-disable-next-line no-underscore-dangle const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - relevance: { score: 0.8, reasoning: 'Good' }, - }, - }; + const responseData = { score: 0.8, reasoning: 'Good' }; - const result = parseResponse(responseData, 'relevance', mockTracker); + const result = parseResponse(responseData); expect(result).toEqual({ score: 0.8, @@ -634,28 +606,21 @@ describe('Judge', () => { }); }); - it('returns undefined for invalid response data', () => { + it('returns undefined for empty response data', () => { // eslint-disable-next-line no-underscore-dangle const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - relevance: { score: 0.8, reasoning: 'Good' }, - }; - const result = parseResponse(responseData, 'relevance', mockTracker); + const result = parseResponse({}); expect(result).toBeUndefined(); }); - it('handles missing score or reasoning fields', () => { + it('handles missing reasoning field', () => { // eslint-disable-next-line no-underscore-dangle const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - relevance: { score: 0.8 }, - }, - }; + const responseData = { score: 0.8 }; - const result = parseResponse(responseData, 'relevance', mockTracker); + const result = parseResponse(responseData); expect(result).toBeUndefined(); }); @@ -663,73 +628,34 @@ describe('Judge', () => { it('handles invalid score values out of range', () => { // eslint-disable-next-line no-underscore-dangle const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - relevance: { score: 1.5, reasoning: 'Good' }, - }, - }; + const responseData = { score: 1.5, reasoning: 'Good' }; - const result = parseResponse(responseData, 'relevance', mockTracker); + const result = parseResponse(responseData); expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith( - expect.stringContaining('Invalid score evaluated for relevance: 1.5'), - mockTrackData, - ); + expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Invalid score: 1.5')); }); it('handles negative score values', () => { // eslint-disable-next-line no-underscore-dangle const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - relevance: { score: -0.1, reasoning: 'Good' }, - }, - }; + const responseData = { score: -0.1, reasoning: 'Good' }; - const result = parseResponse(responseData, 'relevance', mockTracker); + const result = parseResponse(responseData); expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith( - expect.stringContaining('Invalid score evaluated for relevance: -0.1'), - mockTrackData, - ); + expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Invalid score: -0.1')); }); it('handles invalid reasoning type', () => { // eslint-disable-next-line no-underscore-dangle const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - relevance: { score: 0.8, reasoning: 123 }, - }, - }; - - const result = parseResponse(responseData, 'relevance', mockTracker); - - expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith( - expect.stringContaining('Invalid reasoning evaluated for relevance: 123'), - mockTrackData, - ); - }); - - it('handles missing evaluation when key does not exist in response', () => { - // eslint-disable-next-line no-underscore-dangle - const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - accuracy: { score: 0.9, reasoning: 'Accurate' }, - }, - }; + const responseData = { score: 0.8, reasoning: 123 }; - const result = parseResponse(responseData, 'relevance', mockTracker); + const result = parseResponse(responseData); expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith( - 'Missing evaluation for metric key: relevance', - mockTrackData, - ); + expect(mockLogger.warn).toHaveBeenCalledWith('Invalid reasoning: must be a string'); }); it('handles empty evaluationMetricKeys array fallback', async () => { @@ -753,41 +679,5 @@ describe('Judge', () => { mockTrackData, ); }); - - it('handles evaluation value that is not an object', () => { - // eslint-disable-next-line no-underscore-dangle - const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - relevance: 'not an object', - }, - }; - - const result = parseResponse(responseData, 'relevance', mockTracker); - - expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith( - 'Missing evaluation for metric key: relevance', - mockTrackData, - ); - }); - - it('handles null evaluation value', () => { - // eslint-disable-next-line no-underscore-dangle - const parseResponse = (judge as any)._parseEvaluationResponse.bind(judge); - const responseData = { - evaluations: { - relevance: null, - }, - }; - - const result = parseResponse(responseData, 'relevance', mockTracker); - - expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith( - 'Missing evaluation for metric key: relevance', - mockTrackData, - ); - }); }); }); diff --git a/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts b/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts index 06f745a418..99a63cc366 100644 --- a/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts +++ b/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts @@ -1,31 +1,13 @@ /** - * Internal class for building dynamic evaluation response schemas. - * Not exported - only used internally by TrackedJudge. + * Internal class for building evaluation response schemas. + * Not exported - only used internally by Judge. + * + * The schema is a fixed shape: top-level score and reasoning. + * The judge config's evaluationMetricKey is only used when keying the result, + * not in the schema itself. */ class EvaluationSchemaBuilder { - static build(evaluationMetricKey?: string): Record { - if (!evaluationMetricKey) { - return {}; - } - return { - type: 'object', - properties: { - evaluations: { - type: 'object', - description: `Object containing evaluation results for ${evaluationMetricKey} metric`, - properties: { - [evaluationMetricKey]: this._buildKeySchema(evaluationMetricKey), - }, - required: [evaluationMetricKey], - additionalProperties: false, - }, - }, - required: ['evaluations'], - additionalProperties: false, - } as const; - } - - private static _buildKeySchema(key: string) { + static build(): Record { return { type: 'object', properties: { @@ -33,11 +15,11 @@ class EvaluationSchemaBuilder { type: 'number', minimum: 0, maximum: 1, - description: `Score between 0.0 and 1.0 for ${key}`, + description: 'Score between 0.0 and 1.0.', }, reasoning: { type: 'string', - description: `Reasoning behind the score for ${key}`, + description: 'Reasoning behind the score.', }, }, required: ['score', 'reasoning'], diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index e36ab138cd..c446798af9 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -3,7 +3,6 @@ import Mustache from 'mustache'; import { LDLogger } from '@launchdarkly/js-server-sdk-common'; import { ChatResponse } from '../chat/types'; -import { LDAIConfigTracker } from '../config/LDAIConfigTracker'; import { LDAIJudgeConfig, LDMessage } from '../config/types'; import { AIProvider } from '../providers/AIProvider'; import { EvaluationSchemaBuilder } from './EvaluationSchemaBuilder'; @@ -25,8 +24,7 @@ export class Judge { logger?: LDLogger, ) { this._logger = logger; - const evaluationMetricKey = this._getEvaluationMetricKey(); - this._evaluationResponseStructure = EvaluationSchemaBuilder.build(evaluationMetricKey); + this._evaluationResponseStructure = EvaluationSchemaBuilder.build(); } /** @@ -100,7 +98,7 @@ export class Judge { () => this._aiProvider.invokeStructuredModel(messages, this._evaluationResponseStructure), ); - const evalResult = this._parseEvaluationResponse(response.data, evaluationMetricKey, tracker); + const evalResult = this._parseEvaluationResponse(response.data); if (!evalResult) { this._logger?.warn( @@ -181,52 +179,32 @@ export class Judge { } /** - * Parses the structured evaluation response from the AI provider. + * Parses the structured evaluation response. Expects top-level {score, reasoning}. * Returns score and reasoning, or undefined if parsing fails. */ private _parseEvaluationResponse( data: Record, - evaluationMetricKey: string, - tracker: LDAIConfigTracker, ): { score: number; reasoning: string } | undefined { - const evaluations = data.evaluations as Record; - - if (!data.evaluations || typeof data.evaluations !== 'object') { - this._logger?.warn('Invalid response: missing or invalid evaluations object'); - return undefined; - } - - const evaluation = evaluations[evaluationMetricKey]; - - if (!evaluation || typeof evaluation !== 'object') { - this._logger?.warn( - `Missing evaluation for metric key: ${evaluationMetricKey}`, - tracker.getTrackData(), - ); + if (!data || typeof data !== 'object') { + this._logger?.warn('Invalid response: missing or invalid evaluation'); return undefined; } - const evalData = evaluation as Record; - - if (typeof evalData.score !== 'number' || evalData.score < 0 || evalData.score > 1) { + if (typeof data.score !== 'number' || data.score < 0 || data.score > 1) { this._logger?.warn( - `Invalid score evaluated for ${evaluationMetricKey}: ${evalData.score}. Score must be a number between 0 and 1 inclusive`, - tracker.getTrackData(), + `Invalid score: ${data.score}. Score must be a number between 0 and 1 inclusive`, ); return undefined; } - if (typeof evalData.reasoning !== 'string') { - this._logger?.warn( - `Invalid reasoning evaluated for ${evaluationMetricKey}: ${evalData.reasoning}. Reasoning must be a string`, - tracker.getTrackData(), - ); + if (typeof data.reasoning !== 'string') { + this._logger?.warn('Invalid reasoning: must be a string'); return undefined; } return { - score: evalData.score, - reasoning: evalData.reasoning, + score: data.score, + reasoning: data.reasoning, }; } } From f3a8bf34792d476f17507f4da2ec8a412e898740 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 16:07:06 -0500 Subject: [PATCH 2/6] chore: remove unnecessary comment from EvaluationSchemaBuilder Co-Authored-By: Claude Opus 4.6 --- packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts b/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts index 99a63cc366..590c421282 100644 --- a/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts +++ b/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts @@ -3,8 +3,6 @@ * Not exported - only used internally by Judge. * * The schema is a fixed shape: top-level score and reasoning. - * The judge config's evaluationMetricKey is only used when keying the result, - * not in the schema itself. */ class EvaluationSchemaBuilder { static build(): Record { From d81b20216a4081376a2d7be9a5ca4d5ab7206373 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Thu, 16 Apr 2026 16:54:21 -0500 Subject: [PATCH 3/6] refactor: inline evaluation schema as const, consolidate parse warning Delete EvaluationSchemaBuilder.ts and define EVALUATION_SCHEMA as a module-level const in Judge.ts. Remove per-field warnings from _parseEvaluationResponse (keep it pure) and emit a single warning in evaluate() that includes the judge key and raw response data. Co-Authored-By: Claude Opus 4.6 --- .../sdk/server-ai/__tests__/Judge.test.ts | 9 ++++-- .../src/api/judge/EvaluationSchemaBuilder.ts | 29 ----------------- packages/sdk/server-ai/src/api/judge/Judge.ts | 31 ++++++++++++------- 3 files changed, 26 insertions(+), 43 deletions(-) delete mode 100644 packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts diff --git a/packages/sdk/server-ai/__tests__/Judge.test.ts b/packages/sdk/server-ai/__tests__/Judge.test.ts index deec6b03ac..3a9acbad88 100644 --- a/packages/sdk/server-ai/__tests__/Judge.test.ts +++ b/packages/sdk/server-ai/__tests__/Judge.test.ts @@ -415,6 +415,9 @@ describe('Judge', () => { sampled: true, judgeConfigKey: 'test-judge', }); + expect(mockLogger.warn).toHaveBeenCalledWith( + 'Could not parse evaluation response for judge "test-judge": {}', + ); }); it('returns result with success false when response structure is malformed', async () => { @@ -445,6 +448,9 @@ describe('Judge', () => { sampled: true, judgeConfigKey: 'test-judge', }); + expect(mockLogger.warn).toHaveBeenCalledWith( + expect.stringContaining('Could not parse evaluation response for judge "test-judge"'), + ); }); it('handles provider errors gracefully', async () => { @@ -633,7 +639,6 @@ describe('Judge', () => { const result = parseResponse(responseData); expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Invalid score: 1.5')); }); it('handles negative score values', () => { @@ -644,7 +649,6 @@ describe('Judge', () => { const result = parseResponse(responseData); expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith(expect.stringContaining('Invalid score: -0.1')); }); it('handles invalid reasoning type', () => { @@ -655,7 +659,6 @@ describe('Judge', () => { const result = parseResponse(responseData); expect(result).toBeUndefined(); - expect(mockLogger.warn).toHaveBeenCalledWith('Invalid reasoning: must be a string'); }); it('handles empty evaluationMetricKeys array fallback', async () => { diff --git a/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts b/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts deleted file mode 100644 index 590c421282..0000000000 --- a/packages/sdk/server-ai/src/api/judge/EvaluationSchemaBuilder.ts +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Internal class for building evaluation response schemas. - * Not exported - only used internally by Judge. - * - * The schema is a fixed shape: top-level score and reasoning. - */ -class EvaluationSchemaBuilder { - static build(): Record { - return { - type: 'object', - properties: { - score: { - type: 'number', - minimum: 0, - maximum: 1, - description: 'Score between 0.0 and 1.0.', - }, - reasoning: { - type: 'string', - description: 'Reasoning behind the score.', - }, - }, - required: ['score', 'reasoning'], - additionalProperties: false, - }; - } -} - -export { EvaluationSchemaBuilder }; diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index c446798af9..b9ffc59916 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -5,9 +5,26 @@ import { LDLogger } from '@launchdarkly/js-server-sdk-common'; import { ChatResponse } from '../chat/types'; import { LDAIJudgeConfig, LDMessage } from '../config/types'; import { AIProvider } from '../providers/AIProvider'; -import { EvaluationSchemaBuilder } from './EvaluationSchemaBuilder'; import { LDJudgeResult, StructuredResponse } from './types'; +const EVALUATION_SCHEMA: Record = { + type: 'object', + properties: { + score: { + type: 'number', + minimum: 0, + maximum: 1, + description: 'Score between 0.0 and 1.0.', + }, + reasoning: { + type: 'string', + description: 'Reasoning behind the score.', + }, + }, + required: ['score', 'reasoning'], + additionalProperties: false, +}; + /** * Judge implementation that handles evaluation functionality and conversation management. * @@ -16,7 +33,6 @@ import { LDJudgeResult, StructuredResponse } from './types'; */ export class Judge { private readonly _logger?: LDLogger; - private readonly _evaluationResponseStructure: Record; constructor( private readonly _aiConfig: LDAIJudgeConfig, @@ -24,7 +40,6 @@ export class Judge { logger?: LDLogger, ) { this._logger = logger; - this._evaluationResponseStructure = EvaluationSchemaBuilder.build(); } /** @@ -95,15 +110,14 @@ export class Judge { const response = await tracker.trackMetricsOf( (r: StructuredResponse) => r.metrics, - () => this._aiProvider.invokeStructuredModel(messages, this._evaluationResponseStructure), + () => this._aiProvider.invokeStructuredModel(messages, EVALUATION_SCHEMA), ); const evalResult = this._parseEvaluationResponse(response.data); if (!evalResult) { this._logger?.warn( - 'Judge evaluation did not return the expected evaluation', - tracker.getTrackData(), + `Could not parse evaluation response for judge "${this._aiConfig.key}": ${JSON.stringify(response.data)}`, ); return result; } @@ -186,19 +200,14 @@ export class Judge { data: Record, ): { score: number; reasoning: string } | undefined { if (!data || typeof data !== 'object') { - this._logger?.warn('Invalid response: missing or invalid evaluation'); return undefined; } if (typeof data.score !== 'number' || data.score < 0 || data.score > 1) { - this._logger?.warn( - `Invalid score: ${data.score}. Score must be a number between 0 and 1 inclusive`, - ); return undefined; } if (typeof data.reasoning !== 'string') { - this._logger?.warn('Invalid reasoning: must be a string'); return undefined; } From e115ec818c6816179d1a7ad396a731b9cd3391e7 Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 09:47:36 -0500 Subject: [PATCH 4/6] fix: include tracker data in parse-failure warning Co-Authored-By: Claude Opus 4.6 --- packages/sdk/server-ai/__tests__/Judge.test.ts | 2 ++ packages/sdk/server-ai/src/api/judge/Judge.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/packages/sdk/server-ai/__tests__/Judge.test.ts b/packages/sdk/server-ai/__tests__/Judge.test.ts index 3a9acbad88..aaaf9fc171 100644 --- a/packages/sdk/server-ai/__tests__/Judge.test.ts +++ b/packages/sdk/server-ai/__tests__/Judge.test.ts @@ -417,6 +417,7 @@ describe('Judge', () => { }); expect(mockLogger.warn).toHaveBeenCalledWith( 'Could not parse evaluation response for judge "test-judge": {}', + mockTrackData, ); }); @@ -450,6 +451,7 @@ describe('Judge', () => { }); expect(mockLogger.warn).toHaveBeenCalledWith( expect.stringContaining('Could not parse evaluation response for judge "test-judge"'), + mockTrackData, ); }); diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index b9ffc59916..71ccaea402 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -118,6 +118,7 @@ export class Judge { if (!evalResult) { this._logger?.warn( `Could not parse evaluation response for judge "${this._aiConfig.key}": ${JSON.stringify(response.data)}`, + tracker.getTrackData(), ); return result; } From b62677304f8531e9be3fa56f5c2ea8f2e645739f Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 09:54:33 -0500 Subject: [PATCH 5/6] chore: remove redundant judge key from parse-failure warning configKey is already present in tracker.getTrackData(). Co-Authored-By: Claude Opus 4.6 --- packages/sdk/server-ai/__tests__/Judge.test.ts | 4 ++-- packages/sdk/server-ai/src/api/judge/Judge.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/Judge.test.ts b/packages/sdk/server-ai/__tests__/Judge.test.ts index aaaf9fc171..43ea75e0ab 100644 --- a/packages/sdk/server-ai/__tests__/Judge.test.ts +++ b/packages/sdk/server-ai/__tests__/Judge.test.ts @@ -416,7 +416,7 @@ describe('Judge', () => { judgeConfigKey: 'test-judge', }); expect(mockLogger.warn).toHaveBeenCalledWith( - 'Could not parse evaluation response for judge "test-judge": {}', + 'Could not parse evaluation response: {}', mockTrackData, ); }); @@ -450,7 +450,7 @@ describe('Judge', () => { judgeConfigKey: 'test-judge', }); expect(mockLogger.warn).toHaveBeenCalledWith( - expect.stringContaining('Could not parse evaluation response for judge "test-judge"'), + expect.stringContaining('Could not parse evaluation response:'), mockTrackData, ); }); diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index 71ccaea402..a92489ad7d 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -117,7 +117,7 @@ export class Judge { if (!evalResult) { this._logger?.warn( - `Could not parse evaluation response for judge "${this._aiConfig.key}": ${JSON.stringify(response.data)}`, + `Could not parse evaluation response: ${JSON.stringify(response.data)}`, tracker.getTrackData(), ); return result; From 013a80d86c370f8221753e18a824346a54e4011b Mon Sep 17 00:00:00 2001 From: jsonbailey Date: Fri, 17 Apr 2026 11:24:05 -0500 Subject: [PATCH 6/6] chore: use as const for schema, add array guard in parse Address review nits: narrow EVALUATION_SCHEMA type with as const instead of Record, and add Array.isArray check in _parseEvaluationResponse. Co-Authored-By: Claude Opus 4.6 --- packages/sdk/server-ai/src/api/judge/Judge.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/sdk/server-ai/src/api/judge/Judge.ts b/packages/sdk/server-ai/src/api/judge/Judge.ts index a92489ad7d..ef49e3b723 100644 --- a/packages/sdk/server-ai/src/api/judge/Judge.ts +++ b/packages/sdk/server-ai/src/api/judge/Judge.ts @@ -7,7 +7,7 @@ import { LDAIJudgeConfig, LDMessage } from '../config/types'; import { AIProvider } from '../providers/AIProvider'; import { LDJudgeResult, StructuredResponse } from './types'; -const EVALUATION_SCHEMA: Record = { +const EVALUATION_SCHEMA = { type: 'object', properties: { score: { @@ -23,7 +23,7 @@ const EVALUATION_SCHEMA: Record = { }, required: ['score', 'reasoning'], additionalProperties: false, -}; +} as const; /** * Judge implementation that handles evaluation functionality and conversation management. @@ -200,7 +200,7 @@ export class Judge { private _parseEvaluationResponse( data: Record, ): { score: number; reasoning: string } | undefined { - if (!data || typeof data !== 'object') { + if (!data || typeof data !== 'object' || Array.isArray(data)) { return undefined; }