From cd7e3eb5ad2857cf350535d262c528ff0a18ee44 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 8 Jul 2026 04:19:23 +0000 Subject: [PATCH 1/3] feat(server-ai): stamp modelKey and modelVersion on AI usage events (AIC-2858) Read modelKey and modelVersion from the AI Config variation payload and stamp them on emitted metric events alongside existing modelName and providerName fields. Default modelVersion to 1 when absent and exclude both fields from the resumption token. Additive and backward compatible. Co-authored-by: Anthony Torns II --- .../__tests__/LDAIClientImpl.test.ts | 85 ++++++++++++++ .../__tests__/LDAIConfigTrackerImpl.test.ts | 106 ++++++++++++++++++ packages/sdk/server-ai/src/LDAIClientImpl.ts | 2 + .../server-ai/src/LDAIConfigTrackerImpl.ts | 6 + .../src/api/config/LDAIConfigTracker.ts | 2 + .../sdk/server-ai/src/api/config/types.ts | 11 ++ 6 files changed, 212 insertions(+) diff --git a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts index 86c1a0756b..66b0b7c551 100644 --- a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts @@ -846,6 +846,91 @@ describe('createTracker method', () => { configKey: 'my-config', variationKey: 'v1', version: 3, + modelVersion: 1, + }); + expect('modelKey' in tracker.getTrackData()).toBe(false); + }); +}); + +describe('modelKey and modelVersion tracking', () => { + it('stamps modelKey and modelVersion on tracker from flag payload', async () => { + const client = new LDAIClientImpl(mockLdClient); + const key = 'test-flag'; + + mockLdClient.variation.mockResolvedValue({ + model: { + name: 'gpt-4', + modelKey: 'my-model', + modelVersion: 2, + }, + provider: { name: 'openai' }, + messages: [], + _ldMeta: { + variationKey: 'v1', + enabled: true, + mode: 'completion', + version: 7, + }, + }); + + const result = await client.completionConfig(key, testContext); + const tracker = result.createTracker(); + + expect(tracker.getTrackData()).toMatchObject({ + modelKey: 'my-model', + modelVersion: 2, + }); + }); + + it('defaults modelVersion to 1 and omits modelKey when absent from flag payload', async () => { + const client = new LDAIClientImpl(mockLdClient); + const key = 'test-flag'; + + mockLdClient.variation.mockResolvedValue({ + model: { name: 'gpt-4' }, + provider: { name: 'openai' }, + messages: [], + _ldMeta: { + variationKey: 'v1', + enabled: true, + mode: 'completion', + version: 7, + }, + }); + + const result = await client.completionConfig(key, testContext); + const tracker = result.createTracker(); + + const trackData = tracker.getTrackData(); + expect('modelKey' in trackData).toBe(false); + expect(trackData.modelVersion).toBe(1); + }); + + it('exposes modelKey and modelVersion on config.model from flag payload', async () => { + const client = new LDAIClientImpl(mockLdClient); + const key = 'test-flag'; + + mockLdClient.variation.mockResolvedValue({ + model: { + name: 'gpt-4', + modelKey: 'my-model', + modelVersion: 2, + }, + provider: { name: 'openai' }, + messages: [], + _ldMeta: { + variationKey: 'v1', + enabled: true, + mode: 'completion', + }, + }); + + const result = await client.completionConfig(key, testContext); + + expect(result.model).toEqual({ + name: 'gpt-4', + modelKey: 'my-model', + modelVersion: 2, }); }); }); diff --git a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts index 4b625f1bca..ab06f7147c 100644 --- a/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIConfigTrackerImpl.test.ts @@ -31,6 +31,7 @@ const getExpectedTrackData = () => ({ version, modelName, providerName, + modelVersion: 1, runId: testRunId, }); @@ -1193,3 +1194,108 @@ describe('fromResumptionToken', () => { expect('graphKey' in reconstructed.getTrackData()).toBe(false); }); }); + +it('includes modelKey and modelVersion in getTrackData when set on constructor', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + undefined, + 'my-model', + 2, + ); + + expect(tracker.getTrackData()).toEqual({ + ...getExpectedTrackData(), + modelKey: 'my-model', + modelVersion: 2, + }); +}); + +it('omits modelKey from getTrackData when unset', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + ); + + const trackData = tracker.getTrackData(); + expect('modelKey' in trackData).toBe(false); + expect(trackData.modelVersion).toBe(1); +}); + +it('omits modelKey from getTrackData when empty string', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + undefined, + '', + 3, + ); + + const trackData = tracker.getTrackData(); + expect('modelKey' in trackData).toBe(false); + expect(trackData.modelVersion).toBe(3); +}); + +it('excludes modelKey and modelVersion from resumption token', () => { + const tracker = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + undefined, + 'my-model', + 2, + ); + + const decoded = JSON.parse(Buffer.from(tracker.resumptionToken, 'base64url').toString('utf8')); + expect('modelKey' in decoded).toBe(false); + expect('modelVersion' in decoded).toBe(false); +}); + +it('fromResumptionToken defaults modelVersion to 1 and omits modelKey', () => { + const original = new LDAIConfigTrackerImpl( + mockLdClient, + testRunId, + configKey, + variationKey, + version, + modelName, + providerName, + testContext, + undefined, + 'my-model', + 2, + ); + + const reconstructed = LDAIConfigTrackerImpl.fromResumptionToken( + original.resumptionToken, + mockLdClient, + testContext, + ); + + const trackData = reconstructed.getTrackData(); + expect('modelKey' in trackData).toBe(false); + expect(trackData.modelVersion).toBe(1); +}); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index 708fbfc110..f7360d6fc2 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -106,6 +106,8 @@ export class LDAIClientImpl implements LDAIClient { value.provider?.name ?? '', context, graphKey, + value.model?.modelKey, + value.model?.modelVersion ?? 1, ); // Validate mode match diff --git a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts index ab5a6fb918..e400d86a6f 100644 --- a/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts +++ b/packages/sdk/server-ai/src/LDAIConfigTrackerImpl.ts @@ -19,6 +19,8 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { private _providerName: string, private _context: LDContext, private _graphKey?: string, + private _modelKey?: string, + private _modelVersion: number = 1, ) { this._trackedMetrics.resumptionToken = this.resumptionToken; } @@ -30,6 +32,8 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { version: number; modelName: string; providerName: string; + modelVersion: number; + modelKey?: string; graphKey?: string; } { return { @@ -39,6 +43,8 @@ export class LDAIConfigTrackerImpl implements LDAIConfigTracker { version: this._version, modelName: this._modelName, providerName: this._providerName, + modelVersion: this._modelVersion, + ...(this._modelKey ? { modelKey: this._modelKey } : {}), ...(this._graphKey !== undefined ? { graphKey: this._graphKey } : {}), }; } diff --git a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts index 859cfd385f..4eabe504f5 100644 --- a/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts +++ b/packages/sdk/server-ai/src/api/config/LDAIConfigTracker.ts @@ -22,6 +22,8 @@ export interface LDAIConfigTracker { version: number; modelName: string; providerName: string; + modelVersion: number; + modelKey?: string; graphKey?: string; }; diff --git a/packages/sdk/server-ai/src/api/config/types.ts b/packages/sdk/server-ai/src/api/config/types.ts index adcb3c4d9f..d966327125 100644 --- a/packages/sdk/server-ai/src/api/config/types.ts +++ b/packages/sdk/server-ai/src/api/config/types.ts @@ -42,6 +42,17 @@ export interface LDModelConfig { * The region for the model. */ region?: string; + + /** + * The stable, unique key of the model (used for direct lookup; distinct from + * `name`, which is not guaranteed unique). + */ + modelKey?: string; + + /** + * The pinned version of the model that this config variation references. + */ + modelVersion?: number; } export interface LDProviderConfig { From 8ad1db900114494c34ed055e684aa5c29412514f Mon Sep 17 00:00:00 2001 From: Anthony Torns II Date: Wed, 15 Jul 2026 16:28:18 -0700 Subject: [PATCH 2/3] Read modelKey/modelVersion from _ldMeta, matching gonfalon PR 67230 Gonfalon moved these two fields from the payload's model object to _ldMeta (launchdarkly/gonfalon#67230) to avoid modelVersion reading as the underlying LLM's own version. Field names are unchanged; only the JSON location moves. _toBaseConfig now merges modelKey/modelVersion from _ldMeta onto the exposed model object (previously a direct passthrough of flagValue.model, which no longer carries them), only setting them when actually present so config.model's shape is unchanged when they're absent. The tracker's own default-to-1 fallback for stamped events is untouched. Co-Authored-By: Claude Fable 5 --- .../server-ai/__tests__/LDAIClientImpl.test.ts | 8 ++++---- packages/sdk/server-ai/src/LDAIClientImpl.ts | 6 ++++-- .../server-ai/src/api/config/LDAIConfigUtils.ts | 16 +++++++++++++++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts index 66b0b7c551..09d7c7493b 100644 --- a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts @@ -860,8 +860,6 @@ describe('modelKey and modelVersion tracking', () => { mockLdClient.variation.mockResolvedValue({ model: { name: 'gpt-4', - modelKey: 'my-model', - modelVersion: 2, }, provider: { name: 'openai' }, messages: [], @@ -870,6 +868,8 @@ describe('modelKey and modelVersion tracking', () => { enabled: true, mode: 'completion', version: 7, + modelKey: 'my-model', + modelVersion: 2, }, }); @@ -913,8 +913,6 @@ describe('modelKey and modelVersion tracking', () => { mockLdClient.variation.mockResolvedValue({ model: { name: 'gpt-4', - modelKey: 'my-model', - modelVersion: 2, }, provider: { name: 'openai' }, messages: [], @@ -922,6 +920,8 @@ describe('modelKey and modelVersion tracking', () => { variationKey: 'v1', enabled: true, mode: 'completion', + modelKey: 'my-model', + modelVersion: 2, }, }); diff --git a/packages/sdk/server-ai/src/LDAIClientImpl.ts b/packages/sdk/server-ai/src/LDAIClientImpl.ts index f7360d6fc2..8eb6ba04ba 100644 --- a/packages/sdk/server-ai/src/LDAIClientImpl.ts +++ b/packages/sdk/server-ai/src/LDAIClientImpl.ts @@ -106,8 +106,10 @@ export class LDAIClientImpl implements LDAIClient { value.provider?.name ?? '', context, graphKey, - value.model?.modelKey, - value.model?.modelVersion ?? 1, + // eslint-disable-next-line no-underscore-dangle + value._ldMeta?.modelKey, + // eslint-disable-next-line no-underscore-dangle + value._ldMeta?.modelVersion ?? 1, ); // Validate mode match diff --git a/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts b/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts index d77d9049b9..9ed943e1b3 100644 --- a/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts +++ b/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts @@ -26,6 +26,8 @@ export interface LDAIConfigFlagValue { enabled: boolean; version?: number; mode?: LDAIConfigMode; + modelKey?: string; + modelVersion?: number; }; model?: LDModelConfig; messages?: LDMessage[]; @@ -182,11 +184,23 @@ export class LDAIConfigUtils { * @returns Base configuration object */ private static _toBaseConfig(key: string, flagValue: LDAIConfigFlagValue) { + let model = flagValue.model; + if (model) { + // eslint-disable-next-line no-underscore-dangle + const { modelKey, modelVersion } = flagValue._ldMeta ?? {}; + model = { ...model }; + if (modelKey !== undefined) { + model.modelKey = modelKey; + } + if (modelVersion !== undefined) { + model.modelVersion = modelVersion; + } + } return { key, // eslint-disable-next-line no-underscore-dangle enabled: flagValue._ldMeta?.enabled ?? false, - model: flagValue.model, + model, provider: flagValue.provider, }; } From d292250a34956e0dddd3cd3e692f8b15b1bd1a1d Mon Sep 17 00:00:00 2001 From: Anthony Torns II Date: Fri, 17 Jul 2026 13:09:41 -0700 Subject: [PATCH 3/3] fix(server-ai): stop exposing modelKey/modelVersion on LDModelConfig modelKey/modelVersion should only be visible via the tracker's stamped event data, mirroring how variationKey/version are treated. LDAIClientImpl already sourced them from _ldMeta directly for the tracker factory, so only the model reconstruction in _toBaseConfig needed to stop merging them onto the public model object. Co-Authored-By: Claude Sonnet 5 --- .../sdk/server-ai/__tests__/LDAIClientImpl.test.ts | 7 ++++--- .../server-ai/src/api/config/LDAIConfigUtils.ts | 14 +------------- packages/sdk/server-ai/src/api/config/types.ts | 11 ----------- 3 files changed, 5 insertions(+), 27 deletions(-) diff --git a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts index 09d7c7493b..340f1a20fa 100644 --- a/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts +++ b/packages/sdk/server-ai/__tests__/LDAIClientImpl.test.ts @@ -906,7 +906,10 @@ describe('modelKey and modelVersion tracking', () => { expect(trackData.modelVersion).toBe(1); }); - it('exposes modelKey and modelVersion on config.model from flag payload', async () => { + it('does not expose modelKey or modelVersion on config.model', async () => { + // modelKey/modelVersion are intentionally not exposed on LDModelConfig (they'd read as + // properties of the LLM itself, e.g. a version like "5.4"); the only place they surface is + // the tracker's stamped event data, mirroring variationKey/version. const client = new LDAIClientImpl(mockLdClient); const key = 'test-flag'; @@ -929,8 +932,6 @@ describe('modelKey and modelVersion tracking', () => { expect(result.model).toEqual({ name: 'gpt-4', - modelKey: 'my-model', - modelVersion: 2, }); }); }); diff --git a/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts b/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts index 9ed943e1b3..ec576a0581 100644 --- a/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts +++ b/packages/sdk/server-ai/src/api/config/LDAIConfigUtils.ts @@ -184,23 +184,11 @@ export class LDAIConfigUtils { * @returns Base configuration object */ private static _toBaseConfig(key: string, flagValue: LDAIConfigFlagValue) { - let model = flagValue.model; - if (model) { - // eslint-disable-next-line no-underscore-dangle - const { modelKey, modelVersion } = flagValue._ldMeta ?? {}; - model = { ...model }; - if (modelKey !== undefined) { - model.modelKey = modelKey; - } - if (modelVersion !== undefined) { - model.modelVersion = modelVersion; - } - } return { key, // eslint-disable-next-line no-underscore-dangle enabled: flagValue._ldMeta?.enabled ?? false, - model, + model: flagValue.model, provider: flagValue.provider, }; } diff --git a/packages/sdk/server-ai/src/api/config/types.ts b/packages/sdk/server-ai/src/api/config/types.ts index d966327125..adcb3c4d9f 100644 --- a/packages/sdk/server-ai/src/api/config/types.ts +++ b/packages/sdk/server-ai/src/api/config/types.ts @@ -42,17 +42,6 @@ export interface LDModelConfig { * The region for the model. */ region?: string; - - /** - * The stable, unique key of the model (used for direct lookup; distinct from - * `name`, which is not guaranteed unique). - */ - modelKey?: string; - - /** - * The pinned version of the model that this config variation references. - */ - modelVersion?: number; } export interface LDProviderConfig {