diff --git a/evals/text-complexity/ela-reading/vocabulary-complexity/config.json b/evals/text-complexity/ela-reading/vocabulary-complexity/config.json index 9d93c573..8cf70061 100644 --- a/evals/text-complexity/ela-reading/vocabulary-complexity/config.json +++ b/evals/text-complexity/ela-reading/vocabulary-complexity/config.json @@ -85,10 +85,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4o-2024-11-20" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": null }, "parser": { "kind": "structured_output" @@ -195,10 +195,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4.1-2025-04-14" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": null }, "parser": { "kind": "structured_output" diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json index 9d93c573..8cf70061 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json @@ -85,10 +85,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4o-2024-11-20" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": null }, "parser": { "kind": "structured_output" @@ -195,10 +195,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4.1-2025-04-14" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": null }, "parser": { "kind": "structured_output" diff --git a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py index da9f8516..4cbd37e8 100644 --- a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py +++ b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py @@ -20,9 +20,9 @@ TEXT = "Polo went on a 24-year trip to China with his father and uncle, merchants of Venice." KEYS = {"openai_api_key": "o", "google_api_key": "g"} -BACKGROUND_MODEL = "openai:gpt-4o-2024-11-20" +BACKGROUND_MODEL = "openai:gpt-6-luna" GRADES_34_MODEL = "google:gemini-2.5-pro" -OTHER_GRADES_MODEL = "openai:gpt-4.1-2025-04-14" +OTHER_GRADES_MODEL = "openai:gpt-6-luna" def test_metadata_is_the_contracts() -> None: @@ -78,7 +78,9 @@ async def test_grades_5_to_12_rate_on_openai_without_a_readability_score( assert "It is intended for grade 7." in rating["messages"][1]["content"] # The entry's own condition excludes these grades, so the score is never computed. assert "Flesch-Kincaid" not in rating["messages"][1]["content"] - assert evaluation.metadata.model == f"{BACKGROUND_MODEL}+{OTHER_GRADES_MODEL}" + # Both steps run the same model, so the label names it once (SPEC §3.4). + assert OTHER_GRADES_MODEL == BACKGROUND_MODEL + assert evaluation.metadata.model == OTHER_GRADES_MODEL async def test_only_one_branch_runs(self, providers: ProviderFactory) -> None: await VocabularyComplexityEvaluator(**KEYS).evaluate(text=TEXT, grade_level=7) diff --git a/sdks/typescript/src/evaluators/base.ts b/sdks/typescript/src/evaluators/base.ts index 409d1112..ac14fea2 100644 --- a/sdks/typescript/src/evaluators/base.ts +++ b/sdks/typescript/src/evaluators/base.ts @@ -68,8 +68,8 @@ type NormalizedTelemetryOptions = Required> & * Two things to know before overriding: * * - The override is evaluator-wide, not per call site. `VocabularyComplexityEvaluator` - * deliberately uses three models (Gemini 2.5 Pro for grade levels 3-4, GPT-4.1 for - * 5-12, GPT-4o for background knowledge); an override collapses all three. + * deliberately uses two models (Gemini 2.5 Pro for grade levels 3-4, gpt-6-luna for + * 5-12 and for background knowledge); an override collapses both. * - Every evaluator on `defineSingleStepEvaluator` takes its model from its own contract * (`evals////config.json`), so an override diverges from the * shared cross-language config rather than from an SDK default. diff --git a/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts b/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts index cf6bcc03..0b2d2cb7 100644 --- a/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts +++ b/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts @@ -157,10 +157,13 @@ export class VocabularyComplexityEvaluator extends BaseEvaluator { validateInputs(input, INPUT_SCHEMA); ({ text, grade_level: gradeLevel } = input); - // When override is active all providers resolve to the same model — show a single label. - modelLabel = this.config.modelOverride - ? backgroundProviderLabel - : `${backgroundProviderLabel}+${this.complexityProviderFor(gradeLevel).label}`; + // One model reports one label (SPEC §3.4): under an override, or when the grade's + // complexity step runs the same model as background knowledge. + const complexityProviderLabel = this.complexityProviderFor(gradeLevel).label; + modelLabel = + this.config.modelOverride || complexityProviderLabel === backgroundProviderLabel + ? backgroundProviderLabel + : `${backgroundProviderLabel}+${complexityProviderLabel}`; this.logger.info('Starting Vocabulary Complexity evaluation', { evaluator: VocabularyComplexityEvaluator.metadata.id, @@ -327,7 +330,7 @@ export class VocabularyComplexityEvaluator extends BaseEvaluator { * Stage 2: Evaluate vocabulary complexity * * Uses the Qual Text Complexity rubric (SAP) and background knowledge to evaluate vocabulary complexity. - * Grades 3-4 use Gemini 2.5 Pro; grade levels 5-12 use GPT-4.1. + * Grades 3-4 use Gemini 2.5 Pro; grade levels 5-12 use gpt-6-luna. */ private async evaluateComplexity( text: string, diff --git a/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts b/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts index eac2728f..dbebc918 100644 --- a/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts +++ b/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts @@ -92,7 +92,7 @@ describe('Anthropic modelOverride — plumbing', () => { }); describe('Anthropic modelOverride — vocabulary multi-model collapse', () => { - // Vocabulary deliberately uses three models; an override collapses all three. + // Vocabulary has three call sites on two models; an override collapses all three. // Asserted so the CLI can warn rather than surprise users. it('collapses all three vocabulary call sites onto the single override model', () => { new VocabularyComplexityEvaluator(anthropicOnlyConfig()); @@ -102,7 +102,7 @@ describe('Anthropic modelOverride — vocabulary multi-model collapse', () => { expect(models).toEqual([ANTHROPIC_MODEL, ANTHROPIC_MODEL, ANTHROPIC_MODEL]); }); - it('uses three distinct models across two providers without an override', () => { + it('uses two distinct models across two providers without an override', () => { new VocabularyComplexityEvaluator({ googleApiKey: 'g-test', openaiApiKey: 'o-test', @@ -111,6 +111,6 @@ describe('Anthropic modelOverride — vocabulary multi-model collapse', () => { const calls = createProvider.mock.calls.map(([arg]) => arg); expect(calls.map((c) => c.type)).toEqual([Provider.Google, Provider.OpenAI, Provider.OpenAI]); - expect(new Set(calls.map((c) => c.model)).size).toBe(3); + expect(new Set(calls.map((c) => c.model)).size).toBe(2); }); }); diff --git a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts index cc6821f5..6bedd4d3 100644 --- a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts +++ b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts @@ -105,7 +105,7 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { expect(result.result.complexity_score).toBe('moderately_complex'); expect(result.result.reasoning).toContain('grade-appropriate vocabulary'); expect(result.metadata).toBeDefined(); - expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20+openai:gpt-4.1-2025-04-14'); + expect(result.metadata.model).toBe('openai:gpt-6-luna'); // Not > 0: with both providers mocked and this branch computing no readability // score, the whole evaluation can finish inside one clock tick. expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); @@ -120,13 +120,13 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { // Verify background knowledge call const bgCall = vi.mocked(mockBackgroundProvider.generateText).mock.calls[0]; expect(bgCall[0][0].content).toContain(testText); - expect(bgCall[1]).toBe(0); // temperature = 0 + expect(bgCall[1]).toBeNull(); // temperature = null: the parameter is omitted // Verify complexity call includes background knowledge const complexityCall = vi.mocked(mockComplexityProvider.generateStructured).mock.calls[0]; expect(complexityCall[0].messages[1].content).toContain(testText); expect(complexityCall[0].schema).toBeDefined(); - expect(complexityCall[0].temperature).toBe(0); + expect(complexityCall[0].temperature).toBeNull(); }); }); @@ -207,7 +207,7 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { expect(result.metadata).toHaveProperty('processingTimeMs'); // Verify metadata values - expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20+openai:gpt-4.1-2025-04-14'); + expect(result.metadata.model).toBe('openai:gpt-6-luna'); expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); // Mocked calls can be instant (0ms) expect(result.metadata.tokenUsage.inputTokens).toBe(300); expect(result.metadata.tokenUsage.outputTokens).toBe(150); @@ -275,13 +275,13 @@ describe('VocabularyComplexityEvaluator - complexity model per grade', () => { // The grade picks the complexity model, and the reported label has to name the one that // ran. Every other test in this file uses grade 5+, so the grades 3-4 branch went // unexercised and reported the grade 5-12 model for a year. - const BACKGROUND = 'openai:gpt-4o-2024-11-20'; + const BACKGROUND = 'openai:gpt-6-luna'; const CASES = [ { grade: '3', provider: 'grades34ComplexityProvider', model: 'google:gemini-2.5-pro' }, { grade: '4', provider: 'grades34ComplexityProvider', model: 'google:gemini-2.5-pro' }, - { grade: '5', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-4.1-2025-04-14' }, - { grade: '12', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-4.1-2025-04-14' }, + { grade: '5', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-6-luna' }, + { grade: '12', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-6-luna' }, ] as const; it.each(CASES)( @@ -327,8 +327,9 @@ describe('VocabularyComplexityEvaluator - complexity model per grade', () => { // The model that ran. expect(complexity.generateStructured).toHaveBeenCalledTimes(1); expect(unused.generateStructured).not.toHaveBeenCalled(); - // The model that is reported. These two disagreeing is the bug. - expect(result.metadata.model).toBe(`${BACKGROUND}+${model}`); + // The model that is reported. These two disagreeing is the bug. A branch that runs + // the background model again names it once (SPEC §3.4). + expect(result.metadata.model).toBe(model === BACKGROUND ? model : `${BACKGROUND}+${model}`); }, ); }); diff --git a/sdks/typescript/tests/unit/registry-conformance.test.ts b/sdks/typescript/tests/unit/registry-conformance.test.ts index 23a8392a..a4cb7e6e 100644 --- a/sdks/typescript/tests/unit/registry-conformance.test.ts +++ b/sdks/typescript/tests/unit/registry-conformance.test.ts @@ -871,10 +871,12 @@ describe('the reported model matches the contract steps that apply', () => { !step.condition || (step.condition.input === 'grade_level' && step.condition.in.map(String).includes(grade)); - const expected = config.steps - .filter(applies) - .map((step) => `${step.model.provider}:${step.model.name}`) - .join('+'); + // Steps that share a model report it once, in run order (SPEC §3.4). + const expected = [ + ...new Set( + config.steps.filter(applies).map((step) => `${step.model.provider}:${step.model.name}`), + ), + ].join('+'); const result = (await construct(E).evaluate({ text: 'The storm gathered offshore and the harbour emptied before dusk.',