From 839cbdaa650c034430e9101d0f352c06eadf47b8 Mon Sep 17 00:00:00 2001 From: Gary Mu Date: Tue, 3 Nov 2026 09:02:39 -0800 Subject: [PATCH 1/3] feat(evals): move Vocabulary Complexity to gpt-6-luna ahead of the gpt-4o and gpt-4.1 retirement OpenAI retires gpt-4o-2024-11-20 and gpt-4.1-2025-04-14 on Jan 14, 2027. Vocabulary Complexity uses gpt-4o for background knowledge in every grade and gpt-4.1 to rate grades 5-12. Both steps move to gpt-6-luna. It rejects temperature 0, so both steps pin temperature 1. Grades 3-4 keep gemini-2.5-pro at temperature 0 for the rating. Prompts, schemas and parsers are unchanged. The Python contract copy is regenerated with make generate-contracts; both SDKs' vocabulary unit tests and two doc comments now name the new model. Co-Authored-By: Claude Opus 5.5 --- .../ela-reading/vocabulary-complexity/config.json | 8 ++++---- .../ela_reading/vocabulary_complexity/config.json | 8 ++++---- .../ela_reading/test_vocabulary_complexity.py | 4 ++-- sdks/typescript/src/evaluators/base.ts | 4 ++-- .../ela-reading/vocabulary-complexity.ts | 2 +- .../unit/evaluators/vocabulary-complexity.test.ts | 14 +++++++------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/evals/text-complexity/ela-reading/vocabulary-complexity/config.json b/evals/text-complexity/ela-reading/vocabulary-complexity/config.json index 9d93c573c..6f50a0cd4 100644 --- a/evals/text-complexity/ela-reading/vocabulary-complexity/config.json +++ b/evals/text-complexity/ela-reading/vocabulary-complexity/config.json @@ -85,10 +85,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4o-2024-11-20" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": 1 }, "parser": { "kind": "structured_output" @@ -195,10 +195,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4.1-2025-04-14" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": 1 }, "parser": { "kind": "structured_output" diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json index 9d93c573c..6f50a0cd4 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json @@ -85,10 +85,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4o-2024-11-20" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": 1 }, "parser": { "kind": "structured_output" @@ -195,10 +195,10 @@ }, "model": { "provider": "openai", - "name": "gpt-4.1-2025-04-14" + "name": "gpt-6-luna" }, "generation": { - "temperature": 0 + "temperature": 1 }, "parser": { "kind": "structured_output" diff --git a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py index da9f8516e..c67911a25 100644 --- a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py +++ b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py @@ -20,9 +20,9 @@ TEXT = "Polo went on a 24-year trip to China with his father and uncle, merchants of Venice." KEYS = {"openai_api_key": "o", "google_api_key": "g"} -BACKGROUND_MODEL = "openai:gpt-4o-2024-11-20" +BACKGROUND_MODEL = "openai:gpt-6-luna" GRADES_34_MODEL = "google:gemini-2.5-pro" -OTHER_GRADES_MODEL = "openai:gpt-4.1-2025-04-14" +OTHER_GRADES_MODEL = "openai:gpt-6-luna" def test_metadata_is_the_contracts() -> None: diff --git a/sdks/typescript/src/evaluators/base.ts b/sdks/typescript/src/evaluators/base.ts index 409d11121..ac14fea2a 100644 --- a/sdks/typescript/src/evaluators/base.ts +++ b/sdks/typescript/src/evaluators/base.ts @@ -68,8 +68,8 @@ type NormalizedTelemetryOptions = Required> & * Two things to know before overriding: * * - The override is evaluator-wide, not per call site. `VocabularyComplexityEvaluator` - * deliberately uses three models (Gemini 2.5 Pro for grade levels 3-4, GPT-4.1 for - * 5-12, GPT-4o for background knowledge); an override collapses all three. + * deliberately uses two models (Gemini 2.5 Pro for grade levels 3-4, gpt-6-luna for + * 5-12 and for background knowledge); an override collapses both. * - Every evaluator on `defineSingleStepEvaluator` takes its model from its own contract * (`evals////config.json`), so an override diverges from the * shared cross-language config rather than from an SDK default. diff --git a/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts b/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts index cf6bcc037..7366366e2 100644 --- a/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts +++ b/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts @@ -327,7 +327,7 @@ export class VocabularyComplexityEvaluator extends BaseEvaluator { * Stage 2: Evaluate vocabulary complexity * * Uses the Qual Text Complexity rubric (SAP) and background knowledge to evaluate vocabulary complexity. - * Grades 3-4 use Gemini 2.5 Pro; grade levels 5-12 use GPT-4.1. + * Grades 3-4 use Gemini 2.5 Pro; grade levels 5-12 use gpt-6-luna. */ private async evaluateComplexity( text: string, diff --git a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts index cc6821f5d..7cde62eb6 100644 --- a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts +++ b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts @@ -105,7 +105,7 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { expect(result.result.complexity_score).toBe('moderately_complex'); expect(result.result.reasoning).toContain('grade-appropriate vocabulary'); expect(result.metadata).toBeDefined(); - expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20+openai:gpt-4.1-2025-04-14'); + expect(result.metadata.model).toBe('openai:gpt-6-luna+openai:gpt-6-luna'); // Not > 0: with both providers mocked and this branch computing no readability // score, the whole evaluation can finish inside one clock tick. expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); @@ -120,13 +120,13 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { // Verify background knowledge call const bgCall = vi.mocked(mockBackgroundProvider.generateText).mock.calls[0]; expect(bgCall[0][0].content).toContain(testText); - expect(bgCall[1]).toBe(0); // temperature = 0 + expect(bgCall[1]).toBe(1); // temperature = 1 // Verify complexity call includes background knowledge const complexityCall = vi.mocked(mockComplexityProvider.generateStructured).mock.calls[0]; expect(complexityCall[0].messages[1].content).toContain(testText); expect(complexityCall[0].schema).toBeDefined(); - expect(complexityCall[0].temperature).toBe(0); + expect(complexityCall[0].temperature).toBe(1); }); }); @@ -207,7 +207,7 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { expect(result.metadata).toHaveProperty('processingTimeMs'); // Verify metadata values - expect(result.metadata.model).toBe('openai:gpt-4o-2024-11-20+openai:gpt-4.1-2025-04-14'); + expect(result.metadata.model).toBe('openai:gpt-6-luna+openai:gpt-6-luna'); expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); // Mocked calls can be instant (0ms) expect(result.metadata.tokenUsage.inputTokens).toBe(300); expect(result.metadata.tokenUsage.outputTokens).toBe(150); @@ -275,13 +275,13 @@ describe('VocabularyComplexityEvaluator - complexity model per grade', () => { // The grade picks the complexity model, and the reported label has to name the one that // ran. Every other test in this file uses grade 5+, so the grades 3-4 branch went // unexercised and reported the grade 5-12 model for a year. - const BACKGROUND = 'openai:gpt-4o-2024-11-20'; + const BACKGROUND = 'openai:gpt-6-luna'; const CASES = [ { grade: '3', provider: 'grades34ComplexityProvider', model: 'google:gemini-2.5-pro' }, { grade: '4', provider: 'grades34ComplexityProvider', model: 'google:gemini-2.5-pro' }, - { grade: '5', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-4.1-2025-04-14' }, - { grade: '12', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-4.1-2025-04-14' }, + { grade: '5', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-6-luna' }, + { grade: '12', provider: 'otherGradesComplexityProvider', model: 'openai:gpt-6-luna' }, ] as const; it.each(CASES)( From eb554b5a616fe27bdc38a32ed305725c21221009 Mon Sep 17 00:00:00 2001 From: Gary Mu Date: Tue, 3 Nov 2026 09:06:37 -0800 Subject: [PATCH 2/3] fix(sdk): report one model label when vocabulary's steps share a model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With background knowledge and the grades 5-12 rating both on gpt-6-luna, the TypeScript SDK reported "openai:gpt-6-luna+openai:gpt-6-luna" while the Python SDK reported "openai:gpt-6-luna". SPEC §3.4 gives one model one label, so TypeScript now names a shared model once, as Python and the model-override path already do. The registry conformance test derives the expected label the same way, and the tests that counted three distinct vocabulary models now count two. Co-Authored-By: Claude Opus 5.5 --- .../ela_reading/test_vocabulary_complexity.py | 4 +++- .../ela-reading/vocabulary-complexity.ts | 11 +++++++---- .../tests/unit/evaluators/anthropic-override.test.ts | 6 +++--- .../unit/evaluators/vocabulary-complexity.test.ts | 9 +++++---- .../tests/unit/registry-conformance.test.ts | 10 ++++++---- 5 files changed, 24 insertions(+), 16 deletions(-) diff --git a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py index c67911a25..4cbd37e83 100644 --- a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py +++ b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_vocabulary_complexity.py @@ -78,7 +78,9 @@ async def test_grades_5_to_12_rate_on_openai_without_a_readability_score( assert "It is intended for grade 7." in rating["messages"][1]["content"] # The entry's own condition excludes these grades, so the score is never computed. assert "Flesch-Kincaid" not in rating["messages"][1]["content"] - assert evaluation.metadata.model == f"{BACKGROUND_MODEL}+{OTHER_GRADES_MODEL}" + # Both steps run the same model, so the label names it once (SPEC §3.4). + assert OTHER_GRADES_MODEL == BACKGROUND_MODEL + assert evaluation.metadata.model == OTHER_GRADES_MODEL async def test_only_one_branch_runs(self, providers: ProviderFactory) -> None: await VocabularyComplexityEvaluator(**KEYS).evaluate(text=TEXT, grade_level=7) diff --git a/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts b/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts index 7366366e2..0b2d2cb73 100644 --- a/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts +++ b/sdks/typescript/src/evaluators/text-complexity/ela-reading/vocabulary-complexity.ts @@ -157,10 +157,13 @@ export class VocabularyComplexityEvaluator extends BaseEvaluator { validateInputs(input, INPUT_SCHEMA); ({ text, grade_level: gradeLevel } = input); - // When override is active all providers resolve to the same model — show a single label. - modelLabel = this.config.modelOverride - ? backgroundProviderLabel - : `${backgroundProviderLabel}+${this.complexityProviderFor(gradeLevel).label}`; + // One model reports one label (SPEC §3.4): under an override, or when the grade's + // complexity step runs the same model as background knowledge. + const complexityProviderLabel = this.complexityProviderFor(gradeLevel).label; + modelLabel = + this.config.modelOverride || complexityProviderLabel === backgroundProviderLabel + ? backgroundProviderLabel + : `${backgroundProviderLabel}+${complexityProviderLabel}`; this.logger.info('Starting Vocabulary Complexity evaluation', { evaluator: VocabularyComplexityEvaluator.metadata.id, diff --git a/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts b/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts index eac2728ff..dbebc9188 100644 --- a/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts +++ b/sdks/typescript/tests/unit/evaluators/anthropic-override.test.ts @@ -92,7 +92,7 @@ describe('Anthropic modelOverride — plumbing', () => { }); describe('Anthropic modelOverride — vocabulary multi-model collapse', () => { - // Vocabulary deliberately uses three models; an override collapses all three. + // Vocabulary has three call sites on two models; an override collapses all three. // Asserted so the CLI can warn rather than surprise users. it('collapses all three vocabulary call sites onto the single override model', () => { new VocabularyComplexityEvaluator(anthropicOnlyConfig()); @@ -102,7 +102,7 @@ describe('Anthropic modelOverride — vocabulary multi-model collapse', () => { expect(models).toEqual([ANTHROPIC_MODEL, ANTHROPIC_MODEL, ANTHROPIC_MODEL]); }); - it('uses three distinct models across two providers without an override', () => { + it('uses two distinct models across two providers without an override', () => { new VocabularyComplexityEvaluator({ googleApiKey: 'g-test', openaiApiKey: 'o-test', @@ -111,6 +111,6 @@ describe('Anthropic modelOverride — vocabulary multi-model collapse', () => { const calls = createProvider.mock.calls.map(([arg]) => arg); expect(calls.map((c) => c.type)).toEqual([Provider.Google, Provider.OpenAI, Provider.OpenAI]); - expect(new Set(calls.map((c) => c.model)).size).toBe(3); + expect(new Set(calls.map((c) => c.model)).size).toBe(2); }); }); diff --git a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts index 7cde62eb6..9e9f4de90 100644 --- a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts +++ b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts @@ -105,7 +105,7 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { expect(result.result.complexity_score).toBe('moderately_complex'); expect(result.result.reasoning).toContain('grade-appropriate vocabulary'); expect(result.metadata).toBeDefined(); - expect(result.metadata.model).toBe('openai:gpt-6-luna+openai:gpt-6-luna'); + expect(result.metadata.model).toBe('openai:gpt-6-luna'); // Not > 0: with both providers mocked and this branch computing no readability // score, the whole evaluation can finish inside one clock tick. expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); @@ -207,7 +207,7 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { expect(result.metadata).toHaveProperty('processingTimeMs'); // Verify metadata values - expect(result.metadata.model).toBe('openai:gpt-6-luna+openai:gpt-6-luna'); + expect(result.metadata.model).toBe('openai:gpt-6-luna'); expect(result.metadata.processingTimeMs).toBeGreaterThanOrEqual(0); // Mocked calls can be instant (0ms) expect(result.metadata.tokenUsage.inputTokens).toBe(300); expect(result.metadata.tokenUsage.outputTokens).toBe(150); @@ -327,8 +327,9 @@ describe('VocabularyComplexityEvaluator - complexity model per grade', () => { // The model that ran. expect(complexity.generateStructured).toHaveBeenCalledTimes(1); expect(unused.generateStructured).not.toHaveBeenCalled(); - // The model that is reported. These two disagreeing is the bug. - expect(result.metadata.model).toBe(`${BACKGROUND}+${model}`); + // The model that is reported. These two disagreeing is the bug. A branch that runs + // the background model again names it once (SPEC §3.4). + expect(result.metadata.model).toBe(model === BACKGROUND ? model : `${BACKGROUND}+${model}`); }, ); }); diff --git a/sdks/typescript/tests/unit/registry-conformance.test.ts b/sdks/typescript/tests/unit/registry-conformance.test.ts index 23a8392af..a4cb7e6e9 100644 --- a/sdks/typescript/tests/unit/registry-conformance.test.ts +++ b/sdks/typescript/tests/unit/registry-conformance.test.ts @@ -871,10 +871,12 @@ describe('the reported model matches the contract steps that apply', () => { !step.condition || (step.condition.input === 'grade_level' && step.condition.in.map(String).includes(grade)); - const expected = config.steps - .filter(applies) - .map((step) => `${step.model.provider}:${step.model.name}`) - .join('+'); + // Steps that share a model report it once, in run order (SPEC §3.4). + const expected = [ + ...new Set( + config.steps.filter(applies).map((step) => `${step.model.provider}:${step.model.name}`), + ), + ].join('+'); const result = (await construct(E).evaluate({ text: 'The storm gathered offshore and the harbour emptied before dusk.', From 9e342e099c649da899891d296d5615591f5c93ec Mon Sep 17 00:00:00 2001 From: Gary Mu Date: Sat, 7 Nov 2026 09:32:47 -0800 Subject: [PATCH 3/3] fix(evals): send no temperature to gpt-6-luna in Vocabulary Complexity gpt-6-luna takes no temperature but its default: temperature 0 returns "Unsupported parameter: 'temperature' is not supported with this model", while 1 and an omitted parameter are both accepted. That is the case config.schema.json reserves null for, and the team has agreed to run models at their own defaults going forward. Both gpt-6-luna steps (background_knowledge and vocab_complexity_other_grades) now pin null, so neither SDK sends the parameter. The model's default is 1, the value the migration analysis ran at, so outputs should not change. The Gemini grades 3-4 step keeps temperature 0. Co-Authored-By: Claude Opus 5.5 --- .../ela-reading/vocabulary-complexity/config.json | 4 ++-- .../ela_reading/vocabulary_complexity/config.json | 4 ++-- .../tests/unit/evaluators/vocabulary-complexity.test.ts | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/evals/text-complexity/ela-reading/vocabulary-complexity/config.json b/evals/text-complexity/ela-reading/vocabulary-complexity/config.json index 6f50a0cd4..8cf70061b 100644 --- a/evals/text-complexity/ela-reading/vocabulary-complexity/config.json +++ b/evals/text-complexity/ela-reading/vocabulary-complexity/config.json @@ -88,7 +88,7 @@ "name": "gpt-6-luna" }, "generation": { - "temperature": 1 + "temperature": null }, "parser": { "kind": "structured_output" @@ -198,7 +198,7 @@ "name": "gpt-6-luna" }, "generation": { - "temperature": 1 + "temperature": null }, "parser": { "kind": "structured_output" diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json index 6f50a0cd4..8cf70061b 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/vocabulary_complexity/config.json @@ -88,7 +88,7 @@ "name": "gpt-6-luna" }, "generation": { - "temperature": 1 + "temperature": null }, "parser": { "kind": "structured_output" @@ -198,7 +198,7 @@ "name": "gpt-6-luna" }, "generation": { - "temperature": 1 + "temperature": null }, "parser": { "kind": "structured_output" diff --git a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts index 9e9f4de90..6bedd4d3b 100644 --- a/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts +++ b/sdks/typescript/tests/unit/evaluators/vocabulary-complexity.test.ts @@ -120,13 +120,13 @@ describe('VocabularyComplexityEvaluator - Evaluation Flow', () => { // Verify background knowledge call const bgCall = vi.mocked(mockBackgroundProvider.generateText).mock.calls[0]; expect(bgCall[0][0].content).toContain(testText); - expect(bgCall[1]).toBe(1); // temperature = 1 + expect(bgCall[1]).toBeNull(); // temperature = null: the parameter is omitted // Verify complexity call includes background knowledge const complexityCall = vi.mocked(mockComplexityProvider.generateStructured).mock.calls[0]; expect(complexityCall[0].messages[1].content).toContain(testText); expect(complexityCall[0].schema).toBeDefined(); - expect(complexityCall[0].temperature).toBe(1); + expect(complexityCall[0].temperature).toBeNull(); }); });