From fa11fc94aeb773cc7180a8262a5243b026c8c246 Mon Sep 17 00:00:00 2001 From: SeungYeon Lee <307381672+slee430@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:35:09 -0700 Subject: [PATCH 1/4] fix(sentence-structure): align rubric keys and preserve fractional features --- .../sentence-structure/config.json | 4 +-- .../sentence-structure/example_notebook.ipynb | 2 +- .../sentence-structure/rubric-grade-3.txt | 8 ++--- .../sentence-structure/rubric-grade-4.txt | 8 ++--- .../sentence_structure/config.json | 4 +-- .../sentence_structure/rubric-grade-3.txt | 8 ++--- .../sentence_structure/rubric-grade-4.txt | 8 ++--- .../ela_reading/sentence_structure.py | 2 +- .../features/sentence_features.py | 28 ++++++--------- .../ela_reading/test_sentence_structure.py | 12 +++++-- .../unit/features/test_sentence_features.py | 16 +++++++++ .../ela-reading/sentence-structure.ts | 2 +- .../evaluators/sentence-structure.test.ts | 34 +++++++++++++++++++ 13 files changed, 93 insertions(+), 43 deletions(-) diff --git a/evals/text-complexity/ela-reading/sentence-structure/config.json b/evals/text-complexity/ela-reading/sentence-structure/config.json index f8a4fa4f6..a89b472c0 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/config.json +++ b/evals/text-complexity/ela-reading/sentence-structure/config.json @@ -38,7 +38,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-3.txt", - "sha256": "3ddacd7b3c80f1f8e85bfcddd01db945813fcf814d893e719f5ae48babdaf11f", + "sha256": "d2522ad29cf7a1edc4288a13d5509001c64776ed38242b9a7b790d621fa978a8", "condition": { "input": "grade_level", "in": ["3"] }, "implementation": { "python": { @@ -59,7 +59,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-4.txt", - "sha256": "851c6afa4cd8a0ff0f599a38e96c96516dc2cccf0a4a7e945c982efebc89b6b7", + "sha256": "563ee744e2fe295ed1b8e5bcde00b7dfc89aac55730ed01088ed02b2a8175fd2", "condition": { "input": "grade_level", "in": ["4"] }, "implementation": { "python": { diff --git a/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb b/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb index 5413ac10b..09b54e80a 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb +++ b/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb @@ -367,7 +367,7 @@ " try:\n", " analysis = run_sentence_analysis(text)\n", " features = add_engineered_features(analysis[\"formatted_output\"])\n", - " features_json = features_to_json(features)\n", + " features_json = features_to_json(features, cast_to_int=False)\n", " classification = run_classify_complexity(features_json, grade_level, text)\n", " return {\n", " \"analysis\": analysis,\n", diff --git a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt index a803ab0a6..18bddb8cc 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt +++ b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt @@ -5,21 +5,21 @@ * **Description:** The text consists of simple, straightforward language and sentence structures. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: * **Sentence Type:** Primarily simple sentences. (`percent_simple_sentences` is typically > 60%). - * **Sentence Length:** Short sentences. (`avg_sentence_length` is typically < 12 words). + * **Sentence Length:** Short sentences. (`avg_words_per_sentence` is typically < 12 words). * **Subordination:** Very low use of clauses. (`percent_sentences_with_subordinate` is typically < 25%). **Moderately Complex:** * **Description:** The text shows a mix of simple and more complex sentences, introducing some variety in structure without being overly demanding. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: * **Sentence Type:** A balanced mix of sentence types. (`percent_simple_sentences` is typically between 40% and 60%). - * **Sentence Length:** Medium length sentences. (`avg_sentence_length` is typically between 12 and 16 words). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 12 and 16 words). * **Subordination:** A moderate use of clauses. (`percent_sentences_with_subordinate` is typically between 25% and 45%). **Very Complex:** * **Description:** The text features more elaborate sentences with multiple clauses and ideas, requiring more effort from the reader to parse. This is often the default category for grade-level text that isn't simple or exceptionally difficult. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Complex structures are common. (`percent_simple_sentences` is a minority, typically < 40%). - * **Sentence Length:** Longer sentences are frequent. (`avg_sentence_length` is typically between 16 and 19 words). + * **Sentence Length:** Longer sentences are frequent. (`avg_words_per_sentence` is typically between 16 and 19 words). * **Subordination:** Subordinate clauses are a key feature. (`percent_sentences_with_subordinate` is typically > 45%). **Exceedingly Complex:** @@ -30,6 +30,6 @@ * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is consistently present (typically > 12%). * High Syntactic Complexity: `percent_compound_complex_sentences` is significant (typically > 15%). * **Length Indicators:** - * Extreme Sentence Length: `avg_sentence_length` is very long (typically > 19 words). + * Extreme Sentence Length: `avg_words_per_sentence` is very long (typically > 19 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 30%). * Concentrated Length: `percent_very_long_sentences` is notable (typically > 10%). diff --git a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt index 6a81c2b81..815a80a73 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt +++ b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt @@ -5,21 +5,21 @@ * **Description:** The text uses clear, direct language with basic sentence structures appropriate for developing readers. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: * **Sentence Type:** Dominated by simple sentences. (`percent_simple_sentences` is typically > 55%). - * **Sentence Length:** Short to medium sentences. (`avg_sentence_length` is typically < 13 words). + * **Sentence Length:** Short to medium sentences. (`avg_words_per_sentence` is typically < 13 words). * **Subordination:** Infrequent use of clauses. (`percent_sentences_with_subordinate` is typically < 30%). **Moderately Complex:** * **Description:** The text contains a variety of sentence structures, including compound and complex sentences, but remains accessible. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: * **Sentence Type:** A healthy mix of sentence types. (`percent_simple_sentences` is typically between 40% and 55%). - * **Sentence Length:** Medium length sentences. (`avg_sentence_length` is typically between 13 and 17 words). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 13 and 17 words). * **Subordination:** A moderate number of clauses. (`percent_sentences_with_subordinate` is typically between 30% and 50%). **Very Complex:** * **Description:** The text is characterized by longer sentences and the regular use of dependent clauses, requiring readers to track multiple ideas. This is the default for challenging, on-grade-level texts. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Simple sentences are a clear minority. (`percent_simple_sentences` is typically < 40%). - * **Sentence Length:** Sentences are consistently long. (`avg_sentence_length` is typically between 17 and 22 words). + * **Sentence Length:** Sentences are consistently long. (`avg_words_per_sentence` is typically between 17 and 22 words). * **Subordination:** Subordination is a major feature. (`percent_sentences_with_subordinate` is typically > 50%). * **Multiple Subordination:** Sentences with multiple clauses appear more often. (`percent_sentences_with_multiple_subordinates` is typically > 8%). @@ -31,6 +31,6 @@ * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is high and consistent (typically > 15%). * High Syntactic Complexity: `percent_compound_complex_sentences` is a notable feature (typically > 20%). * **Length Indicators:** - * Extreme Sentence Length: `avg_sentence_length` is exceptionally long (typically > 22 words). + * Extreme Sentence Length: `avg_words_per_sentence` is exceptionally long (typically > 22 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 25%). * Concentrated Length: `percent_very_long_sentences` is significant (typically > 15%). diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json index f8a4fa4f6..a89b472c0 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json @@ -38,7 +38,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-3.txt", - "sha256": "3ddacd7b3c80f1f8e85bfcddd01db945813fcf814d893e719f5ae48babdaf11f", + "sha256": "d2522ad29cf7a1edc4288a13d5509001c64776ed38242b9a7b790d621fa978a8", "condition": { "input": "grade_level", "in": ["3"] }, "implementation": { "python": { @@ -59,7 +59,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-4.txt", - "sha256": "851c6afa4cd8a0ff0f599a38e96c96516dc2cccf0a4a7e945c982efebc89b6b7", + "sha256": "563ee744e2fe295ed1b8e5bcde00b7dfc89aac55730ed01088ed02b2a8175fd2", "condition": { "input": "grade_level", "in": ["4"] }, "implementation": { "python": { diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt index a803ab0a6..18bddb8cc 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt @@ -5,21 +5,21 @@ * **Description:** The text consists of simple, straightforward language and sentence structures. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: * **Sentence Type:** Primarily simple sentences. (`percent_simple_sentences` is typically > 60%). - * **Sentence Length:** Short sentences. (`avg_sentence_length` is typically < 12 words). + * **Sentence Length:** Short sentences. (`avg_words_per_sentence` is typically < 12 words). * **Subordination:** Very low use of clauses. (`percent_sentences_with_subordinate` is typically < 25%). **Moderately Complex:** * **Description:** The text shows a mix of simple and more complex sentences, introducing some variety in structure without being overly demanding. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: * **Sentence Type:** A balanced mix of sentence types. (`percent_simple_sentences` is typically between 40% and 60%). - * **Sentence Length:** Medium length sentences. (`avg_sentence_length` is typically between 12 and 16 words). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 12 and 16 words). * **Subordination:** A moderate use of clauses. (`percent_sentences_with_subordinate` is typically between 25% and 45%). **Very Complex:** * **Description:** The text features more elaborate sentences with multiple clauses and ideas, requiring more effort from the reader to parse. This is often the default category for grade-level text that isn't simple or exceptionally difficult. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Complex structures are common. (`percent_simple_sentences` is a minority, typically < 40%). - * **Sentence Length:** Longer sentences are frequent. (`avg_sentence_length` is typically between 16 and 19 words). + * **Sentence Length:** Longer sentences are frequent. (`avg_words_per_sentence` is typically between 16 and 19 words). * **Subordination:** Subordinate clauses are a key feature. (`percent_sentences_with_subordinate` is typically > 45%). **Exceedingly Complex:** @@ -30,6 +30,6 @@ * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is consistently present (typically > 12%). * High Syntactic Complexity: `percent_compound_complex_sentences` is significant (typically > 15%). * **Length Indicators:** - * Extreme Sentence Length: `avg_sentence_length` is very long (typically > 19 words). + * Extreme Sentence Length: `avg_words_per_sentence` is very long (typically > 19 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 30%). * Concentrated Length: `percent_very_long_sentences` is notable (typically > 10%). diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt index 6a81c2b81..815a80a73 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt @@ -5,21 +5,21 @@ * **Description:** The text uses clear, direct language with basic sentence structures appropriate for developing readers. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: * **Sentence Type:** Dominated by simple sentences. (`percent_simple_sentences` is typically > 55%). - * **Sentence Length:** Short to medium sentences. (`avg_sentence_length` is typically < 13 words). + * **Sentence Length:** Short to medium sentences. (`avg_words_per_sentence` is typically < 13 words). * **Subordination:** Infrequent use of clauses. (`percent_sentences_with_subordinate` is typically < 30%). **Moderately Complex:** * **Description:** The text contains a variety of sentence structures, including compound and complex sentences, but remains accessible. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: * **Sentence Type:** A healthy mix of sentence types. (`percent_simple_sentences` is typically between 40% and 55%). - * **Sentence Length:** Medium length sentences. (`avg_sentence_length` is typically between 13 and 17 words). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 13 and 17 words). * **Subordination:** A moderate number of clauses. (`percent_sentences_with_subordinate` is typically between 30% and 50%). **Very Complex:** * **Description:** The text is characterized by longer sentences and the regular use of dependent clauses, requiring readers to track multiple ideas. This is the default for challenging, on-grade-level texts. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Simple sentences are a clear minority. (`percent_simple_sentences` is typically < 40%). - * **Sentence Length:** Sentences are consistently long. (`avg_sentence_length` is typically between 17 and 22 words). + * **Sentence Length:** Sentences are consistently long. (`avg_words_per_sentence` is typically between 17 and 22 words). * **Subordination:** Subordination is a major feature. (`percent_sentences_with_subordinate` is typically > 50%). * **Multiple Subordination:** Sentences with multiple clauses appear more often. (`percent_sentences_with_multiple_subordinates` is typically > 8%). @@ -31,6 +31,6 @@ * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is high and consistent (typically > 15%). * High Syntactic Complexity: `percent_compound_complex_sentences` is a notable feature (typically > 20%). * **Length Indicators:** - * Extreme Sentence Length: `avg_sentence_length` is exceptionally long (typically > 22 words). + * Extreme Sentence Length: `avg_words_per_sentence` is exceptionally long (typically > 22 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 25%). * Concentrated Length: `percent_very_long_sentences` is significant (typically > 15%). diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py b/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py index 5db6682f5..0795728a0 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py @@ -139,7 +139,7 @@ def sentence_features(analysis: Any) -> str: Takes the first step's output, which the contract names as this entry's ``input``. """ counts = analysis.model_dump() if isinstance(analysis, BaseModel) else dict(analysis) - return features_to_json(add_engineered_features(counts)) + return features_to_json(add_engineered_features(counts), cast_to_int=False) class SentenceStructureEvaluator( diff --git a/sdks/python/src/learning_commons_evaluators/features/sentence_features.py b/sdks/python/src/learning_commons_evaluators/features/sentence_features.py index f1e8e2be3..7752b0e78 100644 --- a/sdks/python/src/learning_commons_evaluators/features/sentence_features.py +++ b/sdks/python/src/learning_commons_evaluators/features/sentence_features.py @@ -5,8 +5,8 @@ arithmetic, so the contract declares it as preprocessing (``sentence_features``) rather than asking a model for it, and this module is that computation. -Every derived value is rounded to one place here and cast to a whole number on the way into -the prompt, which is the shape the rubric's thresholds are written against. +Every derived value is rounded to one place here. The classifier preserves that fractional +place so per-sentence averages and phrase densities below one remain meaningful. """ from __future__ import annotations @@ -196,26 +196,20 @@ def add_engineered_features(analysis: Mapping[str, Any]) -> dict[str, Any]: _ratio(_count(analysis, "num_sophisticated_transitions"), transitions) * 100 ) - # Deliberately Python's ties-to-even ``round``, not the SDK's half-up ``round_half_up``. - # These ratios are rationals over small counts, so they land on exact ties often, and a - # tie at ``n.95`` crosses an integer boundary once ``features_to_json`` casts it: 19 - # transitions over 20 sentences is 0.95, which reaches the prompt as 0 under this rule - # and as 1 under the other. This is the arithmetic the notebook and the fixtures were - # produced by; unifying it is DSCR-2223, decided once for both SDKs alongside the - # truncate-versus-round question it belongs with. + # Preserve the notebook's ties-to-even rounding; cross-SDK rounding alignment + # is tracked separately in DSCR-2223. features.update({name: round(value, PRECISION) for name, value in derived.items()}) return features -def features_to_json(features: Mapping[str, Any]) -> str: +def features_to_json(features: Mapping[str, Any], *, cast_to_int: bool = True) -> str: """:data:`FEATURE_COLS` as the JSON object bound to ``{sentence_features}``. - Every value is a whole number: the rubric's thresholds are written in whole percentages - and averages, and the fractional place the model would otherwise read invites precision - it cannot support. A feature the analysis did not produce is sent as ``null`` rather - than zero, so a missing count reads as missing rather than as "none found". + Set ``cast_to_int=False`` to retain one decimal place for the classifier, including + averages and densities below one. The default preserves the legacy serializer API. + A feature the analysis did not produce is sent as ``null`` rather than zero. """ - payload: dict[str, int | None] = {} + payload: dict[str, int | float | None] = {} for name in FEATURE_COLS: value = features.get(name) if isinstance(value, bool) or not isinstance(value, (int, float)): @@ -223,9 +217,7 @@ def features_to_json(features: Mapping[str, Any]) -> str: elif isinstance(value, int): payload[name] = value else: - # Truncated, not rounded: this is the notebook's arithmetic, and the fixtures - # and thresholds this evaluator was validated against were produced by it. - payload[name] = int(value) + payload[name] = int(value) if cast_to_int else round(value, PRECISION) return json.dumps(payload, indent=2) diff --git a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py index fe3881029..4acebc6f6 100644 --- a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py +++ b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py @@ -45,6 +45,9 @@ "num_sentences": 2, "num_words": 20, "sentence_word_counts": [6, 14], + "num_subordinate_clauses": 1, + "num_simple_transitions": 1, + "num_sophisticated_transitions": 0, } ) @@ -91,10 +94,12 @@ async def test_the_classification_step_reads_the_features_derived_from_the_analy ) -> None: await SentenceStructureEvaluator(openai_api_key="o").evaluate(text=TEXT, grade_level=5) user = providers.calls[1]["messages"][1]["content"] - expected = features_to_json(add_engineered_features(ANALYSIS.model_dump())) + expected = features_to_json(add_engineered_features(ANALYSIS.model_dump()), cast_to_int=False) assert expected in user - # Derived, not passed through: 20 words over 2 sentences, truncated for the prompt. + # Derived, not passed through: 20 words over 2 sentences. assert json.loads(expected)["avg_words_per_sentence"] == 10 + assert json.loads(expected)["avg_subordinates_per_sentence"] == 0.5 + assert json.loads(expected)["avg_transitions_per_sentence"] == 0.5 assert TEXT in user @@ -109,6 +114,9 @@ async def test_the_rubric_for_the_grade_is_the_one_the_contract_names( rubric = SentenceStructureEvaluator.contract.document(marker) user = providers.calls[1]["messages"][1]["content"] assert f"# GRADE {grade} RUBRIC\n{rubric}" in user + assert "avg_sentence_length" not in user + if grade in (3, 4): + assert rubric.count("avg_words_per_sentence") == 4 async def test_returns_the_classification_as_the_result(providers: ProviderFactory) -> None: diff --git a/sdks/python/tests/unit/features/test_sentence_features.py b/sdks/python/tests/unit/features/test_sentence_features.py index 449e3168e..b19a0f6dd 100644 --- a/sdks/python/tests/unit/features/test_sentence_features.py +++ b/sdks/python/tests/unit/features/test_sentence_features.py @@ -132,3 +132,19 @@ def test_a_feature_the_analysis_did_not_produce_is_null_rather_than_zero() -> No payload = json.loads(features_to_json({})) assert set(payload) == set(FEATURE_COLS) assert all(value is None for value in payload.values()) + + +@pytest.mark.parametrize( + ("name", "value", "expected"), + [ + ("avg_transitions_per_sentence", 0.20, 0.2), + ("avg_subordinates_per_sentence", 0.43, 0.4), + ("appositive_phrase_density", 0.52, 0.5), + ("participle_phrase_density", 0.29, 0.3), + ("percent_simple_sentences", 60.5, 60.5), + ], +) +def test_fractional_signals_survive_serialization(name: str, value: float, expected: float) -> None: + payload = json.loads(features_to_json({name: value}, cast_to_int=False)) + assert payload[name] == expected + assert payload["num_sentences"] is None diff --git a/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts b/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts index 08712147c..2c811b117 100644 --- a/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts +++ b/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts @@ -83,7 +83,7 @@ export class SentenceStructureEvaluator extends defineMultiStepEvaluator< calculateReadabilityMetrics: (text) => formatGroundTruthCounts(String(text)), // Cast to int and single-space, matching the Python implementation's JSON. addEngineeredFeatures: (analysis) => - featuresToJSON(addEngineeredFeatures(analysis as SentenceAnalysis), 1, true), + featuresToJSON(addEngineeredFeatures(analysis as SentenceAnalysis), 1, false), }, documents: { 'rubric-grade-3.txt': RUBRIC_GRADE_3, diff --git a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts index 3ab155dd4..b0e2fbb53 100644 --- a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts +++ b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts @@ -147,6 +147,40 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { }); }); + it.each(['3', '4'] as const)('preserves fractional signals and valid rubric keys for grade %s', async (grade) => { + vi.mocked(mockProvider.generateStructured) + .mockResolvedValueOnce({ + data: { + ...createMockSentenceAnalysis(), + num_words: 200, + num_subordinate_clauses: 1, + num_simple_transitions: 1, + num_appositive_phrases: 1, + num_participle_phrases: 1, + }, + model: 'gpt-4o', + usage: { inputTokens: 1, outputTokens: 1 }, + latencyMs: 0, + }) + .mockResolvedValueOnce({ + data: { complexity_score: 'slightly_complex', reasoning: 'Mock classification' }, + model: 'gpt-4o', + usage: { inputTokens: 1, outputTokens: 1 }, + latencyMs: 0, + }); + + await evaluator.evaluate({ text: 'Test passage.', grade_level: grade }); + const messages = vi.mocked(mockProvider.generateStructured).mock.calls[1][0].messages; + const prompt = messages.map((message) => message.content).join('\n'); + const features = JSON.parse(prompt.split('# TEXT STATISTICS\n')[1]); + expect(features.avg_subordinates_per_sentence).toBe(0.5); + expect(features.avg_transitions_per_sentence).toBe(0.5); + expect(features.appositive_phrase_density).toBe(0.5); + expect(features.participle_phrase_density).toBe(0.5); + expect(prompt).not.toContain('avg_sentence_length'); + expect(prompt.match(/`avg_words_per_sentence`/g)).toHaveLength(4); + }); + describe('Error Handling', () => { it('should handle sentence analysis API failure', async () => { const testText = 'Test text here for API failure'; From e81747a00c7aade60becf49d55b5ea6d4356d531 Mon Sep 17 00:00:00 2001 From: SeungYeon Lee <307381672+slee430@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:11:52 -0700 Subject: [PATCH 2/4] experiment(sentence-structure): compare raw flooring with explicit rubric boundaries --- .../sentence-structure/config.json | 6 ++-- .../sentence-structure/example_notebook.ipynb | 8 +++-- .../sentence-structure/rubric-grade-3.txt | 22 ++++++------ .../sentence-structure/rubric-grade-4.txt | 24 ++++++------- .../sentence-structure/rubric-grades-5-12.txt | 8 ++--- .../sentence_structure/config.json | 6 ++-- .../sentence_structure/rubric-grade-3.txt | 22 ++++++------ .../sentence_structure/rubric-grade-4.txt | 24 ++++++------- .../sentence_structure/rubric-grades-5-12.txt | 8 ++--- .../ela_reading/sentence_structure.py | 2 +- .../features/sentence_features.py | 28 +++++++++++---- .../ela_reading/test_sentence_structure.py | 23 +++++++----- .../unit/features/test_sentence_features.py | 23 ++++++++++++ .../ela-reading/sentence-structure.ts | 2 +- .../src/features/sentence-features.ts | 8 ++++- .../evaluators/sentence-structure.test.ts | 35 +++++++++++++++---- 16 files changed, 162 insertions(+), 87 deletions(-) diff --git a/evals/text-complexity/ela-reading/sentence-structure/config.json b/evals/text-complexity/ela-reading/sentence-structure/config.json index a89b472c0..52fed76e6 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/config.json +++ b/evals/text-complexity/ela-reading/sentence-structure/config.json @@ -38,7 +38,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-3.txt", - "sha256": "d2522ad29cf7a1edc4288a13d5509001c64776ed38242b9a7b790d621fa978a8", + "sha256": "a62c8f64c951b909bb8f3e2b69bec1c794c797dd1cad14d0b8db7939200e8966", "condition": { "input": "grade_level", "in": ["3"] }, "implementation": { "python": { @@ -59,7 +59,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-4.txt", - "sha256": "563ee744e2fe295ed1b8e5bcde00b7dfc89aac55730ed01088ed02b2a8175fd2", + "sha256": "082c29146c24c654a51658cb97ca49355b90e8961470ffdabcd32d440a1ac733", "condition": { "input": "grade_level", "in": ["4"] }, "implementation": { "python": { @@ -80,7 +80,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grades-5-12.txt", - "sha256": "ae8ac191a5c8c0168e3866d0682c01f987a1b583272a0f6674e3881a84dccfab", + "sha256": "458bae4c6996aee6eb9f27a82f37d51289b508cec671ba4fdcbf7c88731fb3a2", "condition": { "input": "grade_level", "in": ["5", "6", "7", "8", "9", "10", "11", "12"] }, "implementation": { "python": { diff --git a/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb b/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb index 09b54e80a..afd548d2d 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb +++ b/evals/text-complexity/ela-reading/sentence-structure/example_notebook.ipynb @@ -245,13 +245,17 @@ "]\n", "\n", "\n", - "def features_to_json(features: dict, decimals: int = 1, cast_to_int: bool = True) -> str:\n", + "def features_to_json(features: dict, decimals: int = 1, cast_to_int: bool = True, floor_values: bool = False) -> str:\n", + " import math\n", " payload = {}\n", " for key in FEATURE_COLS:\n", " value = features.get(key)\n", " if value is None:\n", " payload[key] = None\n", " continue\n", + " if floor_values:\n", + " payload[key] = math.floor(value)\n", + " continue\n", " value = round(value, decimals)\n", " payload[key] = int(value) if cast_to_int else value\n", " return json.dumps(payload, indent=2)" @@ -367,7 +371,7 @@ " try:\n", " analysis = run_sentence_analysis(text)\n", " features = add_engineered_features(analysis[\"formatted_output\"])\n", - " features_json = features_to_json(features, cast_to_int=False)\n", + " features_json = features_to_json(features, floor_values=True)\n", " classification = run_classify_complexity(features_json, grade_level, text)\n", " return {\n", " \"analysis\": analysis,\n", diff --git a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt index 18bddb8cc..7cc2a0f6a 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt +++ b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-3.txt @@ -4,32 +4,32 @@ **Slightly Complex:** * **Description:** The text consists of simple, straightforward language and sentence structures. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: - * **Sentence Type:** Primarily simple sentences. (`percent_simple_sentences` is typically > 60%). + * **Sentence Type:** Primarily simple sentences. (`percent_simple_sentences` is typically >= 60%). * **Sentence Length:** Short sentences. (`avg_words_per_sentence` is typically < 12 words). * **Subordination:** Very low use of clauses. (`percent_sentences_with_subordinate` is typically < 25%). **Moderately Complex:** * **Description:** The text shows a mix of simple and more complex sentences, introducing some variety in structure without being overly demanding. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: - * **Sentence Type:** A balanced mix of sentence types. (`percent_simple_sentences` is typically between 40% and 60%). - * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 12 and 16 words). - * **Subordination:** A moderate use of clauses. (`percent_sentences_with_subordinate` is typically between 25% and 45%). + * **Sentence Type:** A balanced mix of sentence types. (`percent_simple_sentences` is typically >= 40% and < 60%). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically >= 12 and < 16 words). + * **Subordination:** A moderate use of clauses. (`percent_sentences_with_subordinate` is typically >= 25% and < 45%). **Very Complex:** * **Description:** The text features more elaborate sentences with multiple clauses and ideas, requiring more effort from the reader to parse. This is often the default category for grade-level text that isn't simple or exceptionally difficult. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Complex structures are common. (`percent_simple_sentences` is a minority, typically < 40%). - * **Sentence Length:** Longer sentences are frequent. (`avg_words_per_sentence` is typically between 16 and 19 words). - * **Subordination:** Subordinate clauses are a key feature. (`percent_sentences_with_subordinate` is typically > 45%). + * **Sentence Length:** Longer sentences are frequent. (`avg_words_per_sentence` is typically >= 16 and < 19 words). + * **Subordination:** Subordinate clauses are a key feature. (`percent_sentences_with_subordinate` is typically >= 45%). **Exceedingly Complex:** * **Description:** The text is dense with very long, intricate sentences and a high degree of subordination, making it exceptionally challenging for this grade level. * **Statistical Guidelines:** The text is "Exceedingly Complex" if it shows an extreme combination of sentence length and structural density. It should meet at least **TWO** of the following criteria, including at least **ONE** from the "Structural Density" group. * **Structural Density Indicators:** - * High Subordination: `percent_sentences_with_subordinate` is extensive (typically > 50%). - * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is consistently present (typically > 12%). - * High Syntactic Complexity: `percent_compound_complex_sentences` is significant (typically > 15%). + * High Subordination: `percent_sentences_with_subordinate` is extensive (typically >= 50%). + * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is consistently present (typically >= 12%). + * High Syntactic Complexity: `percent_compound_complex_sentences` is significant (typically >= 15%). * **Length Indicators:** - * Extreme Sentence Length: `avg_words_per_sentence` is very long (typically > 19 words). + * Extreme Sentence Length: `avg_words_per_sentence` is very long (typically >= 19 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 30%). - * Concentrated Length: `percent_very_long_sentences` is notable (typically > 10%). + * Concentrated Length: `percent_very_long_sentences` is notable (typically >= 10%). diff --git a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt index 815a80a73..93fc84eb3 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt +++ b/evals/text-complexity/ela-reading/sentence-structure/rubric-grade-4.txt @@ -4,33 +4,33 @@ **Slightly Complex:** * **Description:** The text uses clear, direct language with basic sentence structures appropriate for developing readers. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: - * **Sentence Type:** Dominated by simple sentences. (`percent_simple_sentences` is typically > 55%). + * **Sentence Type:** Dominated by simple sentences. (`percent_simple_sentences` is typically >= 55%). * **Sentence Length:** Short to medium sentences. (`avg_words_per_sentence` is typically < 13 words). * **Subordination:** Infrequent use of clauses. (`percent_sentences_with_subordinate` is typically < 30%). **Moderately Complex:** * **Description:** The text contains a variety of sentence structures, including compound and complex sentences, but remains accessible. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: - * **Sentence Type:** A healthy mix of sentence types. (`percent_simple_sentences` is typically between 40% and 55%). - * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 13 and 17 words). - * **Subordination:** A moderate number of clauses. (`percent_sentences_with_subordinate` is typically between 30% and 50%). + * **Sentence Type:** A healthy mix of sentence types. (`percent_simple_sentences` is typically >= 40% and < 55%). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically >= 13 and < 17 words). + * **Subordination:** A moderate number of clauses. (`percent_sentences_with_subordinate` is typically >= 30% and < 50%). **Very Complex:** * **Description:** The text is characterized by longer sentences and the regular use of dependent clauses, requiring readers to track multiple ideas. This is the default for challenging, on-grade-level texts. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Simple sentences are a clear minority. (`percent_simple_sentences` is typically < 40%). - * **Sentence Length:** Sentences are consistently long. (`avg_words_per_sentence` is typically between 17 and 22 words). - * **Subordination:** Subordination is a major feature. (`percent_sentences_with_subordinate` is typically > 50%). - * **Multiple Subordination:** Sentences with multiple clauses appear more often. (`percent_sentences_with_multiple_subordinates` is typically > 8%). + * **Sentence Length:** Sentences are consistently long. (`avg_words_per_sentence` is typically >= 17 and < 22 words). + * **Subordination:** Subordination is a major feature. (`percent_sentences_with_subordinate` is typically >= 50%). + * **Multiple Subordination:** Sentences with multiple clauses appear more often. (`percent_sentences_with_multiple_subordinates` is typically >= 8%). **Exceedingly Complex:** * **Description:** The text's structure is highly sophisticated and dense, marked by extensive use of embedded clauses and long, flowing sentences that are well above grade-level expectations. * **Statistical Guidelines:** A text is "Exceedingly Complex" if its structure is highly sophisticated and dense. It should meet at least **TWO** of the following criteria, including at least **ONE** from the "Structural Density" group. * **Structural Density Indicators:** - * High Subordination: `percent_sentences_with_subordinate` is very high (typically > 60%). - * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is high and consistent (typically > 15%). - * High Syntactic Complexity: `percent_compound_complex_sentences` is a notable feature (typically > 20%). + * High Subordination: `percent_sentences_with_subordinate` is very high (typically >= 60%). + * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is high and consistent (typically >= 15%). + * High Syntactic Complexity: `percent_compound_complex_sentences` is a notable feature (typically >= 20%). * **Length Indicators:** - * Extreme Sentence Length: `avg_words_per_sentence` is exceptionally long (typically > 22 words). + * Extreme Sentence Length: `avg_words_per_sentence` is exceptionally long (typically >= 22 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 25%). - * Concentrated Length: `percent_very_long_sentences` is significant (typically > 15%). + * Concentrated Length: `percent_very_long_sentences` is significant (typically >= 15%). diff --git a/evals/text-complexity/ela-reading/sentence-structure/rubric-grades-5-12.txt b/evals/text-complexity/ela-reading/sentence-structure/rubric-grades-5-12.txt index b69c8459c..711ef12e7 100644 --- a/evals/text-complexity/ela-reading/sentence-structure/rubric-grades-5-12.txt +++ b/evals/text-complexity/ela-reading/sentence-structure/rubric-grades-5-12.txt @@ -1,5 +1,5 @@ - **Slightly Complex:** A text is in the Slightly Complex bucket if it has at least 50% simple sentences. If it doesn't, the text is a higher level of complexity. If the % of simple sentences is >= 50% and the % of compound sentences is >= 20%, the text is Moderately Complex, otherwise, the text is Slightly Complex. Slightly Complex texts NEVER have advanced complex sentences — the presence of an advanced complex sentence always leads to a higher level of complexity than Slightly. - **For Moderately Complex:** These texts can take on any distribution of sentence types as long as there aren't more than 2 advanced complex sentences and as long as there aren't so many simple sentences that the text becomes Slightly Complex. That means Moderately Complex texts may have many simple sentences (although not so many that the text is Slightly Complex), compound sentences, and/or basic complex sentences. It's also possible for a moderately complex text to contain one or two advanced complex sentences, as long as there aren't more than 2. If there are more than 2, then the text is either Very or Exceedingly complex. - **Very Complex:** These texts contain 3 or more advanced complex sentences (unless the percentage of advanced complex sentences is >= 65)%, in which case the text becomes Exceedingly Complex). They may still contain many simple, compound, and basic complex sentences, but a text is not Very Complex unless there are 3 or more advanced complex sentences. - **Exceedingly Complex:** These texts have 65%+ of their sentences being advanced complex sentences. + **Slightly Complex:** A text is in the Slightly Complex bucket if it has >= 50% simple sentences. If it doesn't, the text is a higher level of complexity. If the % of simple sentences is >= 50% and the % of compound sentences is >= 20%, the text is Moderately Complex, otherwise, the text is Slightly Complex. Slightly Complex texts NEVER have advanced complex sentences — the presence of an advanced complex sentence always leads to a higher level of complexity than Slightly. + **For Moderately Complex:** These texts can take on any distribution of sentence types as long as there are < 3 advanced complex sentences and as long as there aren't so many simple sentences that the text becomes Slightly Complex. That means Moderately Complex texts may have many simple sentences (although not so many that the text is Slightly Complex), compound sentences, and/or basic complex sentences. It's also possible for a moderately complex text to contain one or two advanced complex sentences, as long as there are < 3. If there are >= 3, then the text is either Very or Exceedingly complex. + **Very Complex:** These texts contain >= 3 advanced complex sentences (unless the percentage of advanced complex sentences is >= 65)%, in which case the text becomes Exceedingly Complex). They may still contain many simple, compound, and basic complex sentences, but a text is not Very Complex unless there are >= 3 advanced complex sentences. + **Exceedingly Complex:** These texts have >= 65% of their sentences being advanced complex sentences. diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json index a89b472c0..52fed76e6 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/config.json @@ -38,7 +38,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-3.txt", - "sha256": "d2522ad29cf7a1edc4288a13d5509001c64776ed38242b9a7b790d621fa978a8", + "sha256": "a62c8f64c951b909bb8f3e2b69bec1c794c797dd1cad14d0b8db7939200e8966", "condition": { "input": "grade_level", "in": ["3"] }, "implementation": { "python": { @@ -59,7 +59,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grade-4.txt", - "sha256": "563ee744e2fe295ed1b8e5bcde00b7dfc89aac55730ed01088ed02b2a8175fd2", + "sha256": "082c29146c24c654a51658cb97ca49355b90e8961470ffdabcd32d440a1ac733", "condition": { "input": "grade_level", "in": ["4"] }, "implementation": { "python": { @@ -80,7 +80,7 @@ "input": "grade_level", "output": "rubric", "source_path": "rubric-grades-5-12.txt", - "sha256": "ae8ac191a5c8c0168e3866d0682c01f987a1b583272a0f6674e3881a84dccfab", + "sha256": "458bae4c6996aee6eb9f27a82f37d51289b508cec671ba4fdcbf7c88731fb3a2", "condition": { "input": "grade_level", "in": ["5", "6", "7", "8", "9", "10", "11", "12"] }, "implementation": { "python": { diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt index 18bddb8cc..7cc2a0f6a 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-3.txt @@ -4,32 +4,32 @@ **Slightly Complex:** * **Description:** The text consists of simple, straightforward language and sentence structures. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: - * **Sentence Type:** Primarily simple sentences. (`percent_simple_sentences` is typically > 60%). + * **Sentence Type:** Primarily simple sentences. (`percent_simple_sentences` is typically >= 60%). * **Sentence Length:** Short sentences. (`avg_words_per_sentence` is typically < 12 words). * **Subordination:** Very low use of clauses. (`percent_sentences_with_subordinate` is typically < 25%). **Moderately Complex:** * **Description:** The text shows a mix of simple and more complex sentences, introducing some variety in structure without being overly demanding. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: - * **Sentence Type:** A balanced mix of sentence types. (`percent_simple_sentences` is typically between 40% and 60%). - * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 12 and 16 words). - * **Subordination:** A moderate use of clauses. (`percent_sentences_with_subordinate` is typically between 25% and 45%). + * **Sentence Type:** A balanced mix of sentence types. (`percent_simple_sentences` is typically >= 40% and < 60%). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically >= 12 and < 16 words). + * **Subordination:** A moderate use of clauses. (`percent_sentences_with_subordinate` is typically >= 25% and < 45%). **Very Complex:** * **Description:** The text features more elaborate sentences with multiple clauses and ideas, requiring more effort from the reader to parse. This is often the default category for grade-level text that isn't simple or exceptionally difficult. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Complex structures are common. (`percent_simple_sentences` is a minority, typically < 40%). - * **Sentence Length:** Longer sentences are frequent. (`avg_words_per_sentence` is typically between 16 and 19 words). - * **Subordination:** Subordinate clauses are a key feature. (`percent_sentences_with_subordinate` is typically > 45%). + * **Sentence Length:** Longer sentences are frequent. (`avg_words_per_sentence` is typically >= 16 and < 19 words). + * **Subordination:** Subordinate clauses are a key feature. (`percent_sentences_with_subordinate` is typically >= 45%). **Exceedingly Complex:** * **Description:** The text is dense with very long, intricate sentences and a high degree of subordination, making it exceptionally challenging for this grade level. * **Statistical Guidelines:** The text is "Exceedingly Complex" if it shows an extreme combination of sentence length and structural density. It should meet at least **TWO** of the following criteria, including at least **ONE** from the "Structural Density" group. * **Structural Density Indicators:** - * High Subordination: `percent_sentences_with_subordinate` is extensive (typically > 50%). - * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is consistently present (typically > 12%). - * High Syntactic Complexity: `percent_compound_complex_sentences` is significant (typically > 15%). + * High Subordination: `percent_sentences_with_subordinate` is extensive (typically >= 50%). + * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is consistently present (typically >= 12%). + * High Syntactic Complexity: `percent_compound_complex_sentences` is significant (typically >= 15%). * **Length Indicators:** - * Extreme Sentence Length: `avg_words_per_sentence` is very long (typically > 19 words). + * Extreme Sentence Length: `avg_words_per_sentence` is very long (typically >= 19 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 30%). - * Concentrated Length: `percent_very_long_sentences` is notable (typically > 10%). + * Concentrated Length: `percent_very_long_sentences` is notable (typically >= 10%). diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt index 815a80a73..93fc84eb3 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grade-4.txt @@ -4,33 +4,33 @@ **Slightly Complex:** * **Description:** The text uses clear, direct language with basic sentence structures appropriate for developing readers. * **Statistical Guidelines:** The text is likely "Slightly Complex" if it meets at least TWO of the following criteria: - * **Sentence Type:** Dominated by simple sentences. (`percent_simple_sentences` is typically > 55%). + * **Sentence Type:** Dominated by simple sentences. (`percent_simple_sentences` is typically >= 55%). * **Sentence Length:** Short to medium sentences. (`avg_words_per_sentence` is typically < 13 words). * **Subordination:** Infrequent use of clauses. (`percent_sentences_with_subordinate` is typically < 30%). **Moderately Complex:** * **Description:** The text contains a variety of sentence structures, including compound and complex sentences, but remains accessible. * **Statistical Guidelines:** If the text is not "Slightly Complex", consider "Moderately Complex" if it generally aligns with these ranges: - * **Sentence Type:** A healthy mix of sentence types. (`percent_simple_sentences` is typically between 40% and 55%). - * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically between 13 and 17 words). - * **Subordination:** A moderate number of clauses. (`percent_sentences_with_subordinate` is typically between 30% and 50%). + * **Sentence Type:** A healthy mix of sentence types. (`percent_simple_sentences` is typically >= 40% and < 55%). + * **Sentence Length:** Medium length sentences. (`avg_words_per_sentence` is typically >= 13 and < 17 words). + * **Subordination:** A moderate number of clauses. (`percent_sentences_with_subordinate` is typically >= 30% and < 50%). **Very Complex:** * **Description:** The text is characterized by longer sentences and the regular use of dependent clauses, requiring readers to track multiple ideas. This is the default for challenging, on-grade-level texts. * **Statistical Guidelines:** If the text is more complex than "Moderately" but does not meet the "Exceedingly" criteria, it is likely "Very Complex". Key indicators include: * **Sentence Type:** Simple sentences are a clear minority. (`percent_simple_sentences` is typically < 40%). - * **Sentence Length:** Sentences are consistently long. (`avg_words_per_sentence` is typically between 17 and 22 words). - * **Subordination:** Subordination is a major feature. (`percent_sentences_with_subordinate` is typically > 50%). - * **Multiple Subordination:** Sentences with multiple clauses appear more often. (`percent_sentences_with_multiple_subordinates` is typically > 8%). + * **Sentence Length:** Sentences are consistently long. (`avg_words_per_sentence` is typically >= 17 and < 22 words). + * **Subordination:** Subordination is a major feature. (`percent_sentences_with_subordinate` is typically >= 50%). + * **Multiple Subordination:** Sentences with multiple clauses appear more often. (`percent_sentences_with_multiple_subordinates` is typically >= 8%). **Exceedingly Complex:** * **Description:** The text's structure is highly sophisticated and dense, marked by extensive use of embedded clauses and long, flowing sentences that are well above grade-level expectations. * **Statistical Guidelines:** A text is "Exceedingly Complex" if its structure is highly sophisticated and dense. It should meet at least **TWO** of the following criteria, including at least **ONE** from the "Structural Density" group. * **Structural Density Indicators:** - * High Subordination: `percent_sentences_with_subordinate` is very high (typically > 60%). - * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is high and consistent (typically > 15%). - * High Syntactic Complexity: `percent_compound_complex_sentences` is a notable feature (typically > 20%). + * High Subordination: `percent_sentences_with_subordinate` is very high (typically >= 60%). + * Multiple Subordinates: `percent_sentences_with_multiple_subordinates` is high and consistent (typically >= 15%). + * High Syntactic Complexity: `percent_compound_complex_sentences` is a notable feature (typically >= 20%). * **Length Indicators:** - * Extreme Sentence Length: `avg_words_per_sentence` is exceptionally long (typically > 22 words). + * Extreme Sentence Length: `avg_words_per_sentence` is exceptionally long (typically >= 22 words). * Low Simplicity: `percent_simple_sentences` is very low (typically < 25%). - * Concentrated Length: `percent_very_long_sentences` is significant (typically > 15%). + * Concentrated Length: `percent_very_long_sentences` is significant (typically >= 15%). diff --git a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grades-5-12.txt b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grades-5-12.txt index b69c8459c..711ef12e7 100644 --- a/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grades-5-12.txt +++ b/sdks/python/src/learning_commons_evaluators/contracts/_generated/text_complexity/ela_reading/sentence_structure/rubric-grades-5-12.txt @@ -1,5 +1,5 @@ - **Slightly Complex:** A text is in the Slightly Complex bucket if it has at least 50% simple sentences. If it doesn't, the text is a higher level of complexity. If the % of simple sentences is >= 50% and the % of compound sentences is >= 20%, the text is Moderately Complex, otherwise, the text is Slightly Complex. Slightly Complex texts NEVER have advanced complex sentences — the presence of an advanced complex sentence always leads to a higher level of complexity than Slightly. - **For Moderately Complex:** These texts can take on any distribution of sentence types as long as there aren't more than 2 advanced complex sentences and as long as there aren't so many simple sentences that the text becomes Slightly Complex. That means Moderately Complex texts may have many simple sentences (although not so many that the text is Slightly Complex), compound sentences, and/or basic complex sentences. It's also possible for a moderately complex text to contain one or two advanced complex sentences, as long as there aren't more than 2. If there are more than 2, then the text is either Very or Exceedingly complex. - **Very Complex:** These texts contain 3 or more advanced complex sentences (unless the percentage of advanced complex sentences is >= 65)%, in which case the text becomes Exceedingly Complex). They may still contain many simple, compound, and basic complex sentences, but a text is not Very Complex unless there are 3 or more advanced complex sentences. - **Exceedingly Complex:** These texts have 65%+ of their sentences being advanced complex sentences. + **Slightly Complex:** A text is in the Slightly Complex bucket if it has >= 50% simple sentences. If it doesn't, the text is a higher level of complexity. If the % of simple sentences is >= 50% and the % of compound sentences is >= 20%, the text is Moderately Complex, otherwise, the text is Slightly Complex. Slightly Complex texts NEVER have advanced complex sentences — the presence of an advanced complex sentence always leads to a higher level of complexity than Slightly. + **For Moderately Complex:** These texts can take on any distribution of sentence types as long as there are < 3 advanced complex sentences and as long as there aren't so many simple sentences that the text becomes Slightly Complex. That means Moderately Complex texts may have many simple sentences (although not so many that the text is Slightly Complex), compound sentences, and/or basic complex sentences. It's also possible for a moderately complex text to contain one or two advanced complex sentences, as long as there are < 3. If there are >= 3, then the text is either Very or Exceedingly complex. + **Very Complex:** These texts contain >= 3 advanced complex sentences (unless the percentage of advanced complex sentences is >= 65)%, in which case the text becomes Exceedingly Complex). They may still contain many simple, compound, and basic complex sentences, but a text is not Very Complex unless there are >= 3 advanced complex sentences. + **Exceedingly Complex:** These texts have >= 65% of their sentences being advanced complex sentences. diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py b/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py index 0795728a0..b23d4fc52 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/text_complexity/ela_reading/sentence_structure.py @@ -139,7 +139,7 @@ def sentence_features(analysis: Any) -> str: Takes the first step's output, which the contract names as this entry's ``input``. """ counts = analysis.model_dump() if isinstance(analysis, BaseModel) else dict(analysis) - return features_to_json(add_engineered_features(counts), cast_to_int=False) + return features_to_json(add_engineered_features(counts, round_values=False), floor_values=True) class SentenceStructureEvaluator( diff --git a/sdks/python/src/learning_commons_evaluators/features/sentence_features.py b/sdks/python/src/learning_commons_evaluators/features/sentence_features.py index 7752b0e78..dafd0cfc0 100644 --- a/sdks/python/src/learning_commons_evaluators/features/sentence_features.py +++ b/sdks/python/src/learning_commons_evaluators/features/sentence_features.py @@ -5,8 +5,8 @@ arithmetic, so the contract declares it as preprocessing (``sentence_features``) rather than asking a model for it, and this module is that computation. -Every derived value is rounded to one place here. The classifier preserves that fractional -place so per-sentence averages and phrase densities below one remain meaningful. +The classifier requests unrounded derived values, then floors them directly for integer +rubric boundaries. The helper defaults retain their previous behavior for other callers. """ from __future__ import annotations @@ -160,9 +160,12 @@ def _length_distribution(word_counts: Sequence[float]) -> dict[str, float]: return distribution -def add_engineered_features(analysis: Mapping[str, Any]) -> dict[str, Any]: +def add_engineered_features( + analysis: Mapping[str, Any], *, round_values: bool = True +) -> dict[str, Any]: """The analysis plus every feature derived from it, each rounded to one place. + Set ``round_values=False`` before flooring so rounding cannot cross a rubric boundary. Takes the first step's output as a mapping and returns a new one; the counts are copied through unchanged, because the classifier prompt asks for several of them alongside the ratios (see :data:`FEATURE_COLS`). @@ -198,15 +201,24 @@ def add_engineered_features(analysis: Mapping[str, Any]) -> dict[str, Any]: # Preserve the notebook's ties-to-even rounding; cross-SDK rounding alignment # is tracked separately in DSCR-2223. - features.update({name: round(value, PRECISION) for name, value in derived.items()}) + features.update( + { + name: round(value, PRECISION) if round_values else value + for name, value in derived.items() + } + ) return features -def features_to_json(features: Mapping[str, Any], *, cast_to_int: bool = True) -> str: +def features_to_json( + features: Mapping[str, Any], *, cast_to_int: bool = True, floor_values: bool = False +) -> str: """:data:`FEATURE_COLS` as the JSON object bound to ``{sentence_features}``. - Set ``cast_to_int=False`` to retain one decimal place for the classifier, including - averages and densities below one. The default preserves the legacy serializer API. + ``floor_values=True`` floors the supplied values without preliminary rounding and + takes precedence over ``cast_to_int``. Supply unrounded features for this mode. + Otherwise, ``cast_to_int=False`` retains one decimal place. Defaults preserve the + legacy serializer API. A feature the analysis did not produce is sent as ``null`` rather than zero. """ payload: dict[str, int | float | None] = {} @@ -216,6 +228,8 @@ def features_to_json(features: Mapping[str, Any], *, cast_to_int: bool = True) - payload[name] = None elif isinstance(value, int): payload[name] = value + elif floor_values: + payload[name] = math.floor(value) else: payload[name] = int(value) if cast_to_int else round(value, PRECISION) return json.dumps(payload, indent=2) diff --git a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py index 4acebc6f6..8152259d1 100644 --- a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py +++ b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py @@ -41,10 +41,10 @@ for name, field in SentenceAnalysis.model_fields.items() if field.annotation is float }, - "reasoning": "two sentences", - "num_sentences": 2, - "num_words": 20, - "sentence_word_counts": [6, 14], + "reasoning": "synthetic sentence-length boundary case", + "num_sentences": 25, + "num_words": 299, + "sentence_word_counts": [12] * 24 + [11], "num_subordinate_clauses": 1, "num_simple_transitions": 1, "num_sophisticated_transitions": 0, @@ -94,12 +94,14 @@ async def test_the_classification_step_reads_the_features_derived_from_the_analy ) -> None: await SentenceStructureEvaluator(openai_api_key="o").evaluate(text=TEXT, grade_level=5) user = providers.calls[1]["messages"][1]["content"] - expected = features_to_json(add_engineered_features(ANALYSIS.model_dump()), cast_to_int=False) + expected = features_to_json( + add_engineered_features(ANALYSIS.model_dump(), round_values=False), floor_values=True + ) assert expected in user - # Derived, not passed through: 20 words over 2 sentences. - assert json.loads(expected)["avg_words_per_sentence"] == 10 - assert json.loads(expected)["avg_subordinates_per_sentence"] == 0.5 - assert json.loads(expected)["avg_transitions_per_sentence"] == 0.5 + # Floor the raw ratio 299/25 = 11.96, without rounding it up to 12 first. + assert json.loads(expected)["avg_words_per_sentence"] == 11 + assert json.loads(expected)["avg_subordinates_per_sentence"] == 0 + assert json.loads(expected)["avg_transitions_per_sentence"] == 0 assert TEXT in user @@ -117,6 +119,9 @@ async def test_the_rubric_for_the_grade_is_the_one_the_contract_names( assert "avg_sentence_length" not in user if grade in (3, 4): assert rubric.count("avg_words_per_sentence") == 4 + assert "typically between " not in rubric + assert " > " not in rubric + assert ">= " in rubric async def test_returns_the_classification_as_the_result(providers: ProviderFactory) -> None: diff --git a/sdks/python/tests/unit/features/test_sentence_features.py b/sdks/python/tests/unit/features/test_sentence_features.py index b19a0f6dd..714f3d2c8 100644 --- a/sdks/python/tests/unit/features/test_sentence_features.py +++ b/sdks/python/tests/unit/features/test_sentence_features.py @@ -148,3 +148,26 @@ def test_fractional_signals_survive_serialization(name: str, value: float, expec payload = json.loads(features_to_json({name: value}, cast_to_int=False)) assert payload[name] == expected assert payload["num_sentences"] is None + + +@pytest.mark.parametrize("boundary", [12, 13, 16, 17, 19, 22, 25, 30, 40, 45, 50, 55, 60]) +@pytest.mark.parametrize("offset", [-0.04, 0, 0.04]) +def test_flooring_preserves_integer_boundary_comparisons(boundary: int, offset: float) -> None: + raw = boundary + offset + payload = json.loads(features_to_json({"avg_words_per_sentence": raw}, floor_values=True)) + floored = payload["avg_words_per_sentence"] + assert (floored < boundary) == (raw < boundary) + assert (floored >= boundary) == (raw >= boundary) + + +def test_floor_uses_the_unrounded_ratio_and_handles_negative_values() -> None: + analysis = {**ANALYSIS, "num_sentences": 25, "num_words": 299, "flesch_kincaid_grade": -0.4} + raw = add_engineered_features(analysis, round_values=False) + assert raw["avg_words_per_sentence"] == 11.96 + payload = json.loads(features_to_json(raw, floor_values=True)) + assert payload["avg_words_per_sentence"] == 11 + assert payload["flesch_kincaid_grade"] == -1 + assert payload["avg_subordinates_per_sentence"] == 0 + # The legacy helper still rounds, and its serializer still truncates. + assert add_engineered_features(analysis)["avg_words_per_sentence"] == 12.0 + assert json.loads(features_to_json(raw))["flesch_kincaid_grade"] == 0 diff --git a/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts b/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts index 2c811b117..7fa5901bb 100644 --- a/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts +++ b/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts @@ -83,7 +83,7 @@ export class SentenceStructureEvaluator extends defineMultiStepEvaluator< calculateReadabilityMetrics: (text) => formatGroundTruthCounts(String(text)), // Cast to int and single-space, matching the Python implementation's JSON. addEngineeredFeatures: (analysis) => - featuresToJSON(addEngineeredFeatures(analysis as SentenceAnalysis), 1, false), + featuresToJSON(addEngineeredFeatures(analysis as SentenceAnalysis), 1, true, true), }, documents: { 'rubric-grade-3.txt': RUBRIC_GRADE_3, diff --git a/sdks/typescript/src/features/sentence-features.ts b/sdks/typescript/src/features/sentence-features.ts index d17e4797a..010765695 100644 --- a/sdks/typescript/src/features/sentence-features.ts +++ b/sdks/typescript/src/features/sentence-features.ts @@ -208,13 +208,19 @@ export const FEATURE_COLS = [ export function featuresToJSON( features: SentenceFeatures, decimals = 1, - castToInt = true + castToInt = true, + floorValues = false ): string { const payload: Record = {}; for (const col of FEATURE_COLS) { const value = features[col as keyof SentenceFeatures]; if (typeof value === 'number') { + // Floor the raw value: preliminary rounding can cross integer rubric boundaries. + if (floorValues) { + payload[col] = Math.floor(value); + continue; + } const rounded = Math.round(value * Math.pow(10, decimals)) / Math.pow(10, decimals); payload[col] = castToInt ? Math.round(rounded) : rounded; } else { diff --git a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts index b0e2fbb53..347ed39d1 100644 --- a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts +++ b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts @@ -2,6 +2,7 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { SentenceStructureEvaluator } from '../../../src/evaluators/text-complexity/ela-reading/sentence-structure.js'; import { ConfigurationError } from '../../../src/errors.js'; import type { LLMProvider } from '../../../src/providers/base.js'; +import { addEngineeredFeatures, featuresToJSON } from '../../../src/features/sentence-features.js'; import CONFIG from '../../../../../evals/text-complexity/ela-reading/sentence-structure/config.json'; // Derived from the contract rather than copied from it, so a model re-pin in @@ -147,12 +148,15 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { }); }); - it.each(['3', '4'] as const)('preserves fractional signals and valid rubric keys for grade %s', async (grade) => { + it.each(['3', '4'] as const)('floors raw features and uses explicit rubric boundaries for grade %s', async (grade) => { vi.mocked(mockProvider.generateStructured) .mockResolvedValueOnce({ data: { ...createMockSentenceAnalysis(), - num_words: 200, + num_sentences: 25, + num_words: 299, + sentence_word_counts: [...Array(24).fill(12), 11], + flesch_kincaid_grade: -0.4, num_subordinate_clauses: 1, num_simple_transitions: 1, num_appositive_phrases: 1, @@ -173,10 +177,15 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { const messages = vi.mocked(mockProvider.generateStructured).mock.calls[1][0].messages; const prompt = messages.map((message) => message.content).join('\n'); const features = JSON.parse(prompt.split('# TEXT STATISTICS\n')[1]); - expect(features.avg_subordinates_per_sentence).toBe(0.5); - expect(features.avg_transitions_per_sentence).toBe(0.5); - expect(features.appositive_phrase_density).toBe(0.5); - expect(features.participle_phrase_density).toBe(0.5); + expect(features.avg_subordinates_per_sentence).toBe(0); + expect(features.avg_transitions_per_sentence).toBe(0); + expect(features.appositive_phrase_density).toBe(0); + expect(features.participle_phrase_density).toBe(0); + expect(features.avg_words_per_sentence).toBe(11); + expect(features.flesch_kincaid_grade).toBe(-1); + expect(prompt).not.toContain('typically between '); + expect(prompt).not.toContain(' > '); + expect(prompt).toContain('>= '); expect(prompt).not.toContain('avg_sentence_length'); expect(prompt.match(/`avg_words_per_sentence`/g)).toHaveLength(4); }); @@ -292,3 +301,17 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { }); }); + +describe('Raw floor boundary behavior', () => { + it.each([12, 13, 16, 17, 19, 22, 25, 30, 40, 45, 50, 55, 60])( + 'preserves comparisons at integer boundary %s', (boundary) => { + for (const offset of [-0.04, 0, 0.04]) { + const raw = boundary + offset; + const features = { ...addEngineeredFeatures(createMockSentenceAnalysis()), avg_words_per_sentence: raw }; + const payload = JSON.parse(featuresToJSON(features, 1, true, true)); + expect(payload.avg_words_per_sentence < boundary).toBe(raw < boundary); + expect(payload.avg_words_per_sentence >= boundary).toBe(raw >= boundary); + } + } + ); +}); From 9b688ce055e52e209f12ac9bab30d167e67c6a72 Mon Sep 17 00:00:00 2001 From: SeungYeon Lee <307381672+slee430@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:06:08 -0800 Subject: [PATCH 3/4] test(sentence-structure): guard percentage boundaries in classifier requests --- .../ela_reading/test_sentence_structure.py | 57 +++++++++++++++++++ .../evaluators/sentence-structure.test.ts | 53 +++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py index 8152259d1..cba92e528 100644 --- a/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py +++ b/sdks/python/tests/unit/evaluators/text_complexity/ela_reading/test_sentence_structure.py @@ -105,6 +105,63 @@ async def test_the_classification_step_reads_the_features_derived_from_the_analy assert TEXT in user +@pytest.mark.parametrize( + ("grade", "count_name", "numerator", "denominator", "expected", "boundary", "at_or_above"), + [ + (3, "num_sentences_with_subordinate", 624, 2500, 24, 25, False), + (3, "num_sentences_with_subordinate", 449, 1000, 44, 45, False), + (3, "num_sentences_with_subordinate", 1124, 2500, 44, 45, False), + (3, "num_sentences_with_subordinate", 45, 100, 45, 45, True), + (3, "num_simple_sentences", 1499, 2500, 59, 60, False), + (3, "num_simple_sentences", 3, 5, 60, 60, True), + (3, "num_simple_sentences", 121, 200, 60, 60, True), + (4, "num_sentences_with_subordinate", 749, 2500, 29, 30, False), + (4, "num_sentences_with_subordinate", 1249, 2500, 49, 50, False), + (4, "num_sentences_with_subordinate", 1, 2, 50, 50, True), + (4, "num_sentences_with_multiple_subordinates", 199, 2500, 7, 8, False), + (4, "num_sentences_with_multiple_subordinates", 2, 25, 8, 8, True), + ], +) +async def test_percentage_boundaries_in_classification_request( + providers: ProviderFactory, + monkeypatch: pytest.MonkeyPatch, + grade: int, + count_name: str, + numerator: int, + denominator: int, + expected: int, + boundary: int, + at_or_above: bool, +) -> None: + # Mock the grammatical counts, then inspect the real classification request. + # In particular, 44.9 catches TS whole-number rounding; 44.96 and 59.96 + # catch rounding to one decimal before truncation in either SDK. + monkeypatch.setitem( + globals(), + "ANALYSIS", + ANALYSIS.model_copy( + update={ + "num_sentences": denominator, + "num_words": denominator * 10, + "sentence_word_counts": [10] * denominator, + count_name: numerator, + } + ), + ) + await SentenceStructureEvaluator(openai_api_key="o").evaluate(text=TEXT, grade_level=grade) + user = providers.calls[1]["messages"][1]["content"] + features = json.loads(user.split("# TEXT STATISTICS\n", 1)[1]) + feature_name = count_name.replace("num_", "percent_", 1) + assert features[feature_name] == expected + assert (features[feature_name] >= boundary) is at_or_above + assert (features[feature_name] < boundary) is (not at_or_above) + assert any( + f"`{feature_name}`" in line and f">= {boundary}%" in line for line in user.splitlines() + ) + # The mocked final label is not evidence of LLM accuracy. This test guards + # the numbers and inclusive cutoff instructions that the classifier receives. + + @pytest.mark.parametrize( ("grade", "marker"), [(3, "rubric-grade-3.txt"), (4, "rubric-grade-4.txt"), (9, "rubric-grades-5-12.txt")], diff --git a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts index 347ed39d1..a49071b3d 100644 --- a/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts +++ b/sdks/typescript/tests/unit/evaluators/sentence-structure.test.ts @@ -190,6 +190,59 @@ describe('SentenceStructureEvaluator - Evaluation Flow', () => { expect(prompt.match(/`avg_words_per_sentence`/g)).toHaveLength(4); }); + it.each([ + ['3', 'num_sentences_with_subordinate', 624, 2500, 24, 25, false], + ['3', 'num_sentences_with_subordinate', 449, 1000, 44, 45, false], + ['3', 'num_sentences_with_subordinate', 1124, 2500, 44, 45, false], + ['3', 'num_sentences_with_subordinate', 45, 100, 45, 45, true], + ['3', 'num_simple_sentences', 1499, 2500, 59, 60, false], + ['3', 'num_simple_sentences', 3, 5, 60, 60, true], + ['3', 'num_simple_sentences', 121, 200, 60, 60, true], + ['4', 'num_sentences_with_subordinate', 749, 2500, 29, 30, false], + ['4', 'num_sentences_with_subordinate', 1249, 2500, 49, 50, false], + ['4', 'num_sentences_with_subordinate', 1, 2, 50, 50, true], + ['4', 'num_sentences_with_multiple_subordinates', 199, 2500, 7, 8, false], + ['4', 'num_sentences_with_multiple_subordinates', 2, 25, 8, 8, true], + ] as const)('percentage boundaries in classification request: grade %s, %s = %i/%i', async ( + grade, countName, numerator, denominator, expected, boundary, atOrAbove, + ) => { + // Mock counts, not the preprocessing. 44.9 catches whole-number rounding; + // 44.96 and 59.96 catch rounding to one decimal before truncation. + vi.mocked(mockProvider.generateStructured) + .mockResolvedValueOnce({ + data: { + ...createMockSentenceAnalysis(), + num_sentences: denominator, + num_words: denominator * 10, + sentence_word_counts: Array(denominator).fill(10), + [countName]: numerator, + }, + model: 'gpt-4o', + usage: { inputTokens: 1, outputTokens: 1 }, + latencyMs: 0, + }) + .mockResolvedValueOnce({ + data: { complexity_score: 'slightly_complex', reasoning: 'Mock classification' }, + model: 'gpt-4o', + usage: { inputTokens: 1, outputTokens: 1 }, + latencyMs: 0, + }); + + await evaluator.evaluate({ text: 'Test passage.', grade_level: grade }); + const messages = vi.mocked(mockProvider.generateStructured).mock.calls[1][0].messages; + const prompt = messages.map((message) => message.content).join('\n'); + const features = JSON.parse(prompt.split('# TEXT STATISTICS\n')[1]); + const featureName = countName.replace('num_', 'percent_'); + expect(features[featureName]).toBe(expected); + expect(features[featureName] >= boundary).toBe(atOrAbove); + expect(features[featureName] < boundary).toBe(!atOrAbove); + expect(prompt.split('\n').some((line) => + line.includes(`\`${featureName}\``) && line.includes(`>= ${boundary}%`), + )).toBe(true); + // The mocked final label does not test LLM accuracy; the request's numeric + // values and inclusive cutoff instructions are the regression contract. + }); + describe('Error Handling', () => { it('should handle sentence analysis API failure', async () => { const testText = 'Test text here for API failure'; From 82d5ee557b4dc5289fd57cc11ae2107a6c81a763 Mon Sep 17 00:00:00 2001 From: SeungYeon Lee <307381672+slee430@users.noreply.github.com> Date: Thu, 5 Nov 2026 15:42:56 -0800 Subject: [PATCH 4/4] fix(sentence-structure): address review cleanup --- .../features/sentence_features.py | 19 ++++++++----------- .../unit/features/test_sentence_features.py | 16 ---------------- .../ela-reading/sentence-structure.ts | 3 ++- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/sdks/python/src/learning_commons_evaluators/features/sentence_features.py b/sdks/python/src/learning_commons_evaluators/features/sentence_features.py index dafd0cfc0..6335cd776 100644 --- a/sdks/python/src/learning_commons_evaluators/features/sentence_features.py +++ b/sdks/python/src/learning_commons_evaluators/features/sentence_features.py @@ -199,8 +199,8 @@ def add_engineered_features( _ratio(_count(analysis, "num_sophisticated_transitions"), transitions) * 100 ) - # Preserve the notebook's ties-to-even rounding; cross-SDK rounding alignment - # is tracked separately in DSCR-2223. + # Keep one-decimal rounding for default callers. The sentence-structure evaluator + # sets round_values=False so its serializer can floor the unrounded values. features.update( { name: round(value, PRECISION) if round_values else value @@ -210,18 +210,15 @@ def add_engineered_features( return features -def features_to_json( - features: Mapping[str, Any], *, cast_to_int: bool = True, floor_values: bool = False -) -> str: +def features_to_json(features: Mapping[str, Any], *, floor_values: bool = False) -> str: """:data:`FEATURE_COLS` as the JSON object bound to ``{sentence_features}``. - ``floor_values=True`` floors the supplied values without preliminary rounding and - takes precedence over ``cast_to_int``. Supply unrounded features for this mode. - Otherwise, ``cast_to_int=False`` retains one decimal place. Defaults preserve the - legacy serializer API. + ``floor_values=True`` floors the supplied values without preliminary rounding. + Supply unrounded features for this mode. The default preserves the legacy behavior + of truncating the supplied values to integers. A feature the analysis did not produce is sent as ``null`` rather than zero. """ - payload: dict[str, int | float | None] = {} + payload: dict[str, int | None] = {} for name in FEATURE_COLS: value = features.get(name) if isinstance(value, bool) or not isinstance(value, (int, float)): @@ -231,7 +228,7 @@ def features_to_json( elif floor_values: payload[name] = math.floor(value) else: - payload[name] = int(value) if cast_to_int else round(value, PRECISION) + payload[name] = int(value) return json.dumps(payload, indent=2) diff --git a/sdks/python/tests/unit/features/test_sentence_features.py b/sdks/python/tests/unit/features/test_sentence_features.py index 714f3d2c8..75fb5ca3c 100644 --- a/sdks/python/tests/unit/features/test_sentence_features.py +++ b/sdks/python/tests/unit/features/test_sentence_features.py @@ -134,22 +134,6 @@ def test_a_feature_the_analysis_did_not_produce_is_null_rather_than_zero() -> No assert all(value is None for value in payload.values()) -@pytest.mark.parametrize( - ("name", "value", "expected"), - [ - ("avg_transitions_per_sentence", 0.20, 0.2), - ("avg_subordinates_per_sentence", 0.43, 0.4), - ("appositive_phrase_density", 0.52, 0.5), - ("participle_phrase_density", 0.29, 0.3), - ("percent_simple_sentences", 60.5, 60.5), - ], -) -def test_fractional_signals_survive_serialization(name: str, value: float, expected: float) -> None: - payload = json.loads(features_to_json({name: value}, cast_to_int=False)) - assert payload[name] == expected - assert payload["num_sentences"] is None - - @pytest.mark.parametrize("boundary", [12, 13, 16, 17, 19, 22, 25, 30, 40, 45, 50, 55, 60]) @pytest.mark.parametrize("offset", [-0.04, 0, 0.04]) def test_flooring_preserves_integer_boundary_comparisons(boundary: int, offset: float) -> None: diff --git a/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts b/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts index 7fa5901bb..b1c5fdac7 100644 --- a/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts +++ b/sdks/typescript/src/evaluators/text-complexity/ela-reading/sentence-structure.ts @@ -81,7 +81,8 @@ export class SentenceStructureEvaluator extends defineMultiStepEvaluator< // The contract names the metrics function; the block layout is what the prompt // declares, so the two travel together. calculateReadabilityMetrics: (text) => formatGroundTruthCounts(String(text)), - // Cast to int and single-space, matching the Python implementation's JSON. + // Floor unrounded values before serialization, matching the Python evaluator. + // TODO: Use an options object for featuresToJSON; floorValues=true ignores decimals and castToInt. addEngineeredFeatures: (analysis) => featuresToJSON(addEngineeredFeatures(analysis as SentenceAnalysis), 1, true, true), },