From 511ee489c396bff7498838eff7a8d726abf8edf3 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Sat, 3 Oct 2026 17:32:11 +0200 Subject: [PATCH] fix(tutor-quality): check every Topic an Example activates for a masterable LEARN path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The authoring gate checked prerequisites and the executable LEARN path only for Topics that a quality case expects. A Topic that a real Example merely activates was never checked. In the production catalog, variables-and-serial was activated on 16 of 35 Examples and unresolved from the first turn (its Concept serial-output required an int-only prerequisite), which blocked the Topic-driven Tutor with content-exhausted for the whole session. New catalog-wide invariant, independent of quality cases: for every Example in the manifest and every Topic the production matcher activates on its main sketch, a learner who answers every planned question successfully (production planner, the Example's effective LEARN strategy) must reach Topic mastery. No provider is involved; weak-answer exhaustion and DEEPEN capacity stay out of scope. Issue code unmasterable-example-activation, with the Example ID. The strict LEARN loop moves into strictLearnPath and is shared with the existing per-case executable-path check (behavior unchanged). RED/GREEN: 6 validator tests and 1 directory test (RED without the wiring). Against the real Course Content: 16 issues at 2ac716f, none with the serial-output prerequisite removed. TutorQuality SSOT §4 states the check. Co-Authored-By: Claude Opus 5.5 --- scripts/validate-tutor-course-content.mjs | 2 +- .../tutor-quality-directory-validator.ts | 17 +++- .../course-content/tutor-quality-validator.ts | 95 ++++++++++++------ ssot/ssot_function_definition_TutorQuality.md | 11 +++ .../tutor-quality-directory-validator.test.ts | 96 ++++++++++++++++++- .../tutor-quality-validator.test.ts | 72 ++++++++++++++ 6 files changed, 261 insertions(+), 32 deletions(-) diff --git a/scripts/validate-tutor-course-content.mjs b/scripts/validate-tutor-course-content.mjs index 403ed03c8..96986ecda 100644 --- a/scripts/validate-tutor-course-content.mjs +++ b/scripts/validate-tutor-course-content.mjs @@ -12,7 +12,7 @@ if (directory === undefined || directory.length === 0) { console.log(`Tutor Course Content quality passed: ${resolved}`); } else { for (const issue of issues) { - const scope = [issue.caseId, issue.topicId, issue.conceptId, issue.indicatorId].filter(Boolean).join("/"); + const scope = [issue.caseId, issue.exampleId, issue.topicId, issue.conceptId, issue.indicatorId].filter(Boolean).join("/"); const scopeSuffix = scope ? ` [${scope}]` : ""; console.error(`${issue.code}${scopeSuffix}: ${issue.message}`); } diff --git a/server/services/course-content/tutor-quality-directory-validator.ts b/server/services/course-content/tutor-quality-directory-validator.ts index 31ddae992..15e67659d 100644 --- a/server/services/course-content/tutor-quality-directory-validator.ts +++ b/server/services/course-content/tutor-quality-directory-validator.ts @@ -5,7 +5,9 @@ import { CourseContentLoader, type LoadedCourseContentSnapshot } from "./course- import { tutorQualityCasesSchema } from "./tutor-quality-schema"; import { BUILT_IN_TUTOR_STRATEGY, type EffectiveTutorStrategy } from "../tutor/strategy/effective-tutor-strategy"; import { + validateExampleTopicActivations, validateTutorContentQuality, + type CatalogExample, type ResolvedTutorQualityCase, type TutorContentQualityIssue, } from "./tutor-quality-validator"; @@ -35,7 +37,10 @@ export async function validateTutorCourseContentDirectory(directory: string): Pr } const resolution = resolveCases(parsed.data.cases, loaded); if (resolution.issues.length > 0) return resolution.issues; - return validateTutorContentQuality(loaded.tutor.topics, resolution.cases); + return [ + ...validateTutorContentQuality(loaded.tutor.topics, resolution.cases), + ...validateExampleTopicActivations(loaded.tutor.topics, catalogExamples(loaded)), + ]; } catch { return [directoryIssue("invalid-quality-cases", "Tutor quality case manifest could not be loaded")]; } @@ -83,6 +88,16 @@ function resolveCases( return { cases: resolved, issues }; } +// Every Example's main sketch, as the Tutor receives it, with its effective LEARN strategy. +function catalogExamples(loaded: LoadedCourseContentSnapshot): CatalogExample[] { + return loaded.examples.flatMap((example) => { + const main = example.files.find(({ name }) => name === example.main); + return main + ? [{ id: example.id, code: main.content, learnStrategy: resolveCaseStrategy(loaded, example.tutorAnnotation?.strategy, "LEARN") }] + : []; + }); +} + function resolveCaseStrategy( loaded: LoadedCourseContentSnapshot, exampleStrategyId: string | undefined, diff --git a/server/services/course-content/tutor-quality-validator.ts b/server/services/course-content/tutor-quality-validator.ts index fccc893eb..8d65a5c5d 100644 --- a/server/services/course-content/tutor-quality-validator.ts +++ b/server/services/course-content/tutor-quality-validator.ts @@ -36,6 +36,7 @@ export type TutorContentQualityIssueCode = | "missing-deepening-question-kind" | "learn-content-exhausted" | "deepen-content-exhausted" + | "unmasterable-example-activation" | "invalid-course-content-bundle" | "invalid-quality-cases" | "quality-case-example-not-found"; @@ -44,6 +45,7 @@ export interface TutorContentQualityIssue { readonly code: TutorContentQualityIssueCode; readonly message: string; readonly caseId?: string; + readonly exampleId?: string; readonly topicId?: string; readonly conceptId?: string; readonly indicatorId?: string; @@ -228,33 +230,7 @@ function validateExecutablePath( issues: TutorContentQualityIssue[], ): void { const planner = new DefaultLearningPlanner(); - const history: TutorDialogTurn[] = []; - let masteryReached = false; - for (let step = 0; step <= topic.questions.length; step += 1) { - const observations = collectObservations(topic, history); - const classification = classifyTopic( - topic, - context.facts, - observations, - new Set(history.flatMap(({ questionId }) => questionId ? [questionId] : [])), - 30, - context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY, - ); - if (classification.status === "mastered") { - masteryReached = true; - break; - } - const plan = planner.start( - topic, - "0".repeat(40), - context.facts, - history, - 30, - context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY, - ); - if (!plan) break; - history.push(successfulTurn(plan.brief)); - } + const { history, masteryReached } = strictLearnPath(topic, context.facts, context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY); if (!masteryReached) { issues.push(issue("learn-content-exhausted", "Default strict progression cannot reach Topic mastery", context.qualityCase.id, topic.id)); return; @@ -288,6 +264,71 @@ function validateExecutablePath( issues.push(issue("deepen-content-exhausted", "Default strict progression cannot satisfy DEEPEN", context.qualityCase.id, topic.id)); } +/** + * The deterministic LEARN path of a learner who answers every planned question successfully, + * using the production planner and Topic classification. It needs no provider: if even this + * path cannot reach Topic mastery, no learner can. + */ +function strictLearnPath( + topic: CurriculumTopic, + facts: SketchFacts, + strategy: EffectiveTutorStrategy, +): { readonly history: TutorDialogTurn[]; readonly masteryReached: boolean } { + const planner = new DefaultLearningPlanner(); + const history: TutorDialogTurn[] = []; + for (let step = 0; step <= topic.questions.length; step += 1) { + const classification = classifyTopic( + topic, + facts, + collectObservations(topic, history), + new Set(history.flatMap(({ questionId }) => questionId ? [questionId] : [])), + 30, + strategy, + ); + if (classification.status === "mastered") return { history, masteryReached: true }; + const plan = planner.start(topic, "0".repeat(40), facts, history, 30, strategy); + if (!plan) break; + history.push(successfulTurn(plan.brief)); + } + return { history, masteryReached: false }; +} + +export interface CatalogExample { + readonly id: string; + readonly code: string; + readonly learnStrategy?: EffectiveTutorStrategy; +} + +/** + * Catalog-wide authoring invariant, independent of quality cases: every Topic that the + * production matcher activates on an Example must be masterable on that Example's sketch by a + * learner who answers every planned question successfully. This finds Topics that are + * unresolved from the first turn (LearningQuestions SSOT 2.3), for example through a + * prerequisite Concept that the sketch cannot probe. Later exhaustion after weak answers and + * post-mastery (DEEPEN) capacity stay out of scope here. + */ +export function validateExampleTopicActivations( + topics: readonly CurriculumTopic[], + examples: readonly CatalogExample[], +): TutorContentQualityIssue[] { + const extractor = new DefaultSketchFactExtractor(); + const matcher = new DefaultTopicMatcher(); + const issues: TutorContentQualityIssue[] = []; + for (const example of examples) { + const facts = extractor.extract(example.code); + for (const { topic } of matcher.match(topics, facts)) { + if (strictLearnPath(topic, facts, example.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY).masteryReached) continue; + issues.push({ + code: "unmasterable-example-activation", + message: `Example ${example.id} activates Topic ${topic.id}, but a learner who answers every planned question successfully cannot reach its mastery`, + exampleId: example.id, + topicId: topic.id, + }); + } + } + return issues; +} + function successfulTurn(brief: { readonly question: string; readonly questionId: string; diff --git a/ssot/ssot_function_definition_TutorQuality.md b/ssot/ssot_function_definition_TutorQuality.md index bc4275999..284ae5777 100644 --- a/ssot/ssot_function_definition_TutorQuality.md +++ b/ssot/ssot_function_definition_TutorQuality.md @@ -87,6 +87,17 @@ It fails on: - loader failures, invalid references, inconsistent hashes, or an invalid Course Content bundle. +Independent of the quality cases, the validator also checks the whole Example +catalog: for every Example in the manifest and every Topic that the production +fact extractor and matcher activate on its main sketch, a learner who answers +every planned question successfully (production planner, the Example's +effective LEARN strategy) MUST reach Topic mastery. This fails on Topics that +are unresolved from the first turn, for example through a prerequisite Concept +that the sketch cannot probe. It does not cover exhaustion that depends on weak +answers, or post-mastery (DEEPEN) capacity outside the quality cases. The +check keeps the gate aligned with every real Example, not only with the cases +an author chose. + Question–Indicator–Objective semantic coherence is outside this gate. Authors and later evaluation stages remain responsible for meaning, clarity, difficulty, feedback quality, and actual learning effect. diff --git a/tests/server/services/course-content/tutor-quality-directory-validator.test.ts b/tests/server/services/course-content/tutor-quality-directory-validator.test.ts index e9e0de814..b17cbc579 100644 --- a/tests/server/services/course-content/tutor-quality-directory-validator.test.ts +++ b/tests/server/services/course-content/tutor-quality-directory-validator.test.ts @@ -21,6 +21,18 @@ describe("Tutor Course Content directory validator", () => { await expect(validateTutorCourseContentDirectory(directory)).resolves.toEqual([]); }); + it("reports an Example without a quality case whose activated Topic cannot be mastered", async () => { + const serialOnly = { id: "serial-literal", source: 'void setup() { Serial.begin(9600); Serial.println("Hallo"); } void loop() {}\n' }; + const withPrerequisite = await courseContentDirectory(true, { topic: topicSource({ outputPrerequisite: true }), extraExample: serialOnly }); + const withoutPrerequisite = await courseContentDirectory(true, { topic: topicSource({ outputPrerequisite: false }), extraExample: serialOnly }); + + await expect(validateTutorCourseContentDirectory(withPrerequisite)).resolves.toEqual(expect.arrayContaining([ + expect.objectContaining({ code: "unmasterable-example-activation", exampleId: "serial-literal", topicId: "variables-and-serial" }), + ])); + const accepted = await validateTutorCourseContentDirectory(withoutPrerequisite); + expect(accepted.filter(({ code }) => code === "unmasterable-example-activation")).toEqual([]); + }); + it("fails closed when a declared Tutor hash is inconsistent", async () => { const directory = await courseContentDirectory(); await writeFile(path.join(directory, "tutor/topics/variables.yaml"), `${topicSource()}\n# changed\n`, "utf8"); @@ -57,15 +69,21 @@ describe("Tutor Course Content directory validator", () => { }); }); -async function courseContentDirectory(withCases = true): Promise { +async function courseContentDirectory( + withCases = true, + options: { readonly topic?: string; readonly extraExample?: { readonly id: string; readonly source: string } } = {}, +): Promise { const directory = await mkdtemp(path.join(tmpdir(), "unosim-tutor-quality-")); temporaryDirectories.push(directory); await mkdir(path.join(directory, "examples"), { recursive: true }); await mkdir(path.join(directory, "tutor/topics"), { recursive: true }); const serial = "int value = 3; void setup() { Serial.println(value); } void loop() {}\n"; const pwm = "const int pin = 9; void setup() {} void loop() { analogWrite(pin, 128); }\n"; - const topic = topicSource(); + const topic = options.topic ?? topicSource(); await writeFile(path.join(directory, "examples/serial.ino"), serial, "utf8"); + if (options.extraExample) { + await writeFile(path.join(directory, `examples/${options.extraExample.id}.ino`), options.extraExample.source, "utf8"); + } await writeFile(path.join(directory, "examples/pwm.ino"), pwm, "utf8"); await writeFile(path.join(directory, "tutor/topics/variables.yaml"), topic, "utf8"); await writeFile(path.join(directory, "manifest.json"), JSON.stringify({ @@ -73,6 +91,13 @@ async function courseContentDirectory(withCases = true): Promise { examples: [ { id: "serial", title: "Serial", category: "Test", files: [{ name: "serial.ino", path: "examples/serial.ino" }], main: "serial.ino" }, { id: "pwm", title: "PWM", category: "Test", files: [{ name: "pwm.ino", path: "examples/pwm.ino" }], main: "pwm.ino" }, + ...(options.extraExample ? [{ + id: options.extraExample.id, + title: "Extra", + category: "Test", + files: [{ name: `${options.extraExample.id}.ino`, path: `examples/${options.extraExample.id}.ino` }], + main: `${options.extraExample.id}.ino`, + }] : []), ], tutor: { manifest: "tutor/manifest.yaml" }, }), "utf8"); @@ -105,7 +130,10 @@ function digest(value: string): string { return createHash("sha256").update(value, "utf8").digest("hex"); } -function topicSource(): string { +// Default: one Concept, three Serial questions. With `outputPrerequisite` set: an int-only Concept +// "values" and a Serial Concept "output" that does (true) or does not (false) require it. +function topicSource(variant?: { readonly outputPrerequisite: boolean }): string { + if (variant) return prerequisiteTopicSource(variant.outputPrerequisite); return [ "schemaVersion: 1", "id: variables-and-serial", @@ -157,3 +185,65 @@ function topicSource(): string { "", ].join("\n"); } + +function prerequisiteTopicSource(outputPrerequisite: boolean): string { + const concept = (id: string, indicator: string, prerequisites: string) => [ + ` - id: ${id}`, + ` title: ${id}`, + " objective: Konzept erklären.", + ` prerequisites: ${prerequisites}`, + " difficulty:", + " entry: [1, 50]", + " transfer: [20, 80]", + " misconceptions: []", + " indicators:", + ` - id: ${indicator}`, + " description: Indikator.", + " mastery:", + " minimumSuccessfulProbes: 1", + " successRatingAtLeast: 3", + ` requiredIndicators: [${indicator}]`, + " minimumDistinctQuestionKinds: 1", + " recentWeakAnswersAllowed: 0", + ]; + const questionLines = (id: string, conceptId: string, indicator: string, kind: string, fact: string, value: string) => [ + ` - id: ${id}`, + ` concept: ${conceptId}`, + ` indicator: ${indicator}`, + ` kind: ${kind}`, + " difficulty: [1, 80]", + " requires:", + ` - fact: ${fact}`, + ` values: [${value}]`, + ` text: Was prüft ${id}?`, + ]; + return [ + "schemaVersion: 1", + "id: variables-and-serial", + "title: Variablen", + "locale: de-DE", + "activation:", + " any:", + " - fact: serial-call", + " values: [print]", + "concepts:", + ...concept("values", "value-use", "[]"), + ...concept("output", "output-use", outputPrerequisite ? "[values]" : "[]"), + "questions:", + ...questionLines("values-recall", "values", "value-use", "recall", "type-used", "int"), + ...questionLines("values-application", "values", "value-use", "application", "type-used", "int"), + ...questionLines("output-concept", "output", "output-use", "concept", "serial-call", "print"), + ...questionLines("output-transfer", "output", "output-use", "transfer", "serial-call", "print"), + ...questionLines("output-prediction", "output", "output-use", "prediction", "serial-call", "print"), + "scaffolds: []", + "progression:", + " entryConcepts: [values]", + " preferredOrder: [values, output]", + " onRating:", + " 1-2: remediate", + " 3: clarify-same-indicator", + " 4: probe-missing-indicator", + " 5: evaluate-mastery-and-advance", + "", + ].join("\n"); +} diff --git a/tests/server/services/course-content/tutor-quality-validator.test.ts b/tests/server/services/course-content/tutor-quality-validator.test.ts index 0bbf70f9c..b0f3807c2 100644 --- a/tests/server/services/course-content/tutor-quality-validator.test.ts +++ b/tests/server/services/course-content/tutor-quality-validator.test.ts @@ -1,6 +1,8 @@ import { describe, expect, it } from "vitest"; import type { CurriculumTopic } from "../../../../server/services/tutor/curriculum/curriculum-schema"; +import { BUILT_IN_TUTOR_STRATEGY } from "../../../../server/services/tutor/strategy/effective-tutor-strategy"; import { + validateExampleTopicActivations, validateTutorContentQuality, type ResolvedTutorQualityCase, } from "../../../../server/services/course-content/tutor-quality-validator"; @@ -170,3 +172,73 @@ describe("deterministic Tutor Course Content quality", () => { ])); }); }); + +// Authoring invariant over the whole Example catalog, independent of quality cases: every Topic +// that the production matcher activates on an Example must let a learner who answers every +// planned question successfully reach Topic mastery. A Topic that is unresolved from the first +// turn blocks the Topic-driven Tutor for the whole session (LearningQuestions SSOT 2.3). +describe("Example catalog Topic activations", () => { + const serialOnlySketch = 'void setup() { Serial.begin(9600); Serial.println("Hallo"); } void loop() {}'; + + // Concept "output" needs only Serial; its prerequisite "values" is probeable only with an int. + function prerequisiteTopic(withPrerequisite: boolean): CurriculumTopic { + const topic = validTopic(); + const values = topic.concepts[0]!; + topic.concepts = [ + { ...values, id: "values", prerequisites: [] }, + { + ...values, + id: "output", + title: "Ausgabe", + indicators: [{ id: "output-use", description: "Ausgabe erklären" }], + mastery: { ...values.mastery, requiredIndicators: ["output-use"] }, + prerequisites: withPrerequisite ? ["values"] : [], + }, + ]; + topic.questions = [ + { ...question("values-recall", "recall"), requires: [{ fact: "type-used" as const, values: ["int"] }] }, + { ...question("output-concept", "concept"), concept: "output", indicator: "output-use" }, + { ...question("output-transfer", "transfer"), concept: "output", indicator: "output-use" }, + ]; + topic.progression.entryConcepts = ["values"]; + topic.progression.preferredOrder = ["values", "output"]; + return topic; + } + + it("reports an Example that activates a Topic no learner can master, without any quality case", () => { + const issues = validateExampleTopicActivations([prerequisiteTopic(true)], [{ id: "serial-only", code: serialOnlySketch }]); + + expect(issues).toEqual([expect.objectContaining({ + code: "unmasterable-example-activation", + exampleId: "serial-only", + topicId: "variables-and-serial", + })]); + }); + + it("accepts the same Topic once the Concept no longer requires an unprobeable prerequisite", () => { + expect(validateExampleTopicActivations([prerequisiteTopic(false)], [{ id: "serial-only", code: serialOnlySketch }])).toEqual([]); + }); + + it("accepts the prerequisite where the sketch makes it probeable", () => { + expect(validateExampleTopicActivations([prerequisiteTopic(true)], [{ id: "int-serial", code: serialSketch }])).toEqual([]); + }); + + it("ignores Examples on which the Topic is not activated", () => { + expect(validateExampleTopicActivations([prerequisiteTopic(true)], [{ id: "pwm", code: pwmSketch }])).toEqual([]); + }); + + it("does not forbid later, learner-dependent exhaustion when a successful learner reaches mastery", () => { + // One question per Concept: weak answers would exhaust the Topic later, but mastery is reachable. + const topic = prerequisiteTopic(false); + topic.questions = topic.questions.filter(({ id }) => id !== "output-transfer"); + + expect(validateExampleTopicActivations([topic], [{ id: "serial-only", code: serialOnlySketch }])).toEqual([]); + }); + + it("uses the Example's effective LEARN strategy for the path", () => { + const relaxed = { ...BUILT_IN_TUTOR_STRATEGY, id: "relaxed-policy", repetition: "relaxed" as const }; + const issues = validateExampleTopicActivations([prerequisiteTopic(true)], [{ id: "serial-only", code: serialOnlySketch, learnStrategy: relaxed }]); + + expect(codes(issues)).toEqual(["unmasterable-example-activation"]); + }); +});