fix(tutor-quality): check every Topic an Example activates for a masterable LEARN path

scripts/validate-tutor-course-content.mjs

@@ -12,7 +12,7 @@ if (directory === undefined || directory.length === 0) {
     console.log(`Tutor Course Content quality passed: ${resolved}`);
   } else {
     for (const issue of issues) {
-      const scope = [issue.caseId, issue.topicId, issue.conceptId, issue.indicatorId].filter(Boolean).join("/");
+      const scope = [issue.caseId, issue.exampleId, issue.topicId, issue.conceptId, issue.indicatorId].filter(Boolean).join("/");
       const scopeSuffix = scope ? ` [${scope}]` : "";
       console.error(`${issue.code}${scopeSuffix}: ${issue.message}`);
     }

server/services/course-content/tutor-quality-directory-validator.ts

@@ -5,7 +5,9 @@ import { CourseContentLoader, type LoadedCourseContentSnapshot } from "./course-
 import { tutorQualityCasesSchema } from "./tutor-quality-schema";
 import { BUILT_IN_TUTOR_STRATEGY, type EffectiveTutorStrategy } from "../tutor/strategy/effective-tutor-strategy";
 import {
+  validateExampleTopicActivations,
   validateTutorContentQuality,
+  type CatalogExample,
   type ResolvedTutorQualityCase,
   type TutorContentQualityIssue,
 } from "./tutor-quality-validator";
@@ -35,7 +37,10 @@ export async function validateTutorCourseContentDirectory(directory: string): Pr
     }
     const resolution = resolveCases(parsed.data.cases, loaded);
     if (resolution.issues.length > 0) return resolution.issues;
-    return validateTutorContentQuality(loaded.tutor.topics, resolution.cases);
+    return [
+      ...validateTutorContentQuality(loaded.tutor.topics, resolution.cases),
+      ...validateExampleTopicActivations(loaded.tutor.topics, catalogExamples(loaded)),
+    ];
   } catch {
     return [directoryIssue("invalid-quality-cases", "Tutor quality case manifest could not be loaded")];
   }
@@ -83,6 +88,16 @@ function resolveCases(
   return { cases: resolved, issues };
 }
 
+// Every Example's main sketch, as the Tutor receives it, with its effective LEARN strategy.
+function catalogExamples(loaded: LoadedCourseContentSnapshot): CatalogExample[] {
+  return loaded.examples.flatMap((example) => {
+    const main = example.files.find(({ name }) => name === example.main);
+    return main
+      ? [{ id: example.id, code: main.content, learnStrategy: resolveCaseStrategy(loaded, example.tutorAnnotation?.strategy, "LEARN") }]
+      : [];
+  });
+}
+
 function resolveCaseStrategy(
   loaded: LoadedCourseContentSnapshot,
   exampleStrategyId: string | undefined,

server/services/course-content/tutor-quality-validator.ts

@@ -36,6 +36,7 @@ export type TutorContentQualityIssueCode =
   | "missing-deepening-question-kind"
   | "learn-content-exhausted"
   | "deepen-content-exhausted"
+  | "unmasterable-example-activation"
   | "invalid-course-content-bundle"
   | "invalid-quality-cases"
   | "quality-case-example-not-found";
@@ -44,6 +45,7 @@ export interface TutorContentQualityIssue {
   readonly code: TutorContentQualityIssueCode;
   readonly message: string;
   readonly caseId?: string;
+  readonly exampleId?: string;
   readonly topicId?: string;
   readonly conceptId?: string;
   readonly indicatorId?: string;
@@ -228,33 +230,7 @@ function validateExecutablePath(
   issues: TutorContentQualityIssue[],
 ): void {
   const planner = new DefaultLearningPlanner();
-  const history: TutorDialogTurn[] = [];
-  let masteryReached = false;
-  for (let step = 0; step <= topic.questions.length; step += 1) {
-    const observations = collectObservations(topic, history);
-    const classification = classifyTopic(
-      topic,
-      context.facts,
-      observations,
-      new Set(history.flatMap(({ questionId }) => questionId ? [questionId] : [])),
-      30,
-      context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY,
-    );
-    if (classification.status === "mastered") {
-      masteryReached = true;
-      break;
-    }
-    const plan = planner.start(
-      topic,
-      "0".repeat(40),
-      context.facts,
-      history,
-      30,
-      context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY,
-    );
-    if (!plan) break;
-    history.push(successfulTurn(plan.brief));
-  }
+  const { history, masteryReached } = strictLearnPath(topic, context.facts, context.qualityCase.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY);
   if (!masteryReached) {
     issues.push(issue("learn-content-exhausted", "Default strict progression cannot reach Topic mastery", context.qualityCase.id, topic.id));
     return;
@@ -288,6 +264,71 @@ function validateExecutablePath(
   issues.push(issue("deepen-content-exhausted", "Default strict progression cannot satisfy DEEPEN", context.qualityCase.id, topic.id));
 }
 
+/**
+ * The deterministic LEARN path of a learner who answers every planned question successfully,
+ * using the production planner and Topic classification. It needs no provider: if even this
+ * path cannot reach Topic mastery, no learner can.
+ */
+function strictLearnPath(
+  topic: CurriculumTopic,
+  facts: SketchFacts,
+  strategy: EffectiveTutorStrategy,
+): { readonly history: TutorDialogTurn[]; readonly masteryReached: boolean } {
+  const planner = new DefaultLearningPlanner();
+  const history: TutorDialogTurn[] = [];
+  for (let step = 0; step <= topic.questions.length; step += 1) {
+    const classification = classifyTopic(
+      topic,
+      facts,
+      collectObservations(topic, history),
+      new Set(history.flatMap(({ questionId }) => questionId ? [questionId] : [])),
+      30,
+      strategy,
+    );
+    if (classification.status === "mastered") return { history, masteryReached: true };
+    const plan = planner.start(topic, "0".repeat(40), facts, history, 30, strategy);
+    if (!plan) break;
+    history.push(successfulTurn(plan.brief));
+  }
+  return { history, masteryReached: false };
+}
+
+export interface CatalogExample {
+  readonly id: string;
+  readonly code: string;
+  readonly learnStrategy?: EffectiveTutorStrategy;
+}
+
+/**
+ * Catalog-wide authoring invariant, independent of quality cases: every Topic that the
+ * production matcher activates on an Example must be masterable on that Example's sketch by a
+ * learner who answers every planned question successfully. This finds Topics that are
+ * unresolved from the first turn (LearningQuestions SSOT 2.3), for example through a
+ * prerequisite Concept that the sketch cannot probe. Later exhaustion after weak answers and
+ * post-mastery (DEEPEN) capacity stay out of scope here.
+ */
+export function validateExampleTopicActivations(
+  topics: readonly CurriculumTopic[],
+  examples: readonly CatalogExample[],
+): TutorContentQualityIssue[] {
+  const extractor = new DefaultSketchFactExtractor();
+  const matcher = new DefaultTopicMatcher();
+  const issues: TutorContentQualityIssue[] = [];
+  for (const example of examples) {
+    const facts = extractor.extract(example.code);
+    for (const { topic } of matcher.match(topics, facts)) {
+      if (strictLearnPath(topic, facts, example.learnStrategy ?? BUILT_IN_TUTOR_STRATEGY).masteryReached) continue;
+      issues.push({
+        code: "unmasterable-example-activation",
+        message: `Example ${example.id} activates Topic ${topic.id}, but a learner who answers every planned question successfully cannot reach its mastery`,
+        exampleId: example.id,
+        topicId: topic.id,
+      });
+    }
+  }
+  return issues;
+}
+
 function successfulTurn(brief: {
   readonly question: string;
   readonly questionId: string;

ssot/ssot_function_definition_TutorQuality.md

@@ -87,6 +87,17 @@ It fails on:
 - loader failures, invalid references, inconsistent hashes, or an invalid
   Course Content bundle.
 
+Independent of the quality cases, the validator also checks the whole Example
+catalog: for every Example in the manifest and every Topic that the production
+fact extractor and matcher activate on its main sketch, a learner who answers
+every planned question successfully (production planner, the Example's
+effective LEARN strategy) MUST reach Topic mastery. This fails on Topics that
+are unresolved from the first turn, for example through a prerequisite Concept
+that the sketch cannot probe. It does not cover exhaustion that depends on weak
+answers, or post-mastery (DEEPEN) capacity outside the quality cases. The
+check keeps the gate aligned with every real Example, not only with the cases
+an author chose.
+
 Question–Indicator–Objective semantic coherence is outside this gate. Authors
 and later evaluation stages remain responsible for meaning, clarity, difficulty,
 feedback quality, and actual learning effect.

tests/server/services/course-content/tutor-quality-directory-validator.test.ts

@@ -21,6 +21,18 @@ describe("Tutor Course Content directory validator", () => {
     await expect(validateTutorCourseContentDirectory(directory)).resolves.toEqual([]);
   });
 
+  it("reports an Example without a quality case whose activated Topic cannot be mastered", async () => {
+    const serialOnly = { id: "serial-literal", source: 'void setup() { Serial.begin(9600); Serial.println("Hallo"); } void loop() {}\n' };
+    const withPrerequisite = await courseContentDirectory(true, { topic: topicSource({ outputPrerequisite: true }), extraExample: serialOnly });
+    const withoutPrerequisite = await courseContentDirectory(true, { topic: topicSource({ outputPrerequisite: false }), extraExample: serialOnly });
+
+    await expect(validateTutorCourseContentDirectory(withPrerequisite)).resolves.toEqual(expect.arrayContaining([
+      expect.objectContaining({ code: "unmasterable-example-activation", exampleId: "serial-literal", topicId: "variables-and-serial" }),
+    ]));
+    const accepted = await validateTutorCourseContentDirectory(withoutPrerequisite);
+    expect(accepted.filter(({ code }) => code === "unmasterable-example-activation")).toEqual([]);
+  });
+
   it("fails closed when a declared Tutor hash is inconsistent", async () => {
     const directory = await courseContentDirectory();
     await writeFile(path.join(directory, "tutor/topics/variables.yaml"), `${topicSource()}\n# changed\n`, "utf8");
@@ -57,22 +69,35 @@ describe("Tutor Course Content directory validator", () => {
   });
 });
 
-async function courseContentDirectory(withCases = true): Promise<string> {
+async function courseContentDirectory(
+  withCases = true,
+  options: { readonly topic?: string; readonly extraExample?: { readonly id: string; readonly source: string } } = {},
+): Promise<string> {
   const directory = await mkdtemp(path.join(tmpdir(), "unosim-tutor-quality-"));
   temporaryDirectories.push(directory);
   await mkdir(path.join(directory, "examples"), { recursive: true });
   await mkdir(path.join(directory, "tutor/topics"), { recursive: true });
   const serial = "int value = 3; void setup() { Serial.println(value); } void loop() {}\n";
   const pwm = "const int pin = 9; void setup() {} void loop() { analogWrite(pin, 128); }\n";
-  const topic = topicSource();
+  const topic = options.topic ?? topicSource();
   await writeFile(path.join(directory, "examples/serial.ino"), serial, "utf8");
+  if (options.extraExample) {
+    await writeFile(path.join(directory, `examples/${options.extraExample.id}.ino`), options.extraExample.source, "utf8");
+  }
   await writeFile(path.join(directory, "examples/pwm.ino"), pwm, "utf8");
   await writeFile(path.join(directory, "tutor/topics/variables.yaml"), topic, "utf8");
   await writeFile(path.join(directory, "manifest.json"), JSON.stringify({
     schemaVersion: 2,
     examples: [
       { id: "serial", title: "Serial", category: "Test", files: [{ name: "serial.ino", path: "examples/serial.ino" }], main: "serial.ino" },
       { id: "pwm", title: "PWM", category: "Test", files: [{ name: "pwm.ino", path: "examples/pwm.ino" }], main: "pwm.ino" },
+      ...(options.extraExample ? [{
+        id: options.extraExample.id,
+        title: "Extra",
+        category: "Test",
+        files: [{ name: `${options.extraExample.id}.ino`, path: `examples/${options.extraExample.id}.ino` }],
+        main: `${options.extraExample.id}.ino`,
+      }] : []),
     ],
     tutor: { manifest: "tutor/manifest.yaml" },
   }), "utf8");
@@ -105,7 +130,10 @@ function digest(value: string): string {
   return createHash("sha256").update(value, "utf8").digest("hex");
 }
 
-function topicSource(): string {
+// Default: one Concept, three Serial questions. With `outputPrerequisite` set: an int-only Concept
+// "values" and a Serial Concept "output" that does (true) or does not (false) require it.
+function topicSource(variant?: { readonly outputPrerequisite: boolean }): string {
+  if (variant) return prerequisiteTopicSource(variant.outputPrerequisite);
   return [
     "schemaVersion: 1",
     "id: variables-and-serial",
@@ -157,3 +185,65 @@ function topicSource(): string {
     "",
   ].join("\n");
 }
+
+function prerequisiteTopicSource(outputPrerequisite: boolean): string {
+  const concept = (id: string, indicator: string, prerequisites: string) => [
+    `  - id: ${id}`,
+    `    title: ${id}`,
+    "    objective: Konzept erklären.",
+    `    prerequisites: ${prerequisites}`,
+    "    difficulty:",
+    "      entry: [1, 50]",
+    "      transfer: [20, 80]",
+    "    misconceptions: []",
+    "    indicators:",
+    `      - id: ${indicator}`,
+    "        description: Indikator.",
+    "    mastery:",
+    "      minimumSuccessfulProbes: 1",
+    "      successRatingAtLeast: 3",
+    `      requiredIndicators: [${indicator}]`,
+    "      minimumDistinctQuestionKinds: 1",
+    "      recentWeakAnswersAllowed: 0",
+  ];
+  const questionLines = (id: string, conceptId: string, indicator: string, kind: string, fact: string, value: string) => [
+    `  - id: ${id}`,
+    `    concept: ${conceptId}`,
+    `    indicator: ${indicator}`,
+    `    kind: ${kind}`,
+    "    difficulty: [1, 80]",
+    "    requires:",
+    `      - fact: ${fact}`,
+    `        values: [${value}]`,
+    `    text: Was prüft ${id}?`,
+  ];
+  return [
+    "schemaVersion: 1",
+    "id: variables-and-serial",
+    "title: Variablen",
+    "locale: de-DE",
+    "activation:",
+    "  any:",
+    "    - fact: serial-call",
+    "      values: [print]",
+    "concepts:",
+    ...concept("values", "value-use", "[]"),
+    ...concept("output", "output-use", outputPrerequisite ? "[values]" : "[]"),
+    "questions:",
+    ...questionLines("values-recall", "values", "value-use", "recall", "type-used", "int"),
+    ...questionLines("values-application", "values", "value-use", "application", "type-used", "int"),
+    ...questionLines("output-concept", "output", "output-use", "concept", "serial-call", "print"),
+    ...questionLines("output-transfer", "output", "output-use", "transfer", "serial-call", "print"),
+    ...questionLines("output-prediction", "output", "output-use", "prediction", "serial-call", "print"),
+    "scaffolds: []",
+    "progression:",
+    "  entryConcepts: [values]",
+    "  preferredOrder: [values, output]",
+    "  onRating:",
+    "    1-2: remediate",
+    "    3: clarify-same-indicator",
+    "    4: probe-missing-indicator",
+    "    5: evaluate-mastery-and-advance",
+    "",
+  ].join("\n");
+}