diff --git a/README.md b/README.md index 1502ab6..1a1b5c5 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,58 @@ Run `weaver rules --root ./my-book` for a human report or add `--json` for a machine-readable result. Blocking findings fail `check` and prevent release; warnings are reported but do not fail the gate. +## Character voice + +The character registry at `world-bible/characters.json` is the author's explicit +list of speakers. `weaver characters seed` adds POV names recorded by import; +it never removes or overwrites an entry. Template books start with an empty +registry. A registry entry has an id, display name, optional aliases, and a POV +flag, for example: + +```json +{ "id": "brask", "name": "Brask", "aliases": ["the sergeant"], "pov": true } +``` + +Dialogue attribution is deliberately conservative. A quoted span is attributed +only by a nearby registry name or alias plus a supported speech verb, by an +action beat naming exactly one registry character outside the quote, or by +continuation from a previous paragraph that opened a quote. Names are whole +words, aliases beginning with `the` are case-insensitive, and pronoun-only tags +such as `she said` remain unattributed. Quotes whose speaker cannot be +determined are excluded from profiles and checks. This is a deterministic +heuristic, not a general-purpose dialogue parser. + +Build profiles from all attributed reader text with: + +```bash +npx weaver voices build --root ./my-book +npx weaver voices check --root ./my-book +npx weaver voices check --pending --root ./my-book +``` + +Profiles record line and word totals, sentence shape, contractions, profanity, +formal markers, sentence punctuation shares, distinctive words, and up to five +typical evidence lines. Evidence requires at least 8 attributed lines and 80 +words. A smaller profile reports `not enough of their lines yet (N lines, M +words)` and produces no finding. Optional author additions live in +`quality/voice.json` under `profanity` and `formal`. + +Voice profiles are derived from the last approved Git commit, not the working +tree. `weaver approve` rebuilds them after its approval commit. They are local +derived files under `world-bible/voices/` and are ignored by the book's +`.gitignore`, so editing them never creates pending scene changes. + +Voice checks use a combined per-feature z-distance threshold of 4.0, with zero +variance guarded, and call a line “sounds like” another sufficiently evidenced +character only when that character is at least 1.0 distance unit closer and a +distinctive word, hard-contrast feature, or unusually large distance margin +supports the comparison. Hard contrasts cover profanity against a zero-rate +profile with at least 150 words and the long formal/conversational contrast +described by the command output. Profiles are stamped with the approved +manuscript hash and Git commit; working-tree edits do not make them stale. +Voice findings are warnings for the author to judge; Weaver never decides that +a voice is right. + ## Claude Code hooks Run `weaver hooks install --root ./my-book` to merge Weaver's PreToolUse and @@ -140,6 +192,9 @@ Re-derive them in order before accepting state again. | `status` | Summarize manuscript hash, words, scenes, and state freshness | | `check` | Run project, critical-path, repetition, and state gates | | `rules` | Run deterministic style rules (`--scene`, `--json` supported) | +| `characters seed` | Add imported POV names to the character registry | +| `voices build` | Build deterministic character voice profiles | +| `voices check` | Show voice warnings (`--scene`, `--pending`, `--json` supported) | | `changes` | Show pending scene edits grouped by chapter (`--json` supported) | | `approve` | Save one chapter's pending edits as an approval | | `reject` | Save and restore one chapter's pending edits | diff --git a/contracts/characters.schema.json b/contracts/characters.schema.json new file mode 100644 index 0000000..e212fd0 --- /dev/null +++ b/contracts/characters.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/pickbitsai/weaver/main/contracts/characters.schema.json", + "title": "Weaver character registry", + "type": "object", + "additionalProperties": false, + "required": ["$schema", "version", "characters"], + "properties": { + "$schema": { "const": "https://pickbits.ai/schemas/weaver/characters-1.json" }, + "version": { "const": 1 }, + "characters": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "name", "aliases", "pov"], + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$" }, + "name": { "type": "string", "minLength": 1 }, + "aliases": { "type": "array", "items": { "type": "string" } }, + "pov": { "type": "boolean" } + } + } + } + } +} diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 71ec6db..f162d70 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -36,6 +36,15 @@ The repair path uses the same author approval boundary: imported or AI-edited scene changes appear in `weaver changes`, and the author approves or rejects one chapter at a time before continuing to state review and export. +For imported books, seed the registry from the imported POV headers, then build +profiles after the prose is in place: + +```bash +weaver characters seed --root ./my-book +weaver voices build --root ./my-book +weaver voices check --root ./my-book +``` + ## 1. Lock the architecture Before drafting, define the ending, scene-level beat map, reveal/reframe ledger, @@ -80,6 +89,17 @@ Run `weaver rules --root ./my-book` during drafting or repair to inspect style findings directly. A blocking style finding makes `check` fail; warnings remain visible without failing the gate. +Run `weaver voices check --scene --root ./my-book` while revising a +scene, or use `--pending` to inspect changed dialogue only. Voice findings show +the evidence lines that informed the warning and never block `check`, `approve`, +or release. Rebuild profiles after an accepted prose change with +`weaver voices build --root ./my-book`. + +Profiles are built from the approved manuscript at `HEAD`. Working-tree edits +are checked against those profiles without making them stale; after +`weaver approve` succeeds, Weaver rebuilds the ignored derived profiles from +the new approval commit. + Run three independent lenses: 1. Continuity — facts, timeline, geography, knowledge boundaries, and active diff --git a/engine/changes.mjs b/engine/changes.mjs index a713858..1173e1d 100644 --- a/engine/changes.mjs +++ b/engine/changes.mjs @@ -6,6 +6,7 @@ import { dirname, join, relative, resolve } from "node:path"; import { countWords, extractProse } from "./manuscript.mjs"; import { runRules } from "./rules.mjs"; import { GENERATOR } from "./identity.mjs"; +import { buildVoiceProfiles, runVoiceCheck } from "./voice.mjs"; const SCENE_PATH = /^chapter-(\d+)\/scene-(\d+)\/scene\.md$/u; @@ -108,11 +109,16 @@ function chapterFindings(root, project, pending) { const available = pending.scenes.filter((scene) => !scene.deleted).map((scene) => scene.id); if (!available.length) return new Map(); const result = runRules(root, project, { sceneIds: available }); + const voice = runVoiceCheck(root, project, { sceneIds: available }); const byScene = new Map(); for (const finding of result.findings) { if (!byScene.has(finding.scene_id)) byScene.set(finding.scene_id, []); byScene.get(finding.scene_id).push(finding); } + for (const finding of voice.findings) { + if (!byScene.has(finding.scene_id)) byScene.set(finding.scene_id, []); + byScene.get(finding.scene_id).push(finding); + } return byScene; } @@ -143,6 +149,7 @@ export function describePendingChanges(root, project) { words_removed: removed, blocking: allFindings.filter((finding) => finding.severity === "block").length, warnings: allFindings.filter((finding) => finding.severity === "warn").length, + voice_warnings: allFindings.filter((finding) => finding.type === "voice").length, scenes: changed }; }); @@ -220,7 +227,23 @@ export function approveChapter(root, project, chapter = null, note = "") { if (commit.status !== 0) throw new Error(`could not save the approval: ${(commit.stderr || commit.stdout || "").trim()}`); const sha = git(root, ["rev-parse", "--short", "HEAD"], { stdio: ["ignore", "pipe", "pipe"] }); if (sha.status !== 0) throw new Error("approval was saved, but its id could not be read from Git"); - return { chapter: selectedChapter, id: sha.stdout.trim(), paths, message: subject }; + const voices = buildVoiceProfiles(root, project); + return { chapter: selectedChapter, id: sha.stdout.trim(), paths, message: subject, voice_profiles: voices.profiles.length }; +} + +// An imported book is the author's existing work: it is the starting point, not a pending change. +// Commit it as the baseline so repairs are reviewed against the original, chapter by chapter. +export function recordImportBaseline(root, project, record, importPath) { + ensureIdentity(root, project); + const paths = [project.source_root, "project.json", relative(root, importPath).replaceAll("\\", "/")]; + const add = git(root, ["add", "--all", "--", ...paths], { stdio: ["ignore", "pipe", "pipe"] }); + if (add.status !== 0) throw new Error(`could not stage the imported book: ${(add.stderr || "").trim()}`); + const subject = `Import original manuscript: ${record.source?.file || "source"}`; + const commit = git(root, ["commit", "-m", subject, "-m", `Imported-with: ${GENERATOR}`, "--", ...paths], { stdio: ["ignore", "pipe", "pipe"] }); + if (commit.status !== 0) throw new Error(`could not save the imported book as the starting point: ${(commit.stderr || commit.stdout || "").trim()}`); + const sha = git(root, ["rev-parse", "--short", "HEAD"], { stdio: ["ignore", "pipe", "pipe"] }); + const voices = buildVoiceProfiles(root, project); + return { id: sha.stdout.trim(), voice_profiles: voices.profiles.length }; } export function rejectChapter(root, project, chapter = null) { diff --git a/engine/characters.mjs b/engine/characters.mjs new file mode 100644 index 0000000..f4d3b85 --- /dev/null +++ b/engine/characters.mjs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { SCHEMA_BASE } from "./identity.mjs"; + +const CHARACTERS_PATH = join("world-bible", "characters.json"); +const ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/u; + +function kebabCase(value) { + return String(value) + .normalize("NFKD") + .replace(/[\u0300-\u036f]/gu, "") + .replace(/[^\p{L}\p{N}]+/gu, "-") + .replace(/([a-z\d])([A-Z])/gu, "$1-$2") + .replace(/^-+|-+$/gu, "") + .toLocaleLowerCase(); +} + +function validateCharacter(character, index) { + const errors = []; + if (!character || typeof character !== "object" || Array.isArray(character)) return [`characters[${index}] must be an object`]; + if (typeof character.id !== "string" || !ID_PATTERN.test(character.id)) errors.push(`characters[${index}].id must be kebab-case`); + if (typeof character.name !== "string" || !character.name.trim()) errors.push(`characters[${index}].name must be a non-empty string`); + if (!Array.isArray(character.aliases) || character.aliases.some((alias) => typeof alias !== "string" || !alias.trim())) { + errors.push(`characters[${index}].aliases must be a string array`); + } + if (typeof character.pov !== "boolean") errors.push(`characters[${index}].pov must be boolean`); + return errors; +} + +export function validateCharacters(document) { + const errors = []; + if (!document || typeof document !== "object" || Array.isArray(document)) return ["characters file must contain an object"]; + if (document.version !== 1) errors.push("version must be 1"); + if (!Array.isArray(document.characters)) errors.push("characters must be an array"); + if (Array.isArray(document.characters)) { + const ids = new Set(); + for (const [index, character] of document.characters.entries()) { + errors.push(...validateCharacter(character, index)); + if (typeof character?.id === "string") { + if (ids.has(character.id)) errors.push(`characters[${index}].id must be unique`); + ids.add(character.id); + } + } + } + return errors; +} + +export function charactersPath(root) { + return join(root, CHARACTERS_PATH); +} + +export function loadCharacters(root) { + const path = charactersPath(root); + if (!existsSync(path)) return { $schema: `${SCHEMA_BASE}characters-1.json`, version: 1, characters: [] }; + let document; + try { + document = JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + throw new Error(`invalid ${CHARACTERS_PATH}: ${error.message}`); + } + const errors = validateCharacters(document); + if (errors.length) throw new Error(`invalid ${CHARACTERS_PATH}:\n- ${errors.join("\n- ")}`); + return document; +} + +export function seedCharacters(root, povNames) { + const path = charactersPath(root); + const current = loadCharacters(root); + const characters = [...current.characters]; + const existingIds = new Set(characters.map((character) => character.id)); + const names = [...new Set(povNames.map((name) => String(name).trim()).filter(Boolean))].sort((a, b) => a.localeCompare(b)); + const added = []; + for (const name of names) { + const id = kebabCase(name); + if (!id || existingIds.has(id)) continue; + const character = { id, name, aliases: [], pov: true }; + characters.push(character); + existingIds.add(id); + added.push(character); + } + characters.sort((left, right) => left.id.localeCompare(right.id)); + const next = { ...current, characters }; + if (added.length || !existsSync(path)) { + mkdirSync(join(root, "world-bible"), { recursive: true }); + writeFileSync(path, `${JSON.stringify(next, null, 2)}\n`); + } + return { path, added, characters: next.characters }; +} + +export function characterKebabCase(value) { + return kebabCase(value); +} diff --git a/engine/hooks.mjs b/engine/hooks.mjs index e698186..394ced8 100644 --- a/engine/hooks.mjs +++ b/engine/hooks.mjs @@ -6,6 +6,7 @@ import { countWords, loadScenes } from "./manuscript.mjs"; import { readProject } from "./project.mjs"; import { loadRules, runRules } from "./rules.mjs"; import { otherPendingChapters } from "./changes.mjs"; +import { runVoiceCheck } from "./voice.mjs"; const MATCHER = "Edit|Write|MultiEdit"; const PRE_COMMAND = "npx --no-install weaver hook pre"; @@ -101,12 +102,16 @@ function postHook(payload) { }; } const result = runRules(scene.cwd, scene.project, { sceneIds: [scene.sceneId] }); + const voices = runVoiceCheck(scene.cwd, scene.project, { sceneIds: [scene.sceneId] }); if (result.blocking) { const blocking = result.findings.filter((finding) => finding.severity === "block").map(findingText).join("\n"); return { decision: "block", reason: `${blocking}\nfix these before continuing` }; } - if (result.warnings) { - const warnings = result.findings.filter((finding) => finding.severity === "warn").map(findingText).join("\n"); + if (result.warnings || voices.warnings) { + const warnings = [ + ...result.findings.filter((finding) => finding.severity === "warn").map(findingText), + ...voices.findings.map((finding) => `voice warning (paragraph ${finding.paragraph}, ${finding.speaker}): ${finding.reason}${finding.reasons?.length ? `\nReasons:\n${finding.reasons.map((reason) => ` - ${reason}`).join("\n")}` : ""}\nLine: ${finding.line}\nTypical lines:\n${finding.evidence.map((line) => ` - ${line.text} [${line.scene_id}, paragraph ${line.paragraph}]`).join("\n")}${finding.maybe_sounds_like ? `\nMaybe sounds like ${finding.maybe_sounds_like.character}:\n${finding.maybe_sounds_like.evidence.map((line) => ` - ${line.text} [${line.scene_id}, paragraph ${line.paragraph}]`).join("\n")}` : ""}`) + ].join("\n"); return { hookSpecificOutput: { hookEventName: "PostToolUse", additionalContext: warnings } }; } return null; diff --git a/engine/init.mjs b/engine/init.mjs index fba0930..e82a275 100644 --- a/engine/init.mjs +++ b/engine/init.mjs @@ -34,7 +34,7 @@ export function initializeProject(target, { projectId = "untitled-book", title = writeFileSync(projectPath, `${JSON.stringify(project, null, 2)}\n`); const gitignorePath = join(destination, ".gitignore"); const existingGitignore = existsSync(gitignorePath) ? readFileSync(gitignorePath, "utf8") : ""; - const requiredIgnoreLines = ["node_modules/", ".weaver/tmp/", ".weaver/rejected/"]; + const requiredIgnoreLines = ["node_modules/", ".weaver/tmp/", ".weaver/rejected/", "world-bible/voices/"]; const missing = requiredIgnoreLines.filter((line) => !existingGitignore.split(/\r?\n/u).some((entry) => entry.trim() === line)); if (missing.length) { const prefix = existingGitignore && !existingGitignore.endsWith("\n") ? "\n" : ""; diff --git a/engine/release.mjs b/engine/release.mjs index 3958d12..9bf82c3 100644 --- a/engine/release.mjs +++ b/engine/release.mjs @@ -14,6 +14,7 @@ import { import { narrativeStateStatus, readStateManifest, statePath } from "./state.mjs"; import { GENERATOR, HOMEPAGE, LICENSE_ID, PRODUCT_NAME, SCHEMA_BASE, VERSION } from "./identity.mjs"; import { runRules } from "./rules.mjs"; +import { runVoiceCheck } from "./voice.mjs"; const RELEASE_ID_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; const escapeHtml = (value) => value @@ -30,6 +31,7 @@ export function runQualityChecks(root, project) { const unapprovedDuplicates = filterIntentionalEchoes(root, duplicates); const state = narrativeStateStatus(root, project); const rules = runRules(root, project); + const voices = runVoiceCheck(root, project); const issues = []; if (!criticalPath.ok) issues.push(`critical-path gate failed (${criticalPath.missing.length} missing)`); if (unapprovedDuplicates.length) issues.push(`repetition gate failed (${unapprovedDuplicates.length} unapproved duplicate(s))`); @@ -37,7 +39,7 @@ export function runQualityChecks(root, project) { issues.push(`narrative state is stale from scene ${state.first_stale}`); } if (rules.blocking) issues.push(`style rules: ${rules.blocking} blocking finding(s)`); - return { ok: issues.length === 0, issues, scenes, criticalPath, duplicates, unapprovedDuplicates, state, rules }; + return { ok: issues.length === 0, issues, scenes, criticalPath, duplicates, unapprovedDuplicates, state, rules, voices }; } function assembleHtml(project, scenes, releaseId) { diff --git a/engine/voice.mjs b/engine/voice.mjs new file mode 100644 index 0000000..2003dba --- /dev/null +++ b/engine/voice.mjs @@ -0,0 +1,725 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { join } from "node:path"; +import { countWords, extractProse, loadScenes, manuscriptHash, sha256 } from "./manuscript.mjs"; +import { readerText } from "./prose.mjs"; +import { loadCharacters } from "./characters.mjs"; + +const VOICES_ROOT = join("world-bible", "voices"); +const PROFILE_VERSION = 2; +export const EVIDENCE_MINIMUM = { lines: 8, words: 80 }; +export const VOICE_DISTANCE_THRESHOLD = 4; +export const SOUNDS_LIKE_MARGIN = 1; + +const BUILTIN_PROFANITY = ["damn", "damned", "hell", "bloody", "shit", "fuck", "bastard", "arse", "piss", "crap", "bugger"]; +const BUILTIN_FORMAL = ["shall", "indeed", "perhaps", "whom", "pray", "madam", "sir", "one must", "I daresay"]; +const STOPWORDS = new Set(`a an and are as at be been but by can could did do does for from had has have he her hers him his how i if in is it its just me might more most my no not of on one or our ours she should so some than that the their theirs them then there these they this to too us was we were what when which who will with would you your yours`.split(" ")); +const VERBS = new Set([ + "said", "asked", "replied", "answered", "shouted", "yelled", "whispered", "muttered", "snapped", "called", + "growled", "murmured", "added", "continued", "demanded", "cried", "sneered", "laughed", "sighed", "told", + "says", "asks", "replies", "answers", "shouts", "yells", "whispers", "mutters", "snaps", "calls", "growls", + "murmurs", "adds", "continues", "demands", "cries", "sneers", "laughs", "sighs", "tells" +]); +const FEATURE_KEYS = [ + "words_per_sentence", + "contractions_per_100_words", + "profanity_per_100_words", + "exclamation_share", + "question_share", + "formal_markers_per_100_words" +]; + +function voiceSettings(root) { + const path = join(root, "quality", "voice.json"); + if (!existsSync(path)) return { profanity: [], formal: [] }; + let value; + try { + value = JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + throw new Error(`invalid quality/voice.json: ${error.message}`); + } + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid quality/voice.json: expected an object"); + for (const key of ["profanity", "formal"]) { + if (value[key] !== undefined && (!Array.isArray(value[key]) || value[key].some((item) => typeof item !== "string" || !item.trim()))) { + throw new Error(`invalid quality/voice.json: ${key} must be a string array`); + } + } + return { profanity: value.profanity || [], formal: value.formal || [] }; +} + +function git(root, args) { + return spawnSync("git", ["-C", root, ...args], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }); +} + +function approvedManuscript(root, project) { + const head = git(root, ["rev-parse", "--verify", "HEAD"]); + if (head.status !== 0) { + const scenes = loadScenes(root, project); + return { scenes, head_commit: null, manuscript_sha256: manuscriptHash(scenes) }; + } + const headCommit = head.stdout.trim(); + const sourceRoot = project.source_root.replaceAll("\\", "/").replace(/\/$/u, ""); + const tree = git(root, ["ls-tree", "-r", "--name-only", headCommit, "--", sourceRoot]); + if (tree.status !== 0) throw new Error("could not read the approved manuscript from Git"); + const paths = tree.stdout.split(/\r?\n/u).filter((path) => new RegExp(`^${escapeRegex(sourceRoot)}/chapter-\\d+/scene-\\d+/scene\\.md$`, "u").test(path)); + const scenes = paths.map((relativePath) => { + const match = new RegExp(`^${escapeRegex(sourceRoot)}/chapter-(\\d+)/scene-(\\d+)/scene\\.md$`, "u").exec(relativePath); + const raw = git(root, ["show", `${headCommit}:${relativePath}`]); + if (raw.status !== 0) throw new Error(`could not read approved scene ${relativePath} from Git`); + const prose = extractProse(raw.stdout, relativePath); + return { + chapter: Number(match[1]), + scene: Number(match[2]), + id: `${Number(match[1])}-${Number(match[2])}`, + path: join(root, relativePath), + relativePath, + prose, + words: countWords(prose), + hash: sha256(prose) + }; + }); + return { scenes, head_commit: headCommit, manuscript_sha256: manuscriptHash(scenes) }; +} + +function escapeRegex(value) { + return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); +} + +function markerRegex(marker) { + const escaped = escapeRegex(marker.trim()).replaceAll("'", "['’]"); + return new RegExp(`(? sum + [...text.matchAll(markerRegex(marker))].length, 0); +} + +function profanityCount(text, words) { + return words.reduce((sum, word) => { + const normalized = word.trim().toLocaleLowerCase(); + if (!normalized) return sum; + const pattern = normalized.includes(" ") + ? markerRegex(normalized) + : new RegExp(`(? quoteClosing(character, opening[1])); + const end = close < 0 ? text.length : start + close; + spans.push({ text: text.slice(start, end), start, end, continuation: true, speaker: continuation.speaker }); + if (close < 0) { + continuation = { speaker: continuation.speaker }; + output.push(...spans.map((span) => ({ ...span, paragraph: paragraph.number }))); + continue; + } + continuation = null; + cursor = end + 1; + } + } + while (cursor < text.length) { + const openingIndex = [...text.slice(cursor)].findIndex((character) => quoteOpening(character)); + if (openingIndex < 0) break; + const start = cursor + openingIndex + 1; + const opening = text[cursor + openingIndex]; + const close = [...text.slice(start)].findIndex((character) => quoteClosing(character, opening)); + const end = close < 0 ? text.length : start + close; + spans.push({ text: text.slice(start, end), start: cursor + openingIndex, end, continuation: false, speaker: null }); + if (close < 0) { + continuation = { speaker: null }; + break; + } + cursor = end + 1; + } + output.push(...spans.map((span) => ({ ...span, paragraph: paragraph.number }))); + if (spans.some((span) => span.continuation && span.end === text.length)) continue; + if (spans.length && spans.at(-1).end === text.length && continuation) continue; + } + return output; +} + +function termPattern(term, insensitive = false) { + return `(? ({ value: alias, insensitive: /^the\s/iu.test(alias) }))]; + for (const term of terms) { + const regex = new RegExp(termPattern(term.value, term.insensitive), term.insensitive ? "giu" : "gu"); + for (const match of text.matchAll(regex)) found.push({ id: character.id, index: match.index, length: match[0].length }); + } + } + return found.sort((a, b) => a.index - b.index || b.length - a.length); +} + +function outsideText(text, spans) { + const mask = [...text]; + for (const span of spans) for (let index = span.start; index <= span.end; index += 1) mask[index] = " "; + return mask.join(""); +} + +function speakerTag(text, span, characters) { + const before = text.slice(Math.max(0, span.start - 120), span.start); + const after = text.slice(span.end + 1, Math.min(text.length, span.end + 121)); + for (const character of characters) { + const terms = [{ value: character.name, insensitive: false }, ...(character.aliases || []).map((alias) => ({ value: alias, insensitive: /^the\s/iu.test(alias) }))]; + for (const term of terms) { + const name = termPattern(term.value, term.insensitive); + const flags = term.insensitive ? "iu" : "u"; + const beforePattern = new RegExp(`(?:${name}\\s+(${[...VERBS].join("|")})|(${[...VERBS].join("|")})\\s+${name})\\s*[,;:—-]?\\s*$`, flags); + const afterPattern = new RegExp(`^\\s*[,;:—-]?\\s*(?:(${[...VERBS].join("|")})\\s+${name}|${name}\\s+(${[...VERBS].join("|")}))\\s*[.!?]*\\s*$`, flags); + if (beforePattern.test(before) || afterPattern.test(after)) return character.id; + } + } + return null; +} + +function attributeSpans(scene, characters) { + const paragraphs = scene.prose.split(/\n\s*\n/u).filter((paragraph) => paragraph.trim()).map((paragraph, index) => ({ + number: index + 1, + text: readerText(paragraph) + })); + const spans = quoteSpans(paragraphs); + const byParagraph = new Map(); + for (const span of spans) { + if (!byParagraph.has(span.paragraph)) byParagraph.set(span.paragraph, []); + byParagraph.get(span.paragraph).push(span); + } + const lines = []; + let pendingSpeaker = null; + for (const paragraph of paragraphs) { + const paragraphSpans = byParagraph.get(paragraph.number) || []; + const outside = outsideText(paragraph.text, paragraphSpans); + const named = new Set(registryMatches(outside, characters).map((match) => match.id)); + for (const span of paragraphSpans) { + let speaker = span.continuation ? pendingSpeaker : speakerTag(paragraph.text, span, characters); + if (!span.continuation && !speaker && named.size === 1) speaker = [...named][0]; + lines.push({ scene_id: scene.id, paragraph: paragraph.number, text: span.text, speaker, method: span.continuation ? "continuation" : speaker ? speakerTag(paragraph.text, span, characters) ? "tag" : "beat" : "unattributed" }); + if (span.end === paragraph.text.length && !span.continuation) pendingSpeaker = speaker; + else if (span.continuation && span.end === paragraph.text.length) pendingSpeaker = speaker; + else pendingSpeaker = null; + } + if (!paragraphSpans.length) pendingSpeaker = null; + } + return lines; +} + +function mean(values) { + return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0; +} + +function standardDeviation(values, average = mean(values)) { + if (values.length < 2) return 0; + return Math.sqrt(mean(values.map((value) => (value - average) ** 2))); +} + +const PROFILE_TOTAL_KEYS = ["lines", "words", "sentences", "contractions", "profanity", "formal_markers", "questions", "exclamations"]; + +function normalizedLineText(text) { + return String(text).trim().replace(/\s+/gu, " "); +} + +function lineRecordKey(record) { + return `${record.scene_id}\u0000${record.paragraph}\u0000${record.text_sha256}`; +} + +function lineKey(line) { + return `${line.scene_id}\u0000${line.paragraph}\u0000${sha256(normalizedLineText(line.text))}`; +} + +function lineRecord(line) { + const features = line.features; + return { + scene_id: line.scene_id, + paragraph: line.paragraph, + text_sha256: sha256(normalizedLineText(line.text)), + features: { + words: features.words, + sentences: features.sentences, + contractions: features.contractions, + profanity: features.profanity, + formal_markers: features.formal_markers, + questions: features.questions, + exclamations: features.exclamations, + words_per_sentence: features.words_per_sentence, + contractions_per_100_words: features.contractions_per_100_words, + profanity_per_100_words: features.profanity_per_100_words, + exclamation_share: features.exclamation_share, + question_share: features.question_share, + formal_markers_per_100_words: features.formal_markers_per_100_words + } + }; +} + +function totalsForLines(lines) { + return { + lines: lines.length, + words: lines.reduce((sum, line) => sum + line.features.words, 0), + sentences: lines.reduce((sum, line) => sum + line.features.sentences, 0), + contractions: lines.reduce((sum, line) => sum + line.features.contractions, 0), + profanity: lines.reduce((sum, line) => sum + line.features.profanity, 0), + formal_markers: lines.reduce((sum, line) => sum + line.features.formal_markers, 0), + questions: lines.reduce((sum, line) => sum + line.features.questions, 0), + exclamations: lines.reduce((sum, line) => sum + line.features.exclamations, 0) + }; +} + +function aggregateFeatures(totals) { + return { + mean_words_per_sentence: totals.sentences ? totals.words / totals.sentences : 0, + contractions_per_100_words: totals.words ? totals.contractions * 100 / totals.words : 0, + profanity_per_100_words: totals.words ? totals.profanity * 100 / totals.words : 0, + exclamation_share: totals.sentences ? totals.exclamations / totals.sentences : 0, + question_share: totals.sentences ? totals.questions / totals.sentences : 0, + formal_markers_per_100_words: totals.words ? totals.formal_markers * 100 / totals.words : 0 + }; +} + +function sortedLineRecords(lines) { + return lines.map(lineRecord).sort((left, right) => lineRecordKey(left).localeCompare(lineRecordKey(right))); +} + +function profileFor(character, lines, settings, hash, headCommit) { + const own = lines.filter((line) => line.speaker === character.id).map((line) => ({ ...line, features: lineFeatures(line.text, settings) })); + const values = Object.fromEntries(FEATURE_KEYS.map((key) => [key, own.map((line) => line.features[key])])); + const totals = totalsForLines(own); + const features = aggregateFeatures(totals); + const distribution = Object.fromEntries(FEATURE_KEYS.map((key) => { + const average = mean(values[key]); + return [key, { mean: average, standard_deviation: standardDeviation(values[key], average) }]; + })); + const typical = own.map((line) => ({ + ...line, + distance: Math.sqrt(FEATURE_KEYS.reduce((sum, key) => { + const sd = distribution[key].standard_deviation; + return sum + (sd ? ((line.features[key] - distribution[key].mean) / sd) ** 2 : 0); + }, 0)) + })).sort((left, right) => left.distance - right.distance || lineKey(left).localeCompare(lineKey(right))); + const wordCounts = new Map(); + const otherCounts = new Map(); + for (const line of own) { + for (const word of (line.text.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) || [])) { + if (STOPWORDS.has(word)) continue; + wordCounts.set(word, (wordCounts.get(word) || 0) + 1); + } + } + for (const line of lines.filter((candidate) => candidate.speaker && candidate.speaker !== character.id)) { + for (const word of (line.text.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) || [])) { + if (!STOPWORDS.has(word)) otherCounts.set(word, (otherCounts.get(word) || 0) + 1); + } + } + const ownTotal = [...wordCounts.values()].reduce((sum, value) => sum + value, 0); + const otherTotal = [...otherCounts.values()].reduce((sum, value) => sum + value, 0); + const distinctiveWords = [...wordCounts.entries()] + .filter(([, count]) => count >= 2) + .map(([word, count]) => ({ word, score: Math.log(((count + 0.5) / (ownTotal + 1)) / (((otherCounts.get(word) || 0) + 0.5) / (otherTotal + 1))) })) + .sort((left, right) => right.score - left.score || left.word.localeCompare(right.word)) + .slice(0, 15) + .map((entry) => entry.word); + return { + version: PROFILE_VERSION, + character_id: character.id, + name: character.name, + manuscript_sha256: hash, + approved_manuscript_sha256: hash, + head_commit: headCommit, + approved_head_commit: headCommit, + lines: totals.lines, + words: totals.words, + enough_evidence: totals.lines >= EVIDENCE_MINIMUM.lines && totals.words >= EVIDENCE_MINIMUM.words, + totals, + line_records: sortedLineRecords(own), + features, + distribution, + distinctive_words: distinctiveWords, + evidence: typical.slice(0, 5).map((line) => ({ text: line.text, scene_id: line.scene_id, paragraph: line.paragraph })) + }; +} + +export function buildVoiceProfiles(root, project) { + const characters = loadCharacters(root).characters; + const approved = approvedManuscript(root, project); + const scenes = approved.scenes; + const hash = approved.manuscript_sha256; + const settings = voiceSettings(root); + const lines = scenes.flatMap((scene) => attributeSpans(scene, characters)); + const directory = join(root, VOICES_ROOT); + mkdirSync(directory, { recursive: true }); + const profiles = []; + for (const character of characters) { + const profile = profileFor(character, lines, settings, hash, approved.head_commit); + writeFileSync(join(directory, `${character.id}.json`), `${JSON.stringify(profile, null, 2)}\n`); + profiles.push(profile); + } + return { manuscript_sha256: hash, profiles }; +} + +function loadProfiles(root, characters) { + return characters.flatMap((character) => { + const path = join(root, VOICES_ROOT, `${character.id}.json`); + if (!existsSync(path)) return []; + try { + return [{ character, profile: JSON.parse(readFileSync(path, "utf8")) }]; + } catch (error) { + throw new Error(`invalid world-bible/voices/${character.id}.json: ${error.message}`); + } + }); +} + +function recordFeature(record, key) { + return record.features?.[key] ?? 0; +} + +function recordTotals(record) { + return { + lines: 1, + words: recordFeature(record, "words"), + sentences: recordFeature(record, "sentences"), + contractions: recordFeature(record, "contractions"), + profanity: recordFeature(record, "profanity"), + formal_markers: recordFeature(record, "formal_markers"), + questions: recordFeature(record, "questions"), + exclamations: recordFeature(record, "exclamations") + }; +} + +function profileTotals(profile, records) { + return Object.fromEntries(PROFILE_TOTAL_KEYS.map((key) => [ + key, + Number.isFinite(profile.totals?.[key]) + ? profile.totals[key] + : key === "lines" + ? (profile.lines ?? records.length) + : key === "words" + ? (profile.words ?? 0) + : 0 + ])); +} + +function comparisonProfile(profile, line) { + if (!Array.isArray(profile.line_records)) return { profile, matched: false, legacy: true }; + const target = lineKey(line); + const matches = profile.line_records.filter((record) => lineRecordKey(record) === target); + if (!matches.length) return { profile, matched: false, legacy: false }; + + // Exclude the checked line and every other line by this speaker in its paragraph. + const excluded = profile.line_records.filter((record) => record.scene_id === line.scene_id && record.paragraph === line.paragraph); + const excludedSet = new Set(excluded); + const remaining = profile.line_records.filter((record) => !excludedSet.has(record)); + const totals = profileTotals(profile, profile.line_records); + for (const record of excluded) { + const contribution = recordTotals(record); + for (const key of PROFILE_TOTAL_KEYS) totals[key] = Math.max(0, totals[key] - contribution[key]); + } + const features = aggregateFeatures(totals); + const distribution = Object.fromEntries(FEATURE_KEYS.map((key) => { + const values = remaining.map((record) => recordFeature(record, key)); + const average = mean(values); + return [key, { mean: average, standard_deviation: standardDeviation(values, average) }]; + })); + return { + matched: true, + legacy: false, + profile: { + ...profile, + lines: totals.lines, + words: totals.words, + enough_evidence: totals.lines >= EVIDENCE_MINIMUM.lines && totals.words >= EVIDENCE_MINIMUM.words, + totals, + features, + distribution + } + }; +} + +function evidenceWithoutLine(profile, line) { + const target = lineKey(line); + return (profile.evidence || []).filter((evidence) => { + if (typeof evidence.text !== "string" || evidence.scene_id === undefined || evidence.paragraph === undefined) return true; + return lineKey(evidence) !== target; + }); +} + +function profileDistance(features, profile) { + return Math.sqrt(FEATURE_KEYS.reduce((sum, key) => { + const expected = profile.distribution?.[key]?.mean ?? profile.features?.[key] ?? 0; + const scale = profile.distribution?.[key]?.standard_deviation || 1; + return sum + ((features[key] - expected) / scale) ** 2; + }, 0)); +} + +function sharedDistinctiveWords(text, profile) { + const words = new Set((text.toLocaleLowerCase().match(/[\p{L}\p{N}]+/gu) || []).filter((word) => !STOPWORDS.has(word))); + return (profile.distinctive_words || []).filter((word) => words.has(word)); +} + +function hardContrastMatches(features, profile) { + return (features.profanity > 0 && profile.features.profanity_per_100_words >= 1) + || (features.formal_markers > 0 && profile.features.formal_markers_per_100_words >= 1) + || (features.contractions > 0 && profile.features.contractions_per_100_words >= 1); +} + +function oneDecimal(value) { + return Number(value.toFixed(1)); +} + +function formalMarkerNames(line, settings) { + const found = new Map(); + for (const marker of [...BUILTIN_FORMAL, ...settings.formal]) { + if (markerRegex(marker).test(line)) found.set(marker.trim().toLocaleLowerCase(), marker.trim()); + } + return [...found.values()]; +} + +function voiceSignals(line, features, profile, settings) { + const profileFeatures = profile.features || {}; + const contractionRate = profileFeatures.contractions_per_100_words || 0; + const profanityRate = profileFeatures.profanity_per_100_words || 0; + const formalRate = profileFeatures.formal_markers_per_100_words || 0; + const sentenceRate = profileFeatures.mean_words_per_sentence || 0; + const formalWords = formalMarkerNames(line, settings); + const rateEligible = features.words >= 12; + const hard = { + profanity: features.profanity > 0 + && profile.words >= 150 + && profanityRate === 0, + formal: profile.words >= 150 + && formalRate <= 0.2 + && (formalWords.length >= 2 || (formalWords.length >= 1 && features.words >= 15)) + }; + const signals = { + long: features.words >= 15 + && sentenceRate > 0 + && features.words_per_sentence >= sentenceRate * 3, + short: rateEligible + && sentenceRate >= 12 + && features.sentences >= 2 + && features.words_per_sentence <= sentenceRate * 0.25, + contraction_absence: rateEligible + && features.words >= 15 + && features.contractions === 0 + && contractionRate >= 4, + contraction_presence: rateEligible + && features.contractions > 0 + && contractionRate < 1, + profanity_presence: rateEligible + && features.profanity > 0 + && profanityRate === 0, + formal_presence: rateEligible + && features.formal_markers > 0 + && formalRate < 1 + }; + const questions = [...line.matchAll(/\?/gu)].length; + const exclamations = [...line.matchAll(/!/gu)].length; + return { + hard, + signals, + formalWords, + questions, + exclamations, + supporting: { + question: rateEligible && questions > 0 && questions / Math.max(features.sentences, 1) > (profileFeatures.question_share || 0), + exclamation: rateEligible && exclamations > 0 && exclamations / Math.max(features.sentences, 1) > (profileFeatures.exclamation_share || 0) + } + }; +} + +function reasonFor(line, features, profile, settings, analysis) { + const reasons = []; + const candidates = []; + const expected = (key) => profile.distribution?.[key]?.mean ?? profile.features[key]; + const deviation = (key) => Math.abs(features[key] - expected(key)) / (profile.distribution?.[key]?.standard_deviation || 1); + const { signals, hard, formalWords, questions, exclamations, supporting } = analysis; + const formalList = formalWords.length ? formalWords.map((word) => `'${word}'`).join(", ") : "formal markers"; + const add = (key, text, priority = 0) => candidates.push({ priority, strength: deviation(key), text }); + const contractionRate = oneDecimal(profile.features.contractions_per_100_words); + const profanityRate = oneDecimal(profile.features.profanity_per_100_words); + const formalRate = oneDecimal(profile.features.formal_markers_per_100_words); + const sentenceRate = oneDecimal(profile.features.mean_words_per_sentence); + if (hard.profanity) { + add("profanity_per_100_words", `This line has ${features.profanity} profanity marker${features.profanity === 1 ? "" : "s"}; ${profile.name} has no profanity markers across ${profile.words} words of evidence.`, 100); + } + if (hard.formal) { + add("formal_markers_per_100_words", `This line uses ${formalList}; ${profile.name} uses about ${formalRate} formal markers per 100 words across ${profile.words} words of evidence.`, 100); + } + if (signals.long) { + add("words_per_sentence", `This line averages ${oneDecimal(features.words_per_sentence)} words per sentence; ${profile.name}'s lines average ${sentenceRate}, so this is at least three times longer.`, 50); + } else if (signals.short) { + add("words_per_sentence", `This line averages ${oneDecimal(features.words_per_sentence)} words per sentence; ${profile.name}'s lines average ${sentenceRate}, so this is at most one quarter as long.`, 50); + } + if (signals.contraction_absence) { + add("contractions_per_100_words", `This line has no contractions in ${features.words} words; ${profile.name} uses about ${contractionRate} contractions per 100 words.`, 40); + } else if (signals.contraction_presence) { + add("contractions_per_100_words", `This line has ${features.contractions} contraction${features.contractions === 1 ? "" : "s"} in ${features.words} words; ${profile.name} uses about ${contractionRate} contractions per 100 words.`, 40); + } + if (signals.profanity_presence && !hard.profanity) { + add("profanity_per_100_words", `This line has ${features.profanity} profanity marker${features.profanity === 1 ? "" : "s"}; ${profile.name} uses about ${profanityRate} profanity markers per 100 words.`, 30); + } + if (signals.formal_presence && !hard.formal) { + add("formal_markers_per_100_words", `This line uses ${formalList} (${features.formal_markers} formal marker${features.formal_markers === 1 ? "" : "s"}); ${profile.name} uses about ${formalRate} formal markers per 100 words.`, 30); + } + if (candidates.length && supporting.exclamation) { + add("exclamation_share", `This line has ${exclamations} exclamation${exclamations === 1 ? "" : "s"}; ${profile.name} averages fewer exclamations per sentence.`, 10); + } + if (candidates.length && supporting.question) { + add("question_share", `This line has ${questions} question${questions === 1 ? "" : "s"}; ${profile.name} averages fewer questions per sentence.`, 10); + } + candidates.sort((left, right) => right.priority - left.priority || right.strength - left.strength || left.text.localeCompare(right.text)); + reasons.push(...candidates.slice(0, 3).map((candidate) => candidate.text)); + if (!reasons.length) { + reasons.push(`This line is out of voice for ${profile.name}'s usual sentence length.`); + } + return reasons; +} + +function pendingLineFilter(lines, scene, pendingScenes) { + if (!pendingScenes) return () => true; + const pending = pendingScenes.find((candidate) => candidate.id === scene.id); + if (!pending) return () => false; + const before = pending.beforeProse.split(/\n\s*\n/u).filter((paragraph) => paragraph.trim()).map(readerText); + const after = scene.prose.split(/\n\s*\n/u).filter((paragraph) => paragraph.trim()).map(readerText); + const changedParagraphs = new Set(after.map((text, index) => text !== before[index] ? index + 1 : null).filter(Boolean)); + return (line) => changedParagraphs.has(line.paragraph); +} + +export function runVoiceCheck(root, project, { sceneIds = null, pendingScenes = null } = {}) { + const characters = loadCharacters(root).characters; + const scenes = loadScenes(root, project).filter((scene) => !sceneIds || sceneIds.map(String).includes(scene.id)); + const approved = approvedManuscript(root, project); + const settings = voiceSettings(root); + const lines = scenes.flatMap((scene) => attributeSpans(scene, characters)); + const records = loadProfiles(root, characters); + const profiles = new Map(records.map(({ character, profile }) => [character.id, { character, profile }])); + const staleProfiles = records.filter(({ profile }) => { + const profileHash = profile.approved_manuscript_sha256 || profile.manuscript_sha256; + const hashMismatch = profileHash !== approved.manuscript_sha256; + const recordedHead = profile.approved_head_commit ?? profile.head_commit; + const headMismatch = (Object.prototype.hasOwnProperty.call(profile, "approved_head_commit") + || Object.prototype.hasOwnProperty.call(profile, "head_commit")) + && recordedHead !== approved.head_commit; + return hashMismatch || headMismatch; + }).map(({ character }) => character.name); + const findings = []; + const notes = []; + const legacyNoted = new Set(); + for (const line of lines) { + const record = profiles.get(line.speaker); + if (!record) continue; + const { character, profile } = record; + const scene = scenes.find((candidate) => candidate.id === line.scene_id); + if (!pendingLineFilter(lines, scene, pendingScenes)(line)) continue; + const comparison = comparisonProfile(profile, line); + if (comparison.legacy && !legacyNoted.has(character.id)) { + notes.push({ scene_id: line.scene_id, paragraph: line.paragraph, speaker: character.name, message: "voice profile has no per-line data; run weaver voices build" }); + legacyNoted.add(character.id); + } + const checkedProfile = comparison.profile; + if (!checkedProfile.enough_evidence) { + notes.push({ scene_id: line.scene_id, paragraph: line.paragraph, speaker: character.name, message: `not enough of their lines yet (${checkedProfile.lines} lines, ${checkedProfile.words} words)` }); + continue; + } + const features = lineFeatures(line.text, settings); + const analysis = voiceSignals(line.text, features, checkedProfile, settings); + const distance = profileDistance(features, checkedProfile); + const hardContrast = analysis.hard.profanity || analysis.hard.formal; + const independentSignals = Object.values(analysis.signals).filter(Boolean).length; + if (!hardContrast && independentSignals < 2) continue; + const finding = { + type: "voice", + severity: "warn", + scene_id: line.scene_id, + paragraph: line.paragraph, + speaker: character.name, + line: line.text, + reasons: reasonFor(line.text, features, checkedProfile, settings, analysis), + evidence: evidenceWithoutLine(checkedProfile, line).slice(0, 3) + }; + finding.reason = finding.reasons[0]; + const candidates = records.filter(({ character: other, profile: otherProfile }) => other.id !== character.id && otherProfile.enough_evidence) + .map(({ character: other, profile: otherProfile }) => ({ + character: other, + profile: otherProfile, + distance: profileDistance(features, otherProfile), + shared: sharedDistinctiveWords(line.text, otherProfile), + contrast: hardContrastMatches(features, otherProfile) + })) + .sort((left, right) => Number(right.contrast) - Number(left.contrast) + || left.distance - right.distance + || left.character.id.localeCompare(right.character.id)); + const best = candidates[0]; + const margin = distance - (best?.distance ?? Number.POSITIVE_INFINITY); + if (best && (best.contrast || (best.distance + SOUNDS_LIKE_MARGIN < distance + && (best.shared.length || margin >= SOUNDS_LIKE_MARGIN * 2)))) { + finding.maybe_sounds_like = { + character: best.character.name, + evidence: evidenceWithoutLine(best.profile, line).slice(0, 2) + }; + } + findings.push(finding); + } + return { findings, notes, stale_profiles: staleProfiles, warnings: findings.length, hash: approved.manuscript_sha256 }; +} + +export function attributeDialogue(scene, characters) { + return attributeSpans(scene, characters); +} + +export function voiceProfilesPath(root) { + return join(root, VOICES_ROOT); +} diff --git a/integrity.json b/integrity.json index 99f73ff..54f6d7b 100644 --- a/integrity.json +++ b/integrity.json @@ -5,8 +5,9 @@ "files": { "LICENSE": "3954ad2ee7d56cfda60326c71dcfd3d7edd79a44727f51b5ca5e4b7c33b8c6d6", "NOTICE": "83abe7f22edf5a06f7b4814e8b4b4b124e1b612bd81dd4c6542c0b119fab4976", - "README.md": "ce359e8675d27bca69cadbac2b594431a651f29e654539fdede4d972afebca60", + "README.md": "b9db82abc022f3dc6accda62b5763953923ee01a1b281a3bb719e30b777c4336", "SECURITY.md": "5c9c72ffaa8dfdc744360ea666d2082c68bba2479c61d13810bbd595498afe2b", + "contracts/characters.schema.json": "f9bd6fcc9c2ad363fd805d8d5209c9a292204b0beaf9035a4b6ceadacfd36c7b", "contracts/import.schema.json": "68a9be97050a3e58bacbc984c9e1050d2b0b1db72066af96e0a5fb4019fe9e85", "contracts/integrity.schema.json": "1a8cb4b28748b0f2bfeffc4d0210f5f422d1b69cfed3a386ff60c5de0bdd8d08", "contracts/loop-manifest.schema.json": "edb1022ab2434ea24890fb0a14ad7daee7dda190968c60e9ab306250412e6258", @@ -19,21 +20,23 @@ "docs/HANDOFF.md": "6e37433f60c54989abb1171c6989ed9c0cbab83be0d414d3d36ff6e985a9ee8c", "docs/LOOP.md": "bf3098c9afbcac6254d1682e87ad6b1e6ed798f0cb5e22a5f76a8fd3941fe8d5", "docs/ORIGIN.md": "d66e5d9a0c3e2c05a983c790dd616fc051d2666417ae0e1fd697478e5ff4cda8", - "docs/WORKFLOW.md": "74e029e5d0977906b9a279787516e4e272478d6345705b48e1144911c539b44a", - "engine/changes.mjs": "faf344fd0a96ea9bc6beb414a30c89d9afa2ef40343af2db184fa7729059d4db", + "docs/WORKFLOW.md": "efeaa318ec7aab346c8cc5f96034967e55a0c269fdac3c871142faa76da1896d", + "engine/changes.mjs": "aa99efacf9a953296cd7b323be505363d442c447006381126214db50b9e667d9", + "engine/characters.mjs": "1d8cc8e621e1d2b42ab81ff787be1df0648c5e852e4256dccc8dc2f7d4f0e166", "engine/doctor.mjs": "9fd9036d14712b12cd425bb3c4b6333cef9e56200f609d2b786ebb72de36c017", "engine/export.mjs": "092ad9a4687cf24ca5daa822c844879c8314adc227542cf46ec7b871e43f9f3c", - "engine/hooks.mjs": "3d30b38ec828fbbb7606c7025cc1c63bd83f35a084ceae7abd8688ac4fccf36a", + "engine/hooks.mjs": "07468f9f390360c7fb1f94092b42d9bad987c0843f2f206439b21bce1cf5c01c", "engine/identity.mjs": "670f211b093aa783a0d6ed24be566239c438deb9da75e9ef20748e2c1b057364", "engine/import.mjs": "36b5f8475563d4fd0ea0ca9209d0232598e53d824733137cf46b8dbc2d3a81cc", - "engine/init.mjs": "dc9523ffd32226ae3624a8cba4997ceaaa11bac5fdc69f0909da03addc11a16d", + "engine/init.mjs": "232d97f1034b12014774ca4a438b0ee6106919ca911e3c657e180409b9903d4d", "engine/integrity.mjs": "5c9ae6d22bbd28515887678e0e1ac214edd77ef2d2312fab63eda52407301e9f", "engine/manuscript.mjs": "eb03ce6d453b9b6d3a7e44b5d1a8d40eb88dcb86ae3963aaf0b4597345b14a59", "engine/project.mjs": "9f31f86a6cee011ea2486b7773491e5f6529a0b7f28e14094f1fb436c6ab2b81", "engine/prose.mjs": "e6c75d599abc9fee5fd494c3cd56798135ae21ee15eb9f16d5c83b7c1c877fcd", - "engine/release.mjs": "0b7099a9d6a3709b3e9e5935b7d0dcb3834a1553ed96d78dbb07d25a7abf0622", + "engine/release.mjs": "ffe5cbc72e888823c9b74514629fe2dcd0b4594df55b121b59fabf9c99d99fe1", "engine/rules.mjs": "5b75d64c8a2eadb7703cf75b9c8778e9b90613856f6b29c8c0f4ad3dc37f6b93", "engine/state.mjs": "89204ffd1d349761d8977860f976099d784cd7b14be0ab91f63b3d9a61bcccb1", + "engine/voice.mjs": "1830ff86c67cc6fa7e84c943b9637cd6ecdfd9eca1d6542be5dcd83d91b29cab", "loop.manifest.json": "340b33bb8b03c60ce4997b9182baa18435fc3d6716d660ea8d15ae466663abc0", "package.json": "d1b143a5559d860707d5085265a175e64ef07af09cf12411634e3522c0cb39be", "prompts/architect.md": "51ed331812edfb4801fd60948293eb40f960a2735d949376d80488dc1c3724e4", @@ -42,7 +45,7 @@ "prompts/reviewers/style.md": "4479a4cb676627ca1ff4f7116f15e0cdc1f50fe2763db1388371d80570c9cdd4", "prompts/state-extractor.md": "b2d92d014892f2a1c201003b1cd16cf9d0adcc5e8611f7d350eab612b8947cc5", "scripts/integrity.mjs": "78c0bcdf7b5858de517f02eb09db1febf6d33eec787f09e7c24178f0cb263243", - "scripts/weaver.mjs": "11f6545c0d130ec71db813cbac503314fafb0dd272d7cc21a9a7b736469ac29a", + "scripts/weaver.mjs": "87f6951485e616593280e584cd1ee437b1387606fac91ab96fb65542c42b0442", "templates/book/.claude/settings.json": "9fb07cc94b12c3cce506c1bd2706c351dca83309f1db2bb25f5897cec1735fe2", "templates/book/AGENTS.md": "6bb83885ca4498cfd5e994f2b36cd1688db04d6174905b0a813c5fbedd7c4a10", "templates/book/README.md": "c33756ab69072410e4583f44daf21d08fa6c576816c5ed240284850d6e4b2659", @@ -60,7 +63,9 @@ "templates/book/quality/editorial-rules.md": "9fc6e7e120f0f2a4824056cf23c48874a6030aaea2d9d16d9c2684977f07f3c6", "templates/book/quality/intentional-echoes.json": "387eeacf109e034fa60096efdcb3d238589b7bc714ab30cb209e34a166634404", "templates/book/quality/rules.json": "2090fc31760e0e74ec6184595d8bf12d573088e753f4c77fa8d24b527407fee0", + "templates/book/quality/voice.json": "8a618a77f3b225d0cdd26e99345eca8da0b20c1dd98922d15326428bb73bb485", "templates/book/style-guide/voice-notes.md": "6ebf41b99ba11a8e3b24c26ec38e67898e84e21ba8771e9cbaafe3318e94fb25", + "templates/book/world-bible/characters.json": "c25893fbbe084213d00647a803906cbdce8cf141384ba15136a59c14e5260119", "templates/book/world-bible/characters.md": "98bb4a5bd165c8f1b52cac9a12a5860b2089624055996c0a0968c0cbb4ecc171", "templates/book/world-bible/locations.md": "ea664be763300342e0b2349ae20df17c5eb20cb5440e6bdedd80075af0f8e768", "templates/book/world-bible/terminology.md": "7c8ee96c0eb5f5f16eafe667ce8ddd343fc46762f9ab28153b56af06df109209" diff --git a/scripts/weaver.mjs b/scripts/weaver.mjs index ab86d1e..1189092 100644 --- a/scripts/weaver.mjs +++ b/scripts/weaver.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // SPDX-License-Identifier: Apache-2.0 // Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { existsSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { runDoctor } from "../engine/doctor.mjs"; import { initializeProject } from "../engine/init.mjs"; @@ -14,7 +14,9 @@ import { acceptNarrativeState, buildGroundingPacket, narrativeStateStatus } from import { GENERATOR } from "../engine/identity.mjs"; import { installHooks, runHook } from "../engine/hooks.mjs"; import { runRules } from "../engine/rules.mjs"; -import { approveChapter, approvalHistory, describePendingChanges, rejectChapter, undoApproval } from "../engine/changes.mjs"; +import { approveChapter, approvalHistory, describePendingChanges, pendingChanges, recordImportBaseline, rejectChapter, undoApproval } from "../engine/changes.mjs"; +import { seedCharacters } from "../engine/characters.mjs"; +import { buildVoiceProfiles, runVoiceCheck } from "../engine/voice.mjs"; const argv = process.argv.slice(2); const command = argv.shift() || "help"; @@ -52,6 +54,36 @@ function pendingSummary(report) { }).join("\n"); } +function importedPovNames(root, project) { + const names = []; + const sourcePov = project.source_book?.pov; + if (typeof sourcePov === "string") names.push(sourcePov); + if (Array.isArray(sourcePov)) names.push(...sourcePov); + const scenesRoot = resolve(root, project.source_root); + function walk(directory) { + if (!existsSync(directory)) return; + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = resolve(directory, entry.name); + if (entry.isDirectory()) walk(path); + else if (entry.isFile() && entry.name === "scene.md") { + const pov = readFileSync(path, "utf8").match(/^\*\*POV:\*\*\s*(.+?)\s*$/mu)?.[1]; + if (pov) names.push(pov); + } + } + } + walk(scenesRoot); + return names; +} + +function voiceFindingText(finding) { + const evidence = finding.evidence.map((line) => ` - ${line.text} [${line.scene_id}, paragraph ${line.paragraph}]`).join("\n"); + const sounds = finding.maybe_sounds_like + ? `\n Maybe sounds like ${finding.maybe_sounds_like.character}:\n${finding.maybe_sounds_like.evidence.map((line) => ` - ${line.text} [${line.scene_id}, paragraph ${line.paragraph}]`).join("\n")}` + : ""; + const reasons = finding.reasons?.length ? `\n Reasons:\n${finding.reasons.map((reason) => ` - ${reason}`).join("\n")}` : ""; + return `voice warn: ${finding.scene_id}, paragraph ${finding.paragraph}, ${finding.speaker}: ${finding.reason}${reasons}\n Line: ${finding.line}\n Typical lines:\n${evidence}${sounds}`; +} + function importOmissionsSummary(notImported = {}) { const comments = Number(notImported.comments || 0); const footnotes = Number(notImported.footnotes || 0) + Number(notImported.endnotes || 0); @@ -150,7 +182,8 @@ try { critical_path: checks.criticalPath, unapproved_high_similarity_duplicates: checks.unapprovedDuplicates, narrative_state: checks.state, - rules: { blocking: checks.rules.blocking, warnings: checks.rules.warnings } + rules: { blocking: checks.rules.blocking, warnings: checks.rules.warnings }, + voices: { warnings: checks.voices.warnings, stale_profiles: checks.voices.stale_profiles } }; print(report); if (!checks.ok) process.exitCode = 1; @@ -270,11 +303,13 @@ try { povRule: option("--pov", "title"), chapterPattern: option("--chapter-pattern", DEFAULT_CHAPTER_PATTERN) }); - if (argv.includes("--json")) print(result.record); + const baseline = recordImportBaseline(root, result.project, result.record, result.importPath); + if (argv.includes("--json")) print({ ...result.record, baseline }); else { print(`Imported ${result.record.totals.chapters} chapter(s), ${result.record.totals.scenes} scene(s), ${result.record.totals.words} word(s).`); print(importOmissionsSummary(result.record.not_imported)); for (const chapter of result.record.chapters) print(`Chapter ${chapter.number}: ${chapter.title || "(untitled)"} — POV ${chapter.pov || "(none)"}`); + print(`Saved your original as the starting point (Approval ${baseline.id}). Repairs are reviewed against it, one chapter at a time.`); } } } else if (command === "export") { @@ -283,6 +318,40 @@ try { const { project } = projectContext(); print(exportBook(root, project, option("--format"), option("--out"))); } + } else if (command === "characters" && positional() === "seed") { + if (refuseIfDoctorBlocks(root, command)) process.exitCode = 1; + else { + const { project } = projectContext(); + const result = seedCharacters(root, importedPovNames(root, project)); + if (argv.includes("--json")) print(result); + else print(result.added.length + ? `Added ${result.added.length} character entr${result.added.length === 1 ? "y" : "ies"} to ${result.path}.` + : `No new POV characters found; registry unchanged at ${result.path}.`); + } + } else if (command === "voices" && positional() === "build") { + if (refuseIfDoctorBlocks(root, command)) process.exitCode = 1; + else { + const { project } = projectContext(); + const result = buildVoiceProfiles(root, project); + if (argv.includes("--json")) print(result); + else print(`Built ${result.profiles.length} voice profile(s) from manuscript ${result.manuscript_sha256}.`); + } + } else if (command === "voices" && positional() === "check") { + const { project } = projectContext(); + const selectedScene = option("--scene"); + const pending = argv.includes("--pending") ? pendingChanges(root, project) : null; + const result = runVoiceCheck(root, project, { + sceneIds: selectedScene ? [selectedScene] : null, + pendingScenes: pending?.scenes || null + }); + if (argv.includes("--json")) print(result); + else { + for (const name of result.stale_profiles) print(`Warning: voice profiles are out of date, run weaver voices build (${name}).`); + for (const note of result.notes) print(`Note: ${note.scene_id}, paragraph ${note.paragraph}, ${note.speaker}: ${note.message}`); + if (!result.findings.length && !result.notes.length) print("No voice warnings."); + for (const finding of result.findings) print(voiceFindingText(finding)); + print(`\n${result.warnings} voice warning(s).`); + } } else if (command === "version" || command === "--version") { print(GENERATOR); } else if (command === "help") { @@ -295,6 +364,9 @@ Usage: weaver status [--root directory] weaver check [--root directory] weaver rules [--root directory] [--scene scene-id] [--json] + weaver characters seed [--root directory] [--json] + weaver voices build [--root directory] [--json] + weaver voices check [--scene scene-id | --pending] [--root directory] [--json] weaver changes [--root directory] [--json] weaver approve [--chapter number] [--note "text"] [--root directory] weaver reject [--chapter number] [--root directory] @@ -316,6 +388,9 @@ Usage: weaver status [--root directory] weaver check [--root directory] weaver rules [--root directory] [--scene scene-id] [--json] + weaver characters seed [--root directory] [--json] + weaver voices build [--root directory] [--json] + weaver voices check [--scene scene-id | --pending] [--root directory] [--json] weaver changes [--root directory] [--json] weaver approve [--chapter number] [--note "text"] [--root directory] weaver reject [--chapter number] [--root directory] diff --git a/templates/book/quality/voice.json b/templates/book/quality/voice.json new file mode 100644 index 0000000..b74d87f --- /dev/null +++ b/templates/book/quality/voice.json @@ -0,0 +1,4 @@ +{ + "profanity": [], + "formal": [] +} diff --git a/templates/book/world-bible/characters.json b/templates/book/world-bible/characters.json new file mode 100644 index 0000000..b897b58 --- /dev/null +++ b/templates/book/world-bible/characters.json @@ -0,0 +1,5 @@ +{ + "$schema": "https://pickbits.ai/schemas/weaver/characters-1.json", + "version": 1, + "characters": [] +} diff --git a/test/approvals.test.mjs b/test/approvals.test.mjs index a1d853f..017fd7d 100644 --- a/test/approvals.test.mjs +++ b/test/approvals.test.mjs @@ -172,3 +172,30 @@ test("approve keeps an author's existing Git identity instead of overriding it l rmSync(globalConfig, { force: true }); } }); + +test("import records the original book as the approved starting point", () => { + const root = mkdtempSync(join(tmpdir(), "weaver-import-baseline-")); + const source = join(tmpdir(), `weaver-baseline-source-${process.pid}.md`); + try { + let result = run(ROOT, ["init", join(root, "book"), "--id", "baseline", "--title", "Baseline", "--empty"]); + assert.equal(result.status, 0, result.stderr); + const book = join(root, "book"); + writeFileSync(source, "# Wren\n\nThe original first scene.\n\n* * *\n\nThe original second scene.\n"); + result = run(ROOT, ["import", source, "--root", book]); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + assert.match(result.stdout, /Saved your original as the starting point/); + assert.match(git(book, ["log", "-1", "--format=%s"]).stdout, /^Import original manuscript: /); + result = run(ROOT, ["changes", "--root", book, "--json"]); + assert.equal(result.status, 0, result.stderr); + const afterImport = JSON.parse(result.stdout); + assert.equal(afterImport.pending, false, "nothing is pending right after import"); + assert.equal(afterImport.chapters.length, 0); + const scene = join(book, "manuscript", "chapter-01", "scene-01", "scene.md"); + writeFileSync(scene, readFileSync(scene, "utf8").replace("original first", "repaired first")); + result = run(ROOT, ["approve", "--root", book, "--chapter", "1"]); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + } finally { + rmSync(root, { recursive: true, force: true }); + rmSync(source, { force: true }); + } +}); diff --git a/test/voice-holdout.test.mjs b/test/voice-holdout.test.mjs new file mode 100644 index 0000000..7273ad5 --- /dev/null +++ b/test/voice-holdout.test.mjs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import assert from "node:assert/strict"; +import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { buildVoiceProfiles, runVoiceCheck } from "../engine/voice.mjs"; +import { readProject } from "../engine/project.mjs"; + +const ROOT = resolve("."); + +function scenePath(root, chapter, scene) { + return join(root, "manuscript", `chapter-${String(chapter).padStart(2, "0")}`, `scene-${String(scene).padStart(2, "0")}`, "scene.md"); +} + +function writeScene(root, chapter, scene, pov, title, prose) { + const path = scenePath(root, chapter, scene); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `# Chapter ${chapter}, Scene ${scene}\n\n**POV:** ${pov}\n\n---\n\n# ${title}\n\n${prose}\n\n---\n\n**Words:** 0\n`); +} + +function holdoutBook() { + const root = mkdtempSync(join(tmpdir(), "weaver-voice-holdout-")); + cpSync(join(ROOT, "templates", "book"), root, { recursive: true }); + writeFileSync(join(root, "world-bible", "characters.json"), `${JSON.stringify({ + $schema: "https://pickbits.ai/schemas/weaver/characters-1.json", + version: 1, + characters: [ + { id: "grell", name: "Grell", aliases: [], pov: false }, + { id: "magister-anselm", name: "Magister Anselm", aliases: ["Anselm"], pov: false } + ] + }, null, 2)}\n`); + const grell = [ + `"Pull, you lazy sods!" Grell shouted.`, + `"Ain't nobody paying us to stand about," Grell said. "Get it done."`, + `Grell spat over the rail. "Bloody tide's turning. Move!"`, + `"Don't you look at me like that," Grell growled. "I've hauled bigger."`, + `"Where's the damn hook?" Grell yelled. "Can't find nothing on this dock."`, + `"We're done when I say we're done," said Grell.`, + `Grell laughed. "You'll learn. Everyone does, sooner or later."`, + `"Hell's teeth, that's heavy!" Grell snapped. "Lift with your legs."`, + `"Won't be pretty, but it'll float," Grell muttered.`, + `"Shut it and haul," said Grell. "Now!"`, + `"Mind the crates, they ain't paid for yet," Grell said.`, + `"Oi! Keep your bloody hands off the cargo," Grell shouted.`, + `"Can't stand here all day gawping, can we?" said Grell.`, + `"You'll get your coin when the job's done, not before," Grell growled.`, + `"Tie it off. Tighter than that, you daft lump!" Grell snapped.` + ]; + const anselm = [ + `"One must consider the statutes before deciding whether the proposed action is permissible," said Magister Anselm.`, + `"Indeed, the council shall require a careful account of every witness before deliberation may properly begin," said Magister Anselm.`, + `"Perhaps the precedent is not decisive, although it offers guidance whom a prudent magistrate would be wise to consult," Magister Anselm said.`, + `"I daresay the evidence warrants patience, for haste in such a matter would be imprudent and difficult to remedy," said Magister Anselm.`, + `"Pray, record the objection precisely, since the statutes distinguish a considered refusal from an accidental omission," said Magister Anselm.`, + `"One must not confuse a council's authority with the conclusion that its members may reach after full deliberation," said Magister Anselm.`, + `"Indeed, the precedent requires that we identify the person to whom the obligation belongs before judgment is announced," said Magister Anselm.`, + `"Perhaps we shall proceed when the record is complete, and I daresay that no responsible scholar would recommend otherwise," said Magister Anselm.`, + `"Whom the statutes bind, the council must hear, for justice is not improved by excluding a relevant account," said Magister Anselm.`, + `"It would be imprudent to dismiss the matter before the council has completed its deliberation and considered every applicable precedent," said Magister Anselm.` + ]; + writeScene(root, 1, 1, "Grell", "Grell", grell.join("\n\n")); + writeScene(root, 2, 1, "Magister Anselm", "Magister Anselm", anselm.join("\n\n")); + const project = readProject(root); + spawnSync("git", ["init", "-q"], { cwd: root }); + spawnSync("git", ["config", "--local", "user.name", "Voice Holdout"], { cwd: root }); + spawnSync("git", ["config", "--local", "user.email", "voice-holdout@example.invalid"], { cwd: root }); + spawnSync("git", ["add", "-A"], { cwd: root }); + const commit = spawnSync("git", ["commit", "-qm", "Voice hold-out baseline"], { cwd: root, encoding: "utf8" }); + assert.equal(commit.status, 0, commit.stderr); + buildVoiceProfiles(root, project); + return { root, project }; +} + +test("hold-out voice check catches both directions with concrete measurements", () => { + const { root, project } = holdoutBook(); + try { + const path = scenePath(root, 3, 1); + writeScene(root, 3, 1, "Grell", "Chapter 3", [ + `"It would be most imprudent to proceed before the council has deliberated upon the matter in full," said Grell.`, + `"Bloody fools, the lot of you," said Anselm.`, + `"Grab the line and don't let go," Grell said.`, + `"We should go," she said.` + ].join("\n\n")); + const result = runVoiceCheck(root, project, { sceneIds: ["3-1"] }); + const formal = result.findings.find((finding) => finding.speaker === "Grell"); + const profanity = result.findings.find((finding) => finding.speaker === "Magister Anselm"); + assert.ok(formal); + assert.equal(formal.maybe_sounds_like.character, "Magister Anselm"); + assert.match(formal.reasons.join("\n"), /\d+ words per sentence|\d+ formal marker|contractions in \d+ words/u); + assert.ok(profanity); + assert.equal(profanity.maybe_sounds_like.character, "Grell"); + assert.match(profanity.reasons.join("\n"), /\d+ profanity/u); + assert.equal(result.findings.some((finding) => finding.line.includes("Grab the line")), false); + assert.equal(result.findings.some((finding) => finding.line.includes("We should go")), false); + assert.equal(result.stale_profiles.length, 0); + assert.match(readFileSync(path, "utf8"), /most imprudent/u); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/voice-loo.test.mjs b/test/voice-loo.test.mjs new file mode 100644 index 0000000..f6c7930 --- /dev/null +++ b/test/voice-loo.test.mjs @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import assert from "node:assert/strict"; +import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { buildVoiceProfiles, runVoiceCheck } from "../engine/voice.mjs"; +import { readProject } from "../engine/project.mjs"; + +const ROOT = resolve("."); +const CLI = join(ROOT, "scripts", "weaver.mjs"); + +function scenePath(root, chapter, scene) { + return join(root, "manuscript", `chapter-${String(chapter).padStart(2, "0")}`, `scene-${String(scene).padStart(2, "0")}`, "scene.md"); +} + +function writeScene(root, chapter, scene, pov, prose) { + const path = scenePath(root, chapter, scene); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `# Chapter ${chapter}, Scene ${scene}\n\n**POV:** ${pov}\n\n---\n\n# ${pov}\n\n${prose}\n\n---\n\n**Words:** 0\n`); +} + +function looBook() { + const root = mkdtempSync(join(tmpdir(), "weaver-voice-loo-")); + cpSync(join(ROOT, "templates", "book"), root, { recursive: true }); + writeFileSync(join(root, "world-bible", "characters.json"), `${JSON.stringify({ + $schema: "https://pickbits.ai/schemas/weaver/characters-1.json", + version: 1, + characters: [ + { id: "quenlor", name: "Quenlor", aliases: [], pov: false }, + { id: "tervik", name: "Tervik", aliases: [], pov: false }, + { id: "bravax", name: "Bravax", aliases: [], pov: false } + ] + }, null, 2)}\n`); + + const scholarLines = [ + `"Sit down, you bloody fool," said Quenlor.`, + ...Array.from({ length: 9 }, (_, index) => `"The council reviewed the harbor ledger before sunrise, and every witness received a patient hearing from the assembled clerks in record ${index + 1}," said Quenlor.`) + ]; + const terseLines = [ + `"Indeed, one must examine the river accounts before the committee can approve this course, because each detail remains relevant to the public record and to the obligations of every office," said Tervik.`, + ...Array.from({ length: 9 }, (_, index) => `"The committee reviewed the harbor ledger before sunrise, and every witness received a patient hearing from the assembled clerks in record ${index + 1}," said Tervik.`) + ]; + const swearingLines = [ + `"Damn this rotten lock, it will not turn for any honest hand," said Bravax.`, + `"Hell, the blasted cart is stuck again beside the northern gate," said Bravax.`, + `"Bloody weather makes every simple crossing a miserable affair," said Bravax.`, + `"Damn the broken rope, someone fetch a stronger length from storage," said Bravax.`, + `"This shit will not move unless we push together on three," said Bravax.`, + `"Hell's teeth, the watch has left another crate across the road," said Bravax.`, + `"Bloody fools, keep your hands clear while I shift the heavy beam," said Bravax.`, + `"Damn it, the signal lantern has gone dark beside the outer wall," said Bravax.`, + `"The bastard hinge is bent, but a hammer should set it straight," said Bravax.`, + `"Crap, the last cart has vanished before we loaded the final bundle," said Bravax.` + ]; + writeScene(root, 1, 1, "Quenlor", scholarLines.join("\n\n")); + writeScene(root, 1, 2, "Tervik", terseLines.join("\n\n")); + writeScene(root, 1, 3, "Bravax", swearingLines.join("\n\n")); + + assert.equal(spawnSync("git", ["init", "-q"], { cwd: root }).status, 0); + assert.equal(spawnSync("git", ["config", "--local", "user.name", "Voice Loo Test"], { cwd: root }).status, 0); + assert.equal(spawnSync("git", ["config", "--local", "user.email", "voice-loo@example.invalid"], { cwd: root }).status, 0); + assert.equal(spawnSync("git", ["add", "-A"], { cwd: root }).status, 0); + const commit = spawnSync("git", ["commit", "-qm", "Voice leave-one-out baseline"], { cwd: root, encoding: "utf8" }); + assert.equal(commit.status, 0, commit.stderr); + const project = readProject(root); + const built = buildVoiceProfiles(root, project); + return { root, project, built }; +} + +test("voice profiles leave the checked line out of hard contrasts and evidence", () => { + const { root, project, built } = looBook(); + try { + const quenlorProfile = built.profiles.find((profile) => profile.character_id === "quenlor"); + assert.deepEqual(Object.keys(quenlorProfile.totals), ["lines", "words", "sentences", "contractions", "profanity", "formal_markers", "questions", "exclamations"]); + assert.equal(quenlorProfile.totals.profanity, 1); + assert.equal(quenlorProfile.line_records.length, quenlorProfile.lines); + + const result = runVoiceCheck(root, project); + const profanity = result.findings.find((finding) => finding.speaker === "Quenlor"); + const formal = result.findings.find((finding) => finding.speaker === "Tervik"); + assert.ok(profanity); + assert.match(profanity.line, /Sit down, you bloody fool/u); + assert.match(profanity.reason, /profanity/u); + assert.equal(profanity.evidence.some((line) => /Sit down, you bloody fool/u.test(line.text)), false); + assert.ok(formal); + assert.match(formal.line, /Indeed, one must/u); + assert.match(formal.reasons.join("\n"), /formal marker/u); + assert.equal(formal.evidence.some((line) => /Indeed, one must/u.test(line.text)), false); + assert.equal(result.findings.some((finding) => finding.speaker === "Bravax"), false); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("old voice profiles remain readable and request a rebuild", () => { + const { root, project } = looBook(); + try { + const path = join(root, "world-bible", "voices", "quenlor.json"); + const profile = JSON.parse(readFileSync(path, "utf8")); + delete profile.line_records; + delete profile.totals; + profile.version = 1; + writeFileSync(path, `${JSON.stringify(profile, null, 2)}\n`); + + const result = runVoiceCheck(root, project); + assert.ok(result.notes.some((note) => /run weaver voices build/u.test(note.message))); + const check = spawnSync(process.execPath, [CLI, "voices", "check", "--root", root], { cwd: ROOT, encoding: "utf8" }); + assert.equal(check.status, 0, `${check.stdout}\n${check.stderr}`); + assert.match(check.stdout, /run weaver voices build/u); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/voice-precision.test.mjs b/test/voice-precision.test.mjs new file mode 100644 index 0000000..8a41bd7 --- /dev/null +++ b/test/voice-precision.test.mjs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import assert from "node:assert/strict"; +import { cpSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { buildVoiceProfiles, runVoiceCheck } from "../engine/voice.mjs"; +import { readProject } from "../engine/project.mjs"; + +const ROOT = resolve("."); + +function scenePath(root, chapter, scene) { + return join(root, "manuscript", `chapter-${String(chapter).padStart(2, "0")}`, `scene-${String(scene).padStart(2, "0")}`, "scene.md"); +} + +function writeScene(root, chapter, scene, pov, prose) { + const path = scenePath(root, chapter, scene); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, `# Chapter ${chapter}, Scene ${scene}\n\n**POV:** ${pov}\n\n---\n\n${prose}\n\n---\n\n**Words:** 0\n`); +} + +function precisionBook() { + const root = mkdtempSync(join(tmpdir(), "weaver-voice-precision-")); + cpSync(join(ROOT, "templates", "book"), root, { recursive: true }); + writeFileSync(join(root, "world-bible", "characters.json"), `${JSON.stringify({ + $schema: "https://pickbits.ai/schemas/weaver/characters-1.json", + version: 1, + characters: [ + { id: "brask", name: "Brask", aliases: [], pov: true }, + { id: "lady-veyra", name: "Lady Veyra", aliases: [], pov: true }, + { id: "hollis", name: "Hollis", aliases: [], pov: true } + ] + }, null, 2)}\n`); + const brask = Array.from({ length: 24 }, () => `Brask said, "I ain't waiting; haul the rope, bloody fools."`); + const veyra = Array.from({ length: 12 }, () => `Lady Veyra said, "Indeed, perhaps the council shall consider the matter with patience before the appointed witnesses arrive this evening."`); + const hollis = Array.from({ length: 32 }, () => `Hollis said, "Hold the gate until dawn."`); + writeScene(root, 1, 1, "Brask", [...brask, ...veyra, ...hollis].join("\n\n")); + const project = readProject(root); + spawnSync("git", ["init", "-q"], { cwd: root }); + spawnSync("git", ["config", "--local", "user.name", "Voice Precision"], { cwd: root }); + spawnSync("git", ["config", "--local", "user.email", "voice-precision@example.invalid"], { cwd: root }); + spawnSync("git", ["add", "-A"], { cwd: root }); + const commit = spawnSync("git", ["commit", "-qm", "Voice precision baseline"], { cwd: root, encoding: "utf8" }); + assert.equal(commit.status, 0, commit.stderr); + buildVoiceProfiles(root, project); + return { root, project }; +} + +test("voice precision ignores short quirks and requires independent long-line signals", () => { + const { root, project } = precisionBook(); + try { + writeScene(root, 2, 1, "Brask", [ + `Brask said, "Hell,"`, + `Brask said, "Bloody hell,"`, + `Brask said, "Where'd you get this?"`, + `Brask said, "Tide turns; keep moving out before dawn."`, + `Lady Veyra said, "Perhaps,"`, + `Lady Veyra said, "Indeed, the council shall review the evidence before we permit any witness to depart from this chamber at sunset tonight. Perhaps the record remains incomplete, but one must preserve every testimony until the appointed clerk has verified each seal properly. Whom the magistrate summons, we shall answer with courtesy, for justice requires patience when the final question has been considered."`, + `Lady Veyra said, "Sit down, you bloody fool,"`, + `Brask said, "Indeed, one must consider the ancient harbor ledger before we depart, because every marked crate conceals a debt that patient hands alone can settle; perhaps the watchmen shall close the western gate against us again when the tide withdraws tonight."`, + `Hollis said, "How many?"`, + `Hollis said, "What did you see?"`, + `Hollis said, "Stand aside; the bridge is unsafe, and indeed we shall secure the prisoners before anyone crosses until the signal arrives from command."` + ].join("\n\n")); + + const result = runVoiceCheck(root, project, { sceneIds: ["2-1"] }); + const finding = (text) => result.findings.find((entry) => entry.line.includes(text)); + for (const text of ["Hell,", "Bloody hell,", "Where'd you get this?", "Tide turns", "Perhaps,", "the council shall review", "How many?", "What did you see?"]) { + assert.equal(finding(text), undefined, `unexpected finding for ${text}`); + } + const brask = finding("ancient harbor ledger"); + assert.ok(brask); + assert.match(brask.reasons.join("\n"), /three times longer|formal marker/u); + assert.ok(brask.maybe_sounds_like, JSON.stringify(result.findings)); + assert.equal(brask.maybe_sounds_like.character, "Lady Veyra"); + + const veyra = finding("Sit down, you bloody fool"); + assert.ok(veyra); + assert.match(veyra.reason, /profanity/u); + + const hollis = finding("Stand aside; the bridge"); + assert.ok(hollis); + assert.match(hollis.reasons.join("\n"), /three times longer|formal marker/u); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/test/voice.test.mjs b/test/voice.test.mjs new file mode 100644 index 0000000..d90c9c6 --- /dev/null +++ b/test/voice.test.mjs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 Mark Pickering and PICKBITS LLC. Part of PickBits Weaver. +import assert from "node:assert/strict"; +import { cpSync, mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { attributeDialogue, buildVoiceProfiles, runVoiceCheck } from "../engine/voice.mjs"; +import { loadCharacters } from "../engine/characters.mjs"; +import { readProject } from "../engine/project.mjs"; + +const ROOT = resolve("."); +const CLI = join(ROOT, "scripts", "weaver.mjs"); + +function scenePath(root, chapter, scene) { + return join(root, "manuscript", `chapter-${String(chapter).padStart(2, "0")}`, `scene-${String(scene).padStart(2, "0")}`, "scene.md"); +} + +function writeScene(root, chapter, scene, pov, prose) { + const path = scenePath(root, chapter, scene); + const directory = dirname(path); + mkdirSync(directory, { recursive: true }); + const header = `# Chapter ${chapter}, Scene ${scene}\n\n**POV:** ${pov}\n\n---\n\n`; + writeFileSync(path, `${header}${prose}\n\n---\n\n**Words:** 0\n`); + return path; +} + +function voiceBook() { + const root = mkdtempSync(join(tmpdir(), "weaver-voice-")); + cpSync(join(ROOT, "templates", "book"), root, { recursive: true }); + spawnSync("git", ["init", "-q"], { cwd: root }); + writeFileSync(join(root, "world-bible", "characters.json"), `${JSON.stringify({ + $schema: "https://pickbits.ai/schemas/weaver/characters-1.json", + version: 1, + characters: [ + { id: "brask", name: "Brask", aliases: ["the sergeant"], pov: true }, + { id: "lady-veyra", name: "Lady Veyra", aliases: [], pov: true }, + { id: "oswin", name: "Oswin", aliases: [], pov: false } + ] + }, null, 2)}\n`); + const brask = [ + `Brask said, "Ain't time to dawdle, bloody fools, move your boots!"`, + `Brask said, "I won't wait, so get your gear."`, + `Brask checked the gate. "Move, and keep low, you lot."`, + `Brask said, "We don't need a map, just eyes."`, + `Brask said, "The rain won't stop us, not today."`, + `Brask said, "I've got the rope, and it holds."`, + `Brask said, "Ain't room for cowards here, bloody hell!"`, + `Brask said, "Keep your head down and follow."`, + `Brask said, "We'll cross before dark, if we hurry."`, + `Brask said, "I can't hear the drums, speak up."`, + `Brask said, "Bloody mud, bloody boots, bloody weather!"`, + `Brask said, "Let's get moving, before dawn breaks."` + ]; + const veyra = [ + `"Perhaps we should consider the eastern road, whose stones remain dry, and the messengers must arrive before the gates are closed for the evening watch," said Lady Veyra.`, + `Lady Veyra asked, "Indeed, shall we permit the council to decide our course?"`, + `"The arrangement is elegant, and shall satisfy the council," said Lady Veyra.`, + `Lady Veyra replied, "Whom would you have us trust when every witness has vanished?"`, + `"Pray, let the lanterns remain lit until the final courier returns," said Lady Veyra.`, + // Keeps Veyra's evidence clearly above the 150-word hard-contrast minimum (it sat at 149). + `"We shall record every answer faithfully, whatever the hour," said Lady Veyra.`, + `Lady Veyra said, "Perhaps the silence contains a warning that patience alone may reveal."`, + `"One must consider the consequence before choosing an answer that cannot be recalled," said Lady Veyra.`, + `Lady Veyra continued, "Indeed, the eastern gate shall open only after the bells have sounded."`, + `"Madam is not the proper address, but the courtesy is appreciated nonetheless," said Lady Veyra.`, + `Lady Veyra answered, "Perhaps we shall proceed with dignity, regardless of the court's impatience."`, + `"I daresay the evidence is persuasive, although the conclusion remains morally uncertain," said Lady Veyra.`, + `Lady Veyra murmured, "Indeed, one must preserve the record so that justice may be measured."` + ]; + const oswin = [`Oswin said, "I wait."`, `Oswin said, "The lamp is low."`, `Oswin said, "I heard footsteps."`]; + writeScene(root, 1, 1, "Brask", [...brask.slice(0, 6), ...veyra.slice(0, 6), `"No one knows," she said.`].join("\n\n")); + writeScene(root, 1, 2, "Lady Veyra", [...brask.slice(6), ...veyra.slice(6)].join("\n\n")); + writeScene(root, 2, 1, "Oswin", oswin.join("\n\n")); + spawnSync("git", ["config", "--local", "user.name", "Voice Test"], { cwd: root }); + spawnSync("git", ["config", "--local", "user.email", "voice@example.invalid"], { cwd: root }); + spawnSync("git", ["add", "-A"], { cwd: root }); + spawnSync("git", ["commit", "-qm", "Voice fixture baseline"], { cwd: root }); + return root; +} + +test("dialogue attribution covers tags, beats, continuation, and pronoun limits", () => { + const characters = [ + { id: "brask", name: "Brask", aliases: ["the sergeant"], pov: true }, + { id: "lady-veyra", name: "Lady Veyra", aliases: [], pov: true } + ]; + const lines = attributeDialogue({ id: "1-1", prose: [ + `Brask said, "Ain't time!"`, + `Lady Veyra asked, "Perhaps we should proceed, indeed?"`, + `Brask checked the gate. "Move, and keep low."`, + `Brask said, "We cross at dawn, and I ain't afraid.`, + `"The river is cold, but we'll cross it."`, + `"No one knows," she said.` + ].join("\n\n") }, characters); + assert.deepEqual(lines.map((line) => line.method), ["tag", "tag", "beat", "tag", "continuation", "unattributed"]); + assert.equal(lines.at(-1).speaker, null); +}); + +test("profiles and checks are deterministic, evidentiary, stale-aware, and advisory", () => { + const root = voiceBook(); + try { + const project = readProject(root); + const first = buildVoiceProfiles(root, project); + const before = Object.fromEntries(first.profiles.map((profile) => [profile.character_id, readFileSync(join(root, "world-bible", "voices", `${profile.character_id}.json`), "utf8")])); + const second = buildVoiceProfiles(root, project); + for (const profile of second.profiles) assert.equal(readFileSync(join(root, "world-bible", "voices", `${profile.character_id}.json`), "utf8"), before[profile.character_id]); + assert.equal(first.profiles.find((profile) => profile.character_id === "brask").enough_evidence, true); + assert.equal(first.profiles.find((profile) => profile.character_id === "lady-veyra").enough_evidence, true); + assert.equal(first.profiles.find((profile) => profile.character_id === "oswin").enough_evidence, false); + + const path = scenePath(root, 1, 2); + const raw = readFileSync(path, "utf8"); + const footer = raw.lastIndexOf("\n---"); + writeFileSync(path, `${raw.slice(0, footer)}\nBrask said, "Indeed, the eastern bastion is an imposing and ceremonious structure whose antiquated stones shall perhaps outlast every ordinary ambition."\n\nBrask said, "I ain't done yet!"\n\n"Bloody hell!" said Lady Veyra.\n\nOswin said, "I wait."\n${raw.slice(footer)}`); + const result = runVoiceCheck(root, project); + assert.deepEqual(result.stale_profiles, []); + assert.ok(result.findings.some((finding) => finding.speaker === "Brask" && finding.maybe_sounds_like?.character === "Lady Veyra")); + assert.ok(result.findings.some((finding) => finding.speaker === "Lady Veyra" && /profanity/i.test(finding.reason))); + assert.ok(result.findings.flatMap((finding) => finding.reasons || []).some((reason) => /\d+(?:\.\d+)? words per sentence|\d+ profanity|\d+ formal marker|\d+ exclamation|\d+ contraction/u.test(reason))); + assert.equal(result.findings.some((finding) => finding.speaker === "Oswin"), false); + assert.ok(result.notes.some((note) => /not enough of their lines yet \(3 lines,/.test(note.message))); + assert.equal(result.findings.every((finding) => finding.severity === "warn" && finding.type === "voice"), true); + + const changes = spawnSync(process.execPath, [CLI, "changes", "--root", root, "--json"], { cwd: ROOT, encoding: "utf8" }); + assert.equal(changes.status, 0, changes.stderr); + assert.ok(JSON.parse(changes.stdout).chapters[0].voice_warnings > 0); + + const check = spawnSync(process.execPath, [CLI, "check", "--root", root], { cwd: ROOT, encoding: "utf8" }); + assert.notEqual(check.status, 0, "state is stale in this fixture, but voice findings are not the failure"); + assert.doesNotMatch(check.stdout, /voice warning/); + assert.equal(JSON.parse(check.stdout).voices.warnings, result.warnings); + + const hook = spawnSync(process.execPath, [CLI, "hook", "post"], { + cwd: root, + input: JSON.stringify({ cwd: root, tool_name: "Edit", tool_input: { file_path: path } }), + encoding: "utf8" + }); + assert.equal(hook.status, 0, hook.stderr); + const hookOutput = JSON.parse(hook.stdout); + assert.equal(hookOutput.decision, undefined); + assert.match(hookOutput.hookSpecificOutput.additionalContext, /Typical lines/); + + const approval = spawnSync(process.execPath, [CLI, "approve", "--chapter", "1", "--root", root], { cwd: ROOT, encoding: "utf8" }); + assert.equal(approval.status, 0, `${approval.stdout}\n${approval.stderr}`); + const approvedProfile = JSON.parse(readFileSync(join(root, "world-bible", "voices", "brask.json"), "utf8")); + const head = spawnSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).stdout.trim(); + assert.equal(approvedProfile.approved_head_commit, head); + assert.equal(runVoiceCheck(root, project).stale_profiles.length, 0); + approvedProfile.approved_manuscript_sha256 = "simulated-stale-profile"; + writeFileSync(join(root, "world-bible", "voices", "brask.json"), `${JSON.stringify(approvedProfile, null, 2)}\n`); + assert.ok(runVoiceCheck(root, project).stale_profiles.includes("Brask")); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); + +test("characters seed adds imported POVs once", () => { + const root = mkdtempSync(join(tmpdir(), "weaver-voice-seed-")); + const source = join(root, "finished.md"); + const book = join(root, "book"); + try { + const initialized = spawnSync(process.execPath, [CLI, "init", book, "--id", "voice-seed", "--title", "Voice Seed", "--empty"], { cwd: ROOT, encoding: "utf8" }); + assert.equal(initialized.status, 0, `${initialized.stdout}\n${initialized.stderr}`); + assert.equal(spawnSync("git", ["init", "-q"], { cwd: book }).status, 0); + writeFileSync(source, "# Brask\n\nA short imported scene.\n\n* * *\n\n# Lady Veyra\n\nA formal imported scene."); + let result = spawnSync(process.execPath, [CLI, "import", source, "--root", book], { cwd: ROOT, encoding: "utf8" }); + assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); + result = spawnSync(process.execPath, [CLI, "characters", "seed", "--root", book, "--json"], { cwd: ROOT, encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(loadCharacters(book).characters.map((character) => character.name), ["Brask", "Lady Veyra"]); + const registry = readFileSync(join(book, "world-bible", "characters.json"), "utf8"); + result = spawnSync(process.execPath, [CLI, "characters", "seed", "--root", book, "--json"], { cwd: ROOT, encoding: "utf8" }); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(join(book, "world-bible", "characters.json"), "utf8"), registry); + assert.equal(JSON.parse(result.stdout).added.length, 0); + } finally { + rmSync(root, { recursive: true, force: true }); + } +});