diff --git a/packages/chat-ui/src/composer.test.tsx b/packages/chat-ui/src/composer.test.tsx index e705121b10..44aeec1ccc 100644 --- a/packages/chat-ui/src/composer.test.tsx +++ b/packages/chat-ui/src/composer.test.tsx @@ -1,6 +1,12 @@ import { describe, expect, test } from "bun:test"; -import { canStopComposer, insertTextAtCaret } from "./composer"; +import { + canStopComposer, + insertTextAtCaret, + spliceDictationTranscript, + speechRecognitionConstructor, + transcriptFromSpeechResults, +} from "./composer"; import { isAwaitingReply } from "./streaming-reply"; describe("insertTextAtCaret", () => { @@ -23,6 +29,71 @@ describe("insertTextAtCaret", () => { }); }); +describe("spliceDictationTranscript", () => { + test("inserts a space so two words do not glue together", () => { + const result = spliceDictationTranscript("hello", "", "world"); + expect(result.text).toBe("hello world"); + expect(result.caret).toBe(11); + }); + + test("does not add a second space when the prefix already ends with one", () => { + const result = spliceDictationTranscript("hello ", "", "world"); + expect(result.text).toBe("hello world"); + expect(result.caret).toBe(11); + }); + + test("keeps a following suffix on its own word boundary", () => { + const result = spliceDictationTranscript("hello", "there", "world"); + expect(result.text).toBe("hello world there"); + expect(result.caret).toBe(11); + }); + + test("drops an empty transcript without changing the draft", () => { + const result = spliceDictationTranscript("hello", " there", " "); + expect(result.text).toBe("hello there"); + expect(result.caret).toBe(5); + }); +}); + +describe("transcriptFromSpeechResults", () => { + test("concatenates every alternative in order", () => { + expect( + transcriptFromSpeechResults([ + { isFinal: true, length: 1, 0: { transcript: "hello " } }, + { isFinal: false, length: 1, 0: { transcript: "world" } }, + ]), + ).toBe("hello world"); + }); +}); + +describe("speechRecognitionConstructor", () => { + test("returns null when the browser has no speech recognition", () => { + expect(speechRecognitionConstructor({})).toBeNull(); + }); + + test("prefers SpeechRecognition over the webkit prefix", () => { + function SpeechRecognition() { + return undefined; + } + function webkitSpeechRecognition() { + return undefined; + } + const found = speechRecognitionConstructor({ + SpeechRecognition, + webkitSpeechRecognition, + }); + expect(Object.is(found, SpeechRecognition)).toBe(true); + }); + + test("falls back to webkitSpeechRecognition", () => { + function webkitSpeechRecognition() { + return undefined; + } + const found = speechRecognitionConstructor({ webkitSpeechRecognition }); + expect(Object.is(found, webkitSpeechRecognition)).toBe(true); + }); +}); + // CL-7201: the composer's stop affordance is a stand-in for "is there a // turn to cancel" — offered whenever the host says a turn is running, // independent of the composer's own `sending`/`preparing` state (queuing diff --git a/packages/chat-ui/src/composer.tsx b/packages/chat-ui/src/composer.tsx index e6ea463820..8a73ec5034 100644 --- a/packages/chat-ui/src/composer.tsx +++ b/packages/chat-ui/src/composer.tsx @@ -7,7 +7,15 @@ // does not compose with an inline mention popover. import { Avatar, Button } from "@corbits/react-ui"; -import { ArrowUp, CircleNotch, Paperclip, Stop, X } from "@corbits/icons"; +import { + ArrowUp, + CircleNotch, + Microphone, + Paperclip, + Stop, + X, +} from "@corbits/icons"; +import { reportError } from "@corbits/error-sink"; import { forwardRef, useEffect, @@ -82,6 +90,107 @@ export function insertTextAtCaret( return { text, caret: before.length + insertion.length }; } +export type SpeechRecognitionAlternativeLike = { + readonly transcript: string; +}; + +export type SpeechRecognitionResultLike = { + readonly isFinal: boolean; + readonly length: number; + readonly 0: SpeechRecognitionAlternativeLike; +}; + +export type SpeechRecognitionEventLike = { + readonly results: ArrayLike; +}; + +export type SpeechRecognitionLike = { + continuous: boolean; + interimResults: boolean; + onresult: ((event: SpeechRecognitionEventLike) => void) | null; + onerror: ((event: unknown) => void) | null; + onend: (() => void) | null; + start: () => void; + stop: () => void; + abort: () => void; +}; + +type SpeechRecognitionCtor = new () => SpeechRecognitionLike; + +function isSpeechRecognitionCtor( + value: unknown, +): value is SpeechRecognitionCtor { + return typeof value === "function"; +} + +/** Browser `SpeechRecognition` / `webkitSpeechRecognition`, or null. */ +export function speechRecognitionConstructor( + global: object = globalThis, +): SpeechRecognitionCtor | null { + if ("SpeechRecognition" in global) { + const ctor = Reflect.get(global, "SpeechRecognition"); + if (isSpeechRecognitionCtor(ctor)) return ctor; + } + if ("webkitSpeechRecognition" in global) { + const ctor = Reflect.get(global, "webkitSpeechRecognition"); + if (isSpeechRecognitionCtor(ctor)) return ctor; + } + return null; +} + +function speechRecognitionErrorCode(event: unknown): string | null { + if (typeof event !== "object" || event === null) return null; + if (!("error" in event)) return null; + const { error } = event; + if (typeof error !== "string") return null; + return error; +} + +function isBenignSpeechRecognitionError(code: string): boolean { + return code === "aborted" || code === "no-speech"; +} + +function detachDictation(rec: SpeechRecognitionLike) { + rec.onresult = null; + rec.onerror = null; + rec.onend = null; +} + +export function transcriptFromSpeechResults( + results: ArrayLike, +): string { + let text = ""; + for (const result of Array.from(results)) { + if (result.length === 0) continue; + text += result[0].transcript; + } + return text; +} + +/** + * Drop a recognition transcript between `prefix` and `suffix`, inserting a + * space when the join would otherwise glue two words together. + */ +export function spliceDictationTranscript( + prefix: string, + suffix: string, + transcript: string, +): { readonly text: string; readonly caret: number } { + const trimmed = transcript.trim(); + if (trimmed.length === 0) { + return { text: `${prefix}${suffix}`, caret: prefix.length }; + } + const head = + prefix.length === 0 || /\s$/u.test(prefix) + ? `${prefix}${trimmed}` + : `${prefix} ${trimmed}`; + const text = + suffix.length === 0 || /^\s/u.test(suffix) + ? `${head}${suffix}` + : `${head} ${suffix}`; + return { text, caret: head.length }; +} + /** * The B2 fix, isolated as a pure rule: a successful send clears the draft; * a failed one keeps exactly what the user had typed so nothing is lost. @@ -418,11 +527,108 @@ export const Composer = forwardRef< // instant a send starts, synchronously ahead of any render, so a second // call in the same tick is turned away (CL-7198). const sendInFlightRef = useRef(false); + const recognitionRef = useRef(null); + const dictatePrefixRef = useRef(""); + const dictateSuffixRef = useRef(""); + const [listening, setListening] = useState(false); + const [dictateAvailable] = useState( + () => speechRecognitionConstructor() !== null, + ); + + function stopDictation() { + const rec = recognitionRef.current; + if (rec === null) return; + setListening(false); + rec.stop(); + } + + function abortDictation() { + const rec = recognitionRef.current; + if (rec === null) return; + recognitionRef.current = null; + setListening(false); + detachDictation(rec); + try { + rec.abort(); + } catch { + // report-error-ignore: abort() after user Stop is InvalidStateError + // once recognition has already ended. + } + } + + function applyDictationTranscript(transcript: string) { + const next = spliceDictationTranscript( + dictatePrefixRef.current, + dictateSuffixRef.current, + transcript, + ); + setValue(next.text); + syncComposerSuggestState(next.text, next.caret); + requestAnimationFrame(() => { + textareaRef.current?.setSelectionRange(next.caret, next.caret); + }); + } + + function startDictation() { + abortDictation(); + const Ctor = speechRecognitionConstructor(); + if (Ctor === null) return; + const textarea = textareaRef.current; + const caret = textarea?.selectionStart ?? value.length; + dictatePrefixRef.current = value.slice(0, caret); + dictateSuffixRef.current = value.slice(caret); + const rec = new Ctor(); + rec.continuous = true; + rec.interimResults = true; + rec.onresult = (event) => { + applyDictationTranscript(transcriptFromSpeechResults(event.results)); + }; + rec.onerror = (event) => { + const code = speechRecognitionErrorCode(event); + if (code === null || !isBenignSpeechRecognitionError(code)) { + reportError(event, { operation: "composer.dictate" }); + } + if (recognitionRef.current === rec) { + recognitionRef.current = null; + setListening(false); + } + }; + rec.onend = () => { + if (recognitionRef.current !== rec) return; + setListening(false); + }; + recognitionRef.current = rec; + try { + rec.start(); + setListening(true); + } catch (cause) { + reportError(cause, { operation: "composer.dictate.start" }); + recognitionRef.current = null; + } + } + + function toggleDictation() { + if (listening) { + stopDictation(); + return; + } + startDictation(); + } useEffect(() => { if (!running) setStopping(false); }, [running]); + useEffect(() => { + return () => { + const rec = recognitionRef.current; + if (rec === null) return; + detachDictation(rec); + recognitionRef.current = null; + rec.abort(); + }; + }, []); + /** Auto-grow: the textarea reports its own content height, so the * measurement resets to the CSS-declared min-height before reading * `scrollHeight` — otherwise a shrinking draft would get stuck at its @@ -454,6 +660,7 @@ export const Composer = forwardRef< ref, () => ({ insertText: (text: string) => { + abortDictation(); const textarea = textareaRef.current; const caret = textarea?.selectionStart ?? value.length; const result = insertTextAtCaret(value, caret, text); @@ -464,6 +671,7 @@ export const Composer = forwardRef< }); }, setText: (text: string) => { + abortDictation(); attachGenerationRef.current += 1; setValue(text); setMention(null); @@ -675,6 +883,7 @@ export const Composer = forwardRef< if (!canSendComposerAction(value, attachments, { sending, preparing })) { return; } + abortDictation(); const payload: ComposerSendPayload = pendingInvites.length > 0 ? { text: value, attachments, invite: pendingInvites } @@ -949,6 +1158,7 @@ export const Composer = forwardRef< onKeyDown={handleKeyDown} onFocus={() => setFocused(true)} onBlur={() => setFocused(false)} + readOnly={listening} rows={1} />
@@ -971,6 +1181,30 @@ export const Composer = forwardRef< {CHAT_STRINGS.composerKeyboardHint}
+ {dictateAvailable ? ( + + ) : null} {canStopComposer({ running }) ? (