From dfb0521f2db0dd4e837cbbf60782fd0a19a1a1d0 Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 22:39:04 -0700 Subject: [PATCH 1/6] Add tests for composer dictate (CL-6424) --- packages/chat-ui/src/composer.test.tsx | 73 +++++++++++++++- packages/chat-ui/test/composer.test.tsx | 110 ++++++++++++++++++++++++ 2 files changed, 182 insertions(+), 1 deletion(-) diff --git a/packages/chat-ui/src/composer.test.tsx b/packages/chat-ui/src/composer.test.tsx index e705121b10..44aeec1ccc 100644 --- a/packages/chat-ui/src/composer.test.tsx +++ b/packages/chat-ui/src/composer.test.tsx @@ -1,6 +1,12 @@ import { describe, expect, test } from "bun:test"; -import { canStopComposer, insertTextAtCaret } from "./composer"; +import { + canStopComposer, + insertTextAtCaret, + spliceDictationTranscript, + speechRecognitionConstructor, + transcriptFromSpeechResults, +} from "./composer"; import { isAwaitingReply } from "./streaming-reply"; describe("insertTextAtCaret", () => { @@ -23,6 +29,71 @@ describe("insertTextAtCaret", () => { }); }); +describe("spliceDictationTranscript", () => { + test("inserts a space so two words do not glue together", () => { + const result = spliceDictationTranscript("hello", "", "world"); + expect(result.text).toBe("hello world"); + expect(result.caret).toBe(11); + }); + + test("does not add a second space when the prefix already ends with one", () => { + const result = spliceDictationTranscript("hello ", "", "world"); + expect(result.text).toBe("hello world"); + expect(result.caret).toBe(11); + }); + + test("keeps a following suffix on its own word boundary", () => { + const result = spliceDictationTranscript("hello", "there", "world"); + expect(result.text).toBe("hello world there"); + expect(result.caret).toBe(11); + }); + + test("drops an empty transcript without changing the draft", () => { + const result = spliceDictationTranscript("hello", " there", " "); + expect(result.text).toBe("hello there"); + expect(result.caret).toBe(5); + }); +}); + +describe("transcriptFromSpeechResults", () => { + test("concatenates every alternative in order", () => { + expect( + transcriptFromSpeechResults([ + { isFinal: true, length: 1, 0: { transcript: "hello " } }, + { isFinal: false, length: 1, 0: { transcript: "world" } }, + ]), + ).toBe("hello world"); + }); +}); + +describe("speechRecognitionConstructor", () => { + test("returns null when the browser has no speech recognition", () => { + expect(speechRecognitionConstructor({})).toBeNull(); + }); + + test("prefers SpeechRecognition over the webkit prefix", () => { + function SpeechRecognition() { + return undefined; + } + function webkitSpeechRecognition() { + return undefined; + } + const found = speechRecognitionConstructor({ + SpeechRecognition, + webkitSpeechRecognition, + }); + expect(Object.is(found, SpeechRecognition)).toBe(true); + }); + + test("falls back to webkitSpeechRecognition", () => { + function webkitSpeechRecognition() { + return undefined; + } + const found = speechRecognitionConstructor({ webkitSpeechRecognition }); + expect(Object.is(found, webkitSpeechRecognition)).toBe(true); + }); +}); + // CL-7201: the composer's stop affordance is a stand-in for "is there a // turn to cancel" — offered whenever the host says a turn is running, // independent of the composer's own `sending`/`preparing` state (queuing diff --git a/packages/chat-ui/test/composer.test.tsx b/packages/chat-ui/test/composer.test.tsx index 3e8a7acc7e..d5dfaa2176 100644 --- a/packages/chat-ui/test/composer.test.tsx +++ b/packages/chat-ui/test/composer.test.tsx @@ -850,3 +850,113 @@ describe("Composer stop affordance (CL-7201)", () => { expect(stopButton().hasAttribute("disabled")).toBe(false); }); }); + +describe("Composer dictate", () => { + const recognitions: FakeSpeechRecognition[] = []; + + class FakeSpeechRecognition { + continuous = false; + interimResults = false; + onresult: ((event: { results: FakeSpeechResult[] }) => void) | null = null; + onerror: ((event: unknown) => void) | null = null; + onend: (() => void) | null = null; + started = false; + + constructor() { + recognitions.push(this); + } + + start() { + this.started = true; + } + + stop() { + this.started = false; + this.onend?.(); + } + + abort() { + this.started = false; + this.onend?.(); + } + + emit(segments: readonly { isFinal: boolean; transcript: string }[]) { + const results = segments.map((segment) => ({ + isFinal: segment.isFinal, + length: 1, + 0: { transcript: segment.transcript }, + })); + this.onresult?.({ results }); + } + } + + type FakeSpeechResult = { + readonly isFinal: boolean; + readonly length: number; + readonly 0: { readonly transcript: string }; + }; + + function installSpeechRecognition() { + Object.defineProperty(window, "SpeechRecognition", { + configurable: true, + writable: true, + value: FakeSpeechRecognition, + }); + } + + afterEach(() => { + recognitions.length = 0; + Reflect.deleteProperty(window, "SpeechRecognition"); + }); + + test("hides the dictate control when speech recognition is unavailable", () => { + mount(() => Promise.resolve(true)); + expect(container?.querySelector('[aria-label="Dictate"]')).toBeNull(); + }); + + test("starts recognition, inserts a transcript on a word boundary, and stops", async () => { + installSpeechRecognition(); + mount(() => Promise.resolve(true)); + typeInto(textarea(), "hello"); + textarea().setSelectionRange(5, 5); + await settle(); + + const dictate = container?.querySelector( + '[aria-label="Dictate"]', + ); + if (dictate === null || dictate === undefined) { + throw new Error("dictate button not found"); + } + + act(() => { + dictate.click(); + }); + await settle(); + + expect(recognitions.at(-1)?.started).toBe(true); + expect(recognitions.at(-1)?.continuous).toBe(true); + expect(recognitions.at(-1)?.interimResults).toBe(true); + expect(dictate.getAttribute("aria-label")).toBe("Stop dictating"); + expect(dictate.getAttribute("aria-pressed")).toBe("true"); + expect(dictate.getAttribute("data-listening")).toBe("true"); + + act(() => { + recognitions.at(-1)?.emit([{ isFinal: true, transcript: "world" }]); + }); + await settle(); + + expect(textarea().value).toBe("hello world"); + + act(() => { + dictate.click(); + }); + await settle(); + + expect(recognitions.at(-1)?.started).toBe(false); + const idle = container?.querySelector( + '[aria-label="Dictate"]', + ); + expect(idle?.getAttribute("aria-pressed")).toBe("false"); + expect(idle?.getAttribute("data-listening")).toBe("false"); + }); +}); From b663154bae142044bdc6738b43996d8f698bb28d Mon Sep 17 00:00:00 2001 From: Sawyer Date: Thu, 3 Sep 2026 22:39:10 -0700 Subject: [PATCH 2/6] Add a dictate control to the composer (CL-6424) The mic starts and stops the browser SpeechRecognition API and inserts the transcript at the caret on a word boundary. The control is omitted when the browser has no recognition constructor. --- packages/chat-ui/src/composer.tsx | 199 +++++++++++++++++++++++++++++- packages/chat-ui/src/styles.css | 9 ++ 2 files changed, 207 insertions(+), 1 deletion(-) diff --git a/packages/chat-ui/src/composer.tsx b/packages/chat-ui/src/composer.tsx index e6ea463820..4a92603ce2 100644 --- a/packages/chat-ui/src/composer.tsx +++ b/packages/chat-ui/src/composer.tsx @@ -7,7 +7,15 @@ // does not compose with an inline mention popover. import { Avatar, Button } from "@corbits/react-ui"; -import { ArrowUp, CircleNotch, Paperclip, Stop, X } from "@corbits/icons"; +import { + ArrowUp, + CircleNotch, + Microphone, + Paperclip, + Stop, + X, +} from "@corbits/icons"; +import { reportError } from "@corbits/error-sink"; import { forwardRef, useEffect, @@ -82,6 +90,89 @@ export function insertTextAtCaret( return { text, caret: before.length + insertion.length }; } +export type SpeechRecognitionAlternativeLike = { + readonly transcript: string; +}; + +export type SpeechRecognitionResultLike = { + readonly isFinal: boolean; + readonly length: number; + readonly 0: SpeechRecognitionAlternativeLike; +}; + +export type SpeechRecognitionEventLike = { + readonly results: ArrayLike; +}; + +export type SpeechRecognitionLike = { + continuous: boolean; + interimResults: boolean; + onresult: ((event: SpeechRecognitionEventLike) => void) | null; + onerror: ((event: unknown) => void) | null; + onend: (() => void) | null; + start: () => void; + stop: () => void; + abort: () => void; +}; + +type SpeechRecognitionCtor = new () => SpeechRecognitionLike; + +function isSpeechRecognitionCtor( + value: unknown, +): value is SpeechRecognitionCtor { + return typeof value === "function"; +} + +/** Browser `SpeechRecognition` / `webkitSpeechRecognition`, or null. */ +export function speechRecognitionConstructor( + global: object = globalThis, +): SpeechRecognitionCtor | null { + if ("SpeechRecognition" in global) { + const ctor = Reflect.get(global, "SpeechRecognition"); + if (isSpeechRecognitionCtor(ctor)) return ctor; + } + if ("webkitSpeechRecognition" in global) { + const ctor = Reflect.get(global, "webkitSpeechRecognition"); + if (isSpeechRecognitionCtor(ctor)) return ctor; + } + return null; +} + +export function transcriptFromSpeechResults( + results: ArrayLike, +): string { + let text = ""; + for (const result of Array.from(results)) { + if (result.length === 0) continue; + text += result[0].transcript; + } + return text; +} + +/** + * Drop a recognition transcript between `prefix` and `suffix`, inserting a + * space when the join would otherwise glue two words together. + */ +export function spliceDictationTranscript( + prefix: string, + suffix: string, + transcript: string, +): { readonly text: string; readonly caret: number } { + const trimmed = transcript.trim(); + if (trimmed.length === 0) { + return { text: `${prefix}${suffix}`, caret: prefix.length }; + } + const head = + prefix.length === 0 || /\s$/u.test(prefix) + ? `${prefix}${trimmed}` + : `${prefix} ${trimmed}`; + const text = + suffix.length === 0 || /^\s/u.test(suffix) + ? `${head}${suffix}` + : `${head} ${suffix}`; + return { text, caret: head.length }; +} + /** * The B2 fix, isolated as a pure rule: a successful send clears the draft; * a failed one keeps exactly what the user had typed so nothing is lost. @@ -418,11 +509,89 @@ export const Composer = forwardRef< // instant a send starts, synchronously ahead of any render, so a second // call in the same tick is turned away (CL-7198). const sendInFlightRef = useRef(false); + const recognitionRef = useRef(null); + const dictatePrefixRef = useRef(""); + const dictateSuffixRef = useRef(""); + const [listening, setListening] = useState(false); + const [dictateAvailable] = useState( + () => speechRecognitionConstructor() !== null, + ); + + function stopDictation() { + const rec = recognitionRef.current; + if (rec === null) return; + recognitionRef.current = null; + setListening(false); + rec.stop(); + } + + function applyDictationTranscript(transcript: string) { + const next = spliceDictationTranscript( + dictatePrefixRef.current, + dictateSuffixRef.current, + transcript, + ); + setValue(next.text); + syncComposerSuggestState(next.text, next.caret); + requestAnimationFrame(() => { + textareaRef.current?.setSelectionRange(next.caret, next.caret); + }); + } + + function startDictation() { + const Ctor = speechRecognitionConstructor(); + if (Ctor === null) return; + const textarea = textareaRef.current; + const caret = textarea?.selectionStart ?? value.length; + dictatePrefixRef.current = value.slice(0, caret); + dictateSuffixRef.current = value.slice(caret); + const rec = new Ctor(); + rec.continuous = true; + rec.interimResults = true; + rec.onresult = (event) => { + applyDictationTranscript(transcriptFromSpeechResults(event.results)); + }; + rec.onerror = (event) => { + reportError(event, { operation: "composer.dictate" }); + if (recognitionRef.current === rec) { + recognitionRef.current = null; + setListening(false); + } + }; + rec.onend = () => { + if (recognitionRef.current !== rec) return; + recognitionRef.current = null; + setListening(false); + }; + recognitionRef.current = rec; + try { + rec.start(); + setListening(true); + } catch (cause) { + reportError(cause, { operation: "composer.dictate.start" }); + recognitionRef.current = null; + } + } + + function toggleDictation() { + if (listening) { + stopDictation(); + return; + } + startDictation(); + } useEffect(() => { if (!running) setStopping(false); }, [running]); + useEffect(() => { + return () => { + recognitionRef.current?.abort(); + recognitionRef.current = null; + }; + }, []); + /** Auto-grow: the textarea reports its own content height, so the * measurement resets to the CSS-declared min-height before reading * `scrollHeight` — otherwise a shrinking draft would get stuck at its @@ -454,6 +623,7 @@ export const Composer = forwardRef< ref, () => ({ insertText: (text: string) => { + stopDictation(); const textarea = textareaRef.current; const caret = textarea?.selectionStart ?? value.length; const result = insertTextAtCaret(value, caret, text); @@ -464,6 +634,7 @@ export const Composer = forwardRef< }); }, setText: (text: string) => { + stopDictation(); attachGenerationRef.current += 1; setValue(text); setMention(null); @@ -675,6 +846,7 @@ export const Composer = forwardRef< if (!canSendComposerAction(value, attachments, { sending, preparing })) { return; } + stopDictation(); const payload: ComposerSendPayload = pendingInvites.length > 0 ? { text: value, attachments, invite: pendingInvites } @@ -949,6 +1121,7 @@ export const Composer = forwardRef< onKeyDown={handleKeyDown} onFocus={() => setFocused(true)} onBlur={() => setFocused(false)} + readOnly={listening} rows={1} />
@@ -971,6 +1144,30 @@ export const Composer = forwardRef< {CHAT_STRINGS.composerKeyboardHint}
+ {dictateAvailable ? ( + + ) : null} {canStopComposer({ running }) ? (