diff --git a/CHANGELOG.md b/CHANGELOG.md index 0553705..e0b84af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,12 +6,12 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and ## [Unreleased] -### Security - -- Update `zip` and `lettre` to fix archive-reading and SMTP security advisories. - ### Fixed +- Failed and stopped Ask AI answers save their final status before notifying clients, so reloading an ended stream cannot leave the answer busy. +- Audio pipeline progress keeps updating after temporary network failures, and interrupted Ask AI answers reconnect without duplicating text. +- Transcript searches refresh after re-transcription, and browser transcription progress and notifications follow the interface language. + - A failure to open or lock the Mac app's instance-lock file now shows a startup error instead of silently exiting as if another copy were running. - Launch at Login keeps its tray check mark in sync after an interface-language change rebuilds the menu. - Changing the Mac app's locale through desktop preferences no longer hangs while rebuilding the tray menu. @@ -19,6 +19,10 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and - The Mac app grants its own page permission to save an interface-language choice through the desktop bridge. - The Mac app builds its bundled ffmpeg, ffprobe and their C dependencies for the app's minimum macOS version, and rejects sidecars that require a newer system. +### Security + +- Update `zip` and `lettre` to fix archive-reading and SMTP security advisories. + ## [1.6.0] - 2026-11-06 ### Added diff --git a/contract-tests/tests/recording-recovery-ui.spec.ts b/contract-tests/tests/recording-recovery-ui.spec.ts new file mode 100644 index 0000000..fc09fe5 --- /dev/null +++ b/contract-tests/tests/recording-recovery-ui.spec.ts @@ -0,0 +1,97 @@ +import { type Browser, expect, test } from "@playwright/test"; +import { configureAccount, newUserContext, uploadRecording } from "../helpers/api"; + +async function recordingPage(browser: Browser) { + const api = await newUserContext("recovery-ui"); + await configureAccount(api); + const id = await uploadRecording(api); + const saved = await api.post(`/api/recordings/${id}/transcription/from-browser`, { + data: { text: "Words for the recording", model: "whisper-base" }, + }); + expect(saved.status()).toBe(200); + const initialChat = await (await api.get(`/api/recordings/${id}/chat`)).json(); + const context = await browser.newContext({ + storageState: await api.storageState(), + baseURL: process.env.OAH_BASE_URL, + viewport: { width: 1440, height: 900 }, + }); + return { id, initialChat, page: await context.newPage(), async dispose() { await context.close(); await api.dispose(); } }; +} + +test("active pipeline polling recovers from a failed poll and stops on completion", async ({ browser }) => { + const account = await recordingPage(browser); + try { + let polls = 0; + await account.page.route(`**/api/recordings/${account.id}/audio-pipeline`, async (route) => { + polls++; + if (polls === 2) return route.abort("failed"); + await route.fulfill({ json: { + job: { id: "job", status: polls >= 4 ? "completed" : "running", phase: "transcribe", progress: polls >= 4 ? 100 : 25 }, + timeline: null, timestampSource: null, transcription: null, + } }); + }); + await account.page.goto(`/recordings/${account.id}`); + await expect(account.page.getByTestId("pipeline-status")).toBeVisible(); + await expect.poll(() => polls, { timeout: 20_000 }).toBe(4); + await expect(account.page.getByTestId("pipeline-status")).toHaveCount(0); + await account.page.clock.install(); + await account.page.clock.fastForward(60_000); + expect(polls).toBe(4); + } finally { + await account.dispose(); + } +}); + +for (const interruption of ["clean EOF", "network error", "none", "exhausted"] as const) { + test(`chat stream recovery: ${interruption}`, async ({ browser }) => { + const account = await recordingPage(browser); + try { + let streams = 0; + let submitted = false; + let completed = false; + const turn = { + id: "recovery-turn", seq: 1, status: "running", question: "What happened?", answer: "", citations: [], partial: false, + provider: "Custom", model: "fake-chat-1", effort: { requested: null, sent: null, known: false }, error: null, + createdAt: new Date().toISOString(), finishedAt: null, + }; + await account.page.route(`**/api/recordings/${account.id}/chat**`, async (route) => { + const path = new URL(route.request().url()).pathname; + if (path.endsWith("/events")) { + streams++; + if (interruption === "network error" && streams === 1) return route.abort("failed"); + if (interruption === "exhausted") return route.fulfill({ contentType: "text/event-stream", body: "" }); + if (interruption === "clean EOF" && streams === 1) { + return route.fulfill({ contentType: "text/event-stream", body: 'event: snapshot\ndata: {"status":"running","answer":"Hello"}\n\nevent: delta\ndata: {"text":" wor"}\n\n' }); + } + completed = true; + return route.fulfill({ contentType: "text/event-stream", body: 'event: snapshot\ndata: {"status":"running","answer":"Hello"}\n\nevent: delta\ndata: {"text":" world"}\n\nevent: done\ndata: {}\n\n' }); + } + if (path.endsWith("/turns") && route.request().method() === "POST") { + submitted = true; + return route.fulfill({ json: { turn } }); + } + if (route.request().method() === "GET" && path.endsWith("/chat")) { + const current = { ...turn, status: completed ? "completed" : "running", answer: completed ? "Hello world" : "" }; + return route.fulfill({ json: { ...account.initialChat, activeTurn: submitted && !completed ? current : null, turns: submitted ? [current] : [] } }); + } + return route.continue(); + }); + await account.page.goto(`/recordings/${account.id}`); + await account.page.getByTestId("recording-ask-ai").click(); + await account.page.getByLabel("Your question").fill(turn.question); + await account.page.getByTestId("chat-send").click(); + if (interruption === "exhausted") { + await expect(account.page.getByText("The connection to the answer was lost.")).toBeVisible({ timeout: 20_000 }); + expect(streams).toBe(5); + } else { + await expect(account.page.getByTestId("chat-turn")).toHaveAttribute("data-status", "completed"); + await expect(account.page.getByTestId("chat-turn").locator(".chat-markdown")).toHaveText("Hello world"); + expect(streams).toBe(interruption === "none" ? 1 : 2); + await expect(account.page.getByTestId("chat-stop")).toHaveCount(0); + await expect(account.page.getByTestId("chat-turn")).toHaveCount(1); + } + } finally { + await account.dispose(); + } + }); +} diff --git a/crates/oah/src/chat/run.rs b/crates/oah/src/chat/run.rs index 20d7071..9070a72 100644 --- a/crates/oah/src/chat/run.rs +++ b/crates/oah/src/chat/run.rs @@ -125,7 +125,9 @@ pub fn spawn(state: AppState, job: Job) { "This answer took longer than this app allows.", ), }; - save(&state, &job, outcome, hub).await; + if let Err(error) = save(&state, &job, outcome, hub).await { + tracing::error!(turn_id, %error, "saving chat outcome failed"); + } super::hub::unregister(&turn_id); unregister(&turn_id); }); @@ -580,13 +582,17 @@ fn answer_messages(job: &Job, material: &Material) -> Value { } /// Saves how a run ended. -async fn save(state: &AppState, job: &Job, outcome: Outcome, hub: Arc) { +async fn save( + state: &AppState, + job: &Job, + outcome: Outcome, + hub: Arc, +) -> Result<(), sqlx::Error> { let key = &state.config.field_key; let turn_id = &job.turn_id; if outcome.stopped { - hub.send(super::hub::HubEvent::State(TurnState::Cancelled)); // A stop keeps the text written so far and is final, whichever run asks. - let _ = store::end_run( + store::end_run( &state.db, key, None, @@ -595,10 +601,11 @@ async fn save(state: &AppState, job: &Job, outcome: Outcome, hub: Arc bool { #[cfg(test)] mod tests { use super::*; + use std::future::Future; + use std::task::Poll; + + async fn pending_turn() -> (AppState, Job, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let state = AppState::new( + crate::Config::for_tests(dir.path()), + crate::db::open_memory().await.unwrap(), + ); + sqlx::raw_sql( + "INSERT INTO users (id, email, created_at, updated_at) VALUES ('u', 'u@test', 0, 0); + INSERT INTO recordings (id, user_id, device_sn, plaud_file_id, filename, duration, + start_time, end_time, filesize, file_md5, storage_type, storage_path, plaud_version, + created_at, updated_at) + VALUES ('r', 'u', 'd', 'p', 'name', 0, 0, 0, 0, '', 'local', 'r', '1', 0, 0); + INSERT INTO recording_chats (id, recording_id, user_id, created_at, updated_at) + VALUES ('chat', 'r', 'u', 0, 0); + INSERT INTO recording_chat_turns (id, chat_id, recording_id, user_id, request_id, seq, + status, generation, question, revision, created_at, updated_at) + VALUES ('turn', 'chat', 'r', 'u', 'request', 1, 'running', 0, 'question', 'rev', 0, 0);", + ) + .execute(&state.db) + .await + .unwrap(); + let job = Job { + recording_id: "r".into(), + user_id: "u".into(), + turn_id: "turn".into(), + generation: 0, + attempt: 1, + revision: Revision { + hash: "rev".into(), + transcript_id: "tr".into(), + source: "openaudiohub".into(), + timing: super::super::revision::Timing::None, + }, + question: "question".into(), + history: Vec::new(), + provider: crate::ai::Provider::default(), + effort_choice: EffortChoice::Default, + effort: EffortDecision::Default, + units: Vec::new(), + whole: false, + }; + (state, job, dir) + } + + #[tokio::test] + async fn terminal_events_follow_the_saved_outcome() { + for (status, code) in [ + (TurnState::Failed, Some("AI_PROVIDER_ERROR")), + (TurnState::Interrupted, Some("CHAT_INTERRUPTED")), + (TurnState::Cancelled, None), + ] { + let (state, job, _dir) = pending_turn().await; + let hub = Arc::new(super::super::hub::TurnHub::new(&job.turn_id)); + let mut events = hub.subscribe(); + let outcome = Outcome { + answer: String::new(), + partial: false, + failure: code.map(|code| (code, "failed".into())), + stopped: status == TurnState::Cancelled, + }; + let connection = state.db.acquire().await.unwrap(); + let saving = save(&state, &job, outcome, hub.clone()); + tokio::pin!(saving); + std::future::poll_fn(|cx| { + assert!(saving.as_mut().poll(cx).is_pending()); + Poll::Ready(()) + }) + .await; + assert!(matches!( + events.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + assert!(!hub.is_finished()); + drop(connection); + saving.await.unwrap(); + assert!( + matches!(events.recv().await.unwrap(), super::super::hub::HubEvent::State(saved) if saved == status) + ); + let saved: (String, Option) = sqlx::query_as( + "SELECT status, error_code FROM recording_chat_turns WHERE id = 'turn'", + ) + .fetch_one(&state.db) + .await + .unwrap(); + assert_eq!(saved, (status.as_str().into(), code.map(str::to_owned))); + assert!(matches!( + events.recv().await.unwrap(), + super::super::hub::HubEvent::Done + )); + } + } + + #[tokio::test] + async fn a_failed_save_does_not_announce_a_terminal_outcome() { + let (state, job, _dir) = pending_turn().await; + let hub = Arc::new(super::super::hub::TurnHub::new(&job.turn_id)); + let mut events = hub.subscribe(); + state.db.close().await; + assert!( + save( + &state, + &job, + Outcome::failed("AI_PROVIDER_ERROR", "failed"), + hub.clone() + ) + .await + .is_err() + ); + assert!(!hub.is_finished()); + assert!(matches!( + events.try_recv(), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + )); + } #[test] fn a_whole_recording_question_is_recognised() { diff --git a/spec/recording-chat.md b/spec/recording-chat.md index 77da3c2..ea12d4d 100644 --- a/spec/recording-chat.md +++ b/spec/recording-chat.md @@ -104,6 +104,8 @@ Server-sent events for one turn. Auth: session. The response is `private, no-sto - Live tokens and deltas stream directly through an in-process broadcast channel as they arrive from the provider, with periodic database checkpoints for reconnection resilience. - When an upstream provider answers with a complete JSON body instead of an event stream, the content is parsed and delivered as a completed answer. - `version` grows with each saved change. A client that loses events reconnects and reads the new snapshot instead of asking the provider again. +- The web client retries network failures and an EOF before a terminal event up to five total connection attempts, waiting 500, 1000, 2000 and 4000 milliseconds between attempts. A snapshot replaces the displayed answer before new deltas are appended. After the last failure it reports "The connection to the answer was lost." Closing the view cancels pending retries; a final snapshot, final state or `done` ends the watch. +- Terminal state events and `done` are broadcast only after the final turn is persisted, so a client reloading on either observes a final status. A persistence error is logged rather than announcing an unsaved outcome. - Opening the stream never starts a provider call. A finished turn answers a single snapshot and `done`. - The stream ends after `done`, after 15 minutes without a change to the turn, or when the client disconnects. diff --git a/spec/transcription.md b/spec/transcription.md index fe9ba46..6c2b66a 100644 --- a/spec/transcription.md +++ b/spec/transcription.md @@ -219,6 +219,7 @@ Response: ``` Poll `GET /api/recordings/{id}/audio-pipeline` until the job reaches a final state. See `audio-pipeline.md`. + The web client polls active jobs every two seconds. Failed polls double the delay up to 30 seconds; a successful poll resets it to two seconds. Polling stops when the job is inactive or the view unmounts. Errors: diff --git a/web/src/components/recording/browser-transcribe.ts b/web/src/components/recording/browser-transcribe.ts index 6a8cf42..1c08029 100644 --- a/web/src/components/recording/browser-transcribe.ts +++ b/web/src/components/recording/browser-transcribe.ts @@ -2,17 +2,19 @@ import { useCallback, useSyncExternalStore } from "react"; import { toast } from "sonner"; import { api } from "@/lib/api"; import { type BrowserAsrStage, transcribeRecordingInBrowser } from "@/lib/browser-asr"; +import { t } from "@/lib/i18n"; +import { RECORDINGS_CHANGED } from "@/lib/recordings"; function stageLabel(s: BrowserAsrStage): string { switch (s.stage) { case "decode": - return "Decoding audio…"; + return t("recording.browser.decode"); case "model": - return `Loading model… ${s.percent}%`; + return t("recording.browser.model", { percent: s.percent }); case "transcribe": - return "Transcribing…"; + return t("recording.browser.transcribe"); case "save": - return "Saving…"; + return t("recording.browser.save"); } } @@ -55,7 +57,7 @@ export async function transcribeInBrowser( if (inFlight.has(recordingId)) return; const first = stageLabel({ stage: "decode" }); setInFlight(recordingId, first); - const toastId = toast.loading("Transcribing in browser…"); + const toastId = toast.loading(t("recording.browser.loading")); try { const language = await defaultLanguage(); await transcribeRecordingInBrowser(recordingId, language, (s) => { @@ -63,10 +65,11 @@ export async function transcribeInBrowser( setInFlight(recordingId, label); onStage(label); }); - toast.success("Transcribed in browser", { id: toastId }); + toast.success(t("recording.browser.success"), { id: toastId }); + window.dispatchEvent(new Event(RECORDINGS_CHANGED)); onDone(); } catch (error) { - const message = error instanceof Error && error.message ? error.message : "Browser transcription failed"; + const message = error instanceof Error && error.message ? error.message : t("recording.browser.failed"); toast.error(message, { id: toastId }); } finally { setInFlight(recordingId, null); diff --git a/web/src/components/recording/chat/events.ts b/web/src/components/recording/chat/events.ts index 2a90ad6..94f6ebd 100644 --- a/web/src/components/recording/chat/events.ts +++ b/web/src/components/recording/chat/events.ts @@ -4,7 +4,7 @@ * The first event is the whole turn and later events are changes, so a stream that drops is * reopened and the answer it already had is kept: nothing here asks a provider for anything. */ -import { chatEventsPath, type ChatTurn, type ChatTurnEvent } from "./types"; +import { chatEventsPath, isFinalStatus, type ChatTurn, type ChatTurnEvent } from "./types"; export interface StreamHandlers { onSnapshot(event: Extract): void; @@ -23,18 +23,42 @@ export function openTurnStream(recordingId: string, turnId: string, handlers: St } async function read(recordingId: string, turnId: string, handlers: StreamHandlers, signal: AbortSignal): Promise { - let closed = false; - try { - const response = await fetch(chatEventsPath(recordingId, turnId), { - credentials: "same-origin", - headers: { accept: "text/event-stream" }, - signal, - }); - if (!response.ok || !response.body) { - handlers.onError(`Request failed (${response.status})`); + for (let attempt = 0; attempt < 5 && !signal.aborted; attempt++) { + try { + await readOnce(recordingId, turnId, handlers, signal); return; + } catch { + if (signal.aborted) return; + if (attempt === 4) { + handlers.onError("The connection to the answer was lost."); + return; + } + await new Promise((resolve) => { + const finish = () => { + clearTimeout(timer); + signal.removeEventListener("abort", finish); + resolve(); + }; + const timer = setTimeout(finish, Math.min(500 * 2 ** attempt, 8000)); + signal.addEventListener("abort", finish, { once: true }); + }); } - const reader = response.body.getReader(); + } +} + +async function readOnce(recordingId: string, turnId: string, handlers: StreamHandlers, signal: AbortSignal): Promise { + let closed = false; + const response = await fetch(chatEventsPath(recordingId, turnId), { + credentials: "same-origin", + headers: { accept: "text/event-stream" }, + signal, + }); + if (!response.ok || !response.body) { + handlers.onError(`Request failed (${response.status})`); + return; + } + const reader = response.body.getReader(); + try { const decoder = new TextDecoder(); let buffer = ""; for (;;) { @@ -46,18 +70,20 @@ async function read(recordingId: string, turnId: string, handlers: StreamHandler while (boundary >= 0) { const frame = buffer.slice(0, boundary); buffer = buffer.slice(boundary + 2); - if (handle(frame, handlers)) closed = true; + if (handle(frame, handlers)) { + closed = true; + break; + } boundary = buffer.indexOf("\n\n"); } if (closed) break; } if (!closed && buffer.trim()) { - handle(buffer, handlers); + closed = handle(buffer, handlers); } + if (!closed) throw new Error("Incomplete answer stream"); + } finally { reader.cancel().catch(() => undefined); - } catch (err) { - if (signal.aborted) return; - handlers.onError(err instanceof Error ? err.message : "The connection to the answer was lost."); } } @@ -86,12 +112,20 @@ function handle(frame: string, handlers: StreamHandlers): boolean { provider: nullableString(payload.provider), error: errorOf(payload.error), }); + if (isFinalStatus(String(payload.status ?? ""))) { + handlers.onDone(); + return true; + } break; case "delta": handlers.onDelta({ type: "delta", version, text: String(payload.text ?? "") }); break; case "state": handlers.onState({ type: "state", version, status: String(payload.status ?? "") }); + if (isFinalStatus(String(payload.status ?? ""))) { + handlers.onDone(); + return true; + } break; case "citations": handlers.onCitations({ type: "citations", version, citations: Array.isArray(payload.citations) ? payload.citations : [] }); @@ -101,7 +135,7 @@ function handle(frame: string, handlers: StreamHandlers): boolean { return true; case "error": handlers.onError(String(payload.message ?? "The answer failed.")); - break; + return true; default: break; } diff --git a/web/src/components/recording/data.ts b/web/src/components/recording/data.ts index ad9fe12..04a4ba7 100644 --- a/web/src/components/recording/data.ts +++ b/web/src/components/recording/data.ts @@ -91,9 +91,22 @@ export function usePipelineWatch(id: string, onSettled: () => void) { useEffect(() => { if (!active) return; - const timer = window.setTimeout(() => void refresh(), 2000); - return () => window.clearTimeout(timer); - }, [active, pipeline, refresh]); + let stopped = false; + let delay = 2000; + let timer: number; + const poll = async () => { + const next = await refresh(); + if (stopped) return; + if (next?.job && !isActiveStatus(next.job.status)) return; + delay = next ? 2000 : Math.min(delay * 2, 30000); + timer = window.setTimeout(() => void poll(), delay); + }; + timer = window.setTimeout(() => void poll(), delay); + return () => { + stopped = true; + window.clearTimeout(timer); + }; + }, [active, refresh]); const queue = useCallback(() => { setQueuedLocally(true); diff --git a/web/src/components/recordings/hooks.ts b/web/src/components/recordings/hooks.ts index c41f0e8..ef49e06 100644 --- a/web/src/components/recordings/hooks.ts +++ b/web/src/components/recordings/hooks.ts @@ -126,7 +126,7 @@ export function useTranscriptSearch(items: Recording[], needle: string): { // A search is about the library as it is now: a recording that was just transcribed, renamed // or deleted has to be asked about again, which is what this signature changes on. const library = items - .map((r) => `${r.id}:${r.hasTranscript ? 1 : 0}:${r.filename}`) + .map((r) => `${r.id}:${r.hasTranscript ? 1 : 0}:${r.filename}:${r.updatedAt}`) .sort() .join("|"); diff --git a/web/src/locales/en/recording.json b/web/src/locales/en/recording.json index b22e46d..944195c 100644 --- a/web/src/locales/en/recording.json +++ b/web/src/locales/en/recording.json @@ -73,6 +73,13 @@ "defaultModel": "the provider's default model" }, "browser": { + "decode": "Decoding audio…", + "model": "Loading model… {percent}%", + "transcribe": "Transcribing…", + "save": "Saving…", + "loading": "Transcribing in browser…", + "success": "Transcribed in browser", + "failed": "Browser transcription failed", "action": "Transcribe in browser", "title": "Run Whisper right here in your browser. No API key needed, and the audio never leaves your device." }, diff --git a/web/src/locales/ja/recording.json b/web/src/locales/ja/recording.json index 5ce9705..9048795 100644 --- a/web/src/locales/ja/recording.json +++ b/web/src/locales/ja/recording.json @@ -73,6 +73,13 @@ "defaultModel": "プロバイダーの既定モデル" }, "browser": { + "decode": "音声をデコード中…", + "model": "モデルを読み込み中… {percent}%", + "transcribe": "文字起こし中…", + "save": "保存中…", + "loading": "ブラウザーで文字起こし中…", + "success": "ブラウザーでの文字起こしが完了しました", + "failed": "ブラウザーでの文字起こしに失敗しました", "action": "ブラウザーで文字起こし", "title": "ブラウザー上で Whisper を実行します。API キーは不要で、音声がデバイスの外に出ることはありません。" }, diff --git a/web/src/locales/ko/recording.json b/web/src/locales/ko/recording.json index d8e9f05..5135e3a 100644 --- a/web/src/locales/ko/recording.json +++ b/web/src/locales/ko/recording.json @@ -73,6 +73,13 @@ "defaultModel": "제공자의 기본 모델" }, "browser": { + "decode": "오디오 디코딩 중…", + "model": "모델 로딩 중… {percent}%", + "transcribe": "스크립트 작성 중…", + "save": "저장 중…", + "loading": "브라우저에서 스크립트 작성 중…", + "success": "브라우저에서 스크립트 작성 완료", + "failed": "브라우저 스크립트 작성 실패", "action": "브라우저에서 스크립트 작성", "title": "브라우저에서 바로 Whisper를 실행합니다. API 키가 필요 없고 오디오는 기기를 떠나지 않습니다." }, diff --git a/web/src/locales/zh-CN/recording.json b/web/src/locales/zh-CN/recording.json index 97972ff..67af91a 100644 --- a/web/src/locales/zh-CN/recording.json +++ b/web/src/locales/zh-CN/recording.json @@ -73,6 +73,13 @@ "defaultModel": "服务商的默认模型" }, "browser": { + "decode": "正在解码音频…", + "model": "正在加载模型… {percent}%", + "transcribe": "正在转写…", + "save": "正在保存…", + "loading": "正在浏览器中转写…", + "success": "已在浏览器中完成转写", + "failed": "浏览器转写失败", "action": "在浏览器中转写", "title": "直接在浏览器中运行 Whisper。无需 API 密钥,音频不会离开你的设备。" },