From 87c02b53757fa918a80ccaaad77a34c968a08623 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 5 Aug 2026 09:39:59 +0000 Subject: [PATCH 1/4] Harden the backend, split the frontend, add backup and new exports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the weak spots found in the review, plus three requested features. Security - Refuse API requests from off-machine until a password is set. Loopback is unchanged (no password, no prompt), so local use is unaffected, but the documented Traefik deployment no longer serves the library, the audio and the stored Hugging Face token to anyone who finds the hostname. - Stop echoing secrets: settings report whether a token/key is stored, never its value, and an unedited field cannot overwrite a stored credential. - Add a Security panel, an API token for headless clients, and login throttling. Reliability - Requeue interrupted transcriptions on restart instead of flipping them to 'error' with no explanation; mark the unresumable ones with a reason. - Replace ad-hoc ALTER TABLE statements wrapped in `except: pass` with numbered migrations recorded in schema_version, which fail loudly. - Escape FTS5 query terms so ordinary punctuation ("covid-19", "C++") stops raising syntax errors and silently downgrading the search. - Chunk LLM analysis that exceeds the context budget, so long recordings are summarised in parts and merged rather than truncated from the front. - Prefetch URL downloads concurrently while keeping model inference serialized. - Keep a tombstone for expired jobs so their routes answer 410 with the recording id rather than a 404. Features - Export/import the whole library as a zip bundle (settings excluded). - WebVTT and CSV exports, in the web UI, bulk menu and TUI. - Optional automatic summary when a captured meeting finishes. Maintenance - Split the 4,800-line inline script into 20 ES modules under frontend/js/, loaded natively — no build step, no bundler. - Add route-level tests (128 -> 329) and make CI run the whole suite instead of ten named files. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01JmgRm98BjMyfRok7eN9L12 --- .env.example | 15 + .github/workflows/tests.yml | 9 +- CHANGELOG.md | 108 + README.md | 28 +- backend/api/routes/analyses.py | 55 +- backend/api/routes/auth.py | 118 + backend/api/routes/backup.py | 340 ++ backend/api/routes/folders_tags.py | 37 +- backend/api/routes/library.py | 28 +- backend/api/routes/llm.py | 36 +- backend/api/routes/settings.py | 29 +- backend/api/routes/transcription.py | 72 +- backend/auth.py | 306 ++ backend/core/analysis.py | 596 +++- backend/core/analysis_jobs.py | 159 + backend/core/transcription.py | 111 +- backend/db.py | 74 +- backend/exports.py | 114 + backend/main.py | 175 +- backend/migrations.py | 226 ++ backend/models/__init__.py | 9 + backend/search_query.py | 67 + backend/settings.py | 46 +- docs/ROADMAP.md | 21 +- docs/doc.md | 136 +- frontend/index.html | 4949 +-------------------------- frontend/js/analysis.js | 544 +++ frontend/js/app.js | 57 + frontend/js/auth.js | 186 + frontend/js/backup.js | 98 + frontend/js/changelog.js | 36 + frontend/js/exports.js | 56 + frontend/js/folders.js | 166 + frontend/js/http.js | 42 + frontend/js/jobs.js | 134 + frontend/js/layout.js | 60 + frontend/js/library-init.js | 271 ++ frontend/js/library.js | 607 ++++ frontend/js/main.js | 141 + frontend/js/prefs.js | 180 + frontend/js/queue.js | 155 + frontend/js/search.js | 141 + frontend/js/shortcuts.js | 60 + frontend/js/state.js | 236 ++ frontend/js/tabs.js | 89 + frontend/js/tags.js | 252 ++ frontend/js/transcript.js | 657 ++++ frontend/js/upload.js | 878 +++++ frontend/js/version.js | 83 + frontend/js/watcher.js | 219 ++ scripts/meeting_watcher/watcher.py | 46 +- tests/conftest.py | 152 +- tests/test_analysis_chunking.py | 306 ++ tests/test_api_library_routes.py | 349 ++ tests/test_auth.py | 281 ++ tests/test_auto_summary.py | 204 ++ tests/test_backup_bundle.py | 212 ++ tests/test_export_formats.py | 190 + tests/test_frontend_assets.py | 95 + tests/test_job_recovery.py | 318 ++ tests/test_meeting_watcher.py | 8 +- tests/test_migrations.py | 134 + tests/test_search_escaping.py | 160 +- tui/api.py | 15 +- tui/commands.py | 12 +- tui/screens/transcript.py | 3 +- 66 files changed, 10540 insertions(+), 5157 deletions(-) create mode 100644 backend/api/routes/auth.py create mode 100644 backend/api/routes/backup.py create mode 100644 backend/auth.py create mode 100644 backend/core/analysis_jobs.py create mode 100644 backend/migrations.py create mode 100644 backend/search_query.py create mode 100644 frontend/js/analysis.js create mode 100644 frontend/js/app.js create mode 100644 frontend/js/auth.js create mode 100644 frontend/js/backup.js create mode 100644 frontend/js/changelog.js create mode 100644 frontend/js/exports.js create mode 100644 frontend/js/folders.js create mode 100644 frontend/js/http.js create mode 100644 frontend/js/jobs.js create mode 100644 frontend/js/layout.js create mode 100644 frontend/js/library-init.js create mode 100644 frontend/js/library.js create mode 100644 frontend/js/main.js create mode 100644 frontend/js/prefs.js create mode 100644 frontend/js/queue.js create mode 100644 frontend/js/search.js create mode 100644 frontend/js/shortcuts.js create mode 100644 frontend/js/state.js create mode 100644 frontend/js/tabs.js create mode 100644 frontend/js/tags.js create mode 100644 frontend/js/transcript.js create mode 100644 frontend/js/upload.js create mode 100644 frontend/js/version.js create mode 100644 frontend/js/watcher.js create mode 100644 tests/test_analysis_chunking.py create mode 100644 tests/test_api_library_routes.py create mode 100644 tests/test_auth.py create mode 100644 tests/test_auto_summary.py create mode 100644 tests/test_backup_bundle.py create mode 100644 tests/test_export_formats.py create mode 100644 tests/test_job_recovery.py create mode 100644 tests/test_migrations.py diff --git a/.env.example b/.env.example index 55308b7..c754c43 100644 --- a/.env.example +++ b/.env.example @@ -9,3 +9,18 @@ TRAEFIK_CERTRESOLVER=le # Hugging Face token for speaker diarization (optional) HF_TOKEN= + +# --- Access control ------------------------------------------------------- +# AmicoScript refuses requests from outside the host machine until a password +# is set. Set one here, or from the app's Security panel on the host itself. +# Required for any deployment reachable from a network. +AMICOSCRIPT_PASSWORD= + +# auto (default) — loopback is trusted, the network needs a session +# always — every request needs a session, including from this machine +# off — no authentication at all (only when something else guards it) +AMICOSCRIPT_AUTH=auto + +# Token for headless clients (TUI, meeting watcher) when AMICOSCRIPT_AUTH=always. +# Leave empty to use the one the app generates when you set a password. +AMICOSCRIPT_API_TOKEN= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7f96dfd..bbdc9ec 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,8 +25,11 @@ jobs: run: | python -m pip install --upgrade pip pip install -r backend/requirements.txt - pip install pytest + pip install -r tui/requirements.txt + pip install pytest httpx + # The whole suite, not a hand-maintained list of files. The old workflow + # named ten test files explicitly, so anything added since — including + # every route-level test — was never run by CI. - name: Run tests - run: | - pytest -q tests/test_diarization.py tests/test_audio_utils.py tests/test_transcription_model_cache.py tests/test_error_classifiers.py tests/test_transcription_flow.py tests/test_colab_proxy_flow.py tests/test_job_helpers_sync_retry.py tests/test_settings.py tests/test_meeting_watcher.py tests/test_watcher_status.py + run: pytest -q diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2c741..825d0d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,114 @@ Keep a Changelog format. ## [Unreleased] +### 🔐 Access control + +- **AmicoScript now refuses network requests until a password is set.** The + project documents a Traefik deployment on a public domain, but every API route + was open there — anyone who found the hostname could read the library, download + the audio and read the stored Hugging Face token out of `GET /api/settings`. + Requests from the machine AmicoScript runs on behave exactly as before, with no + password and no prompt; requests from anywhere else are refused with an + explanation until a password exists. Exposing the app unconfigured now fails + closed instead of silently publishing your transcripts. +- **Security panel** in the sidebar to set, change or remove the password, and to + read the API token that headless clients (the TUI, the meeting watcher) use. + `AMICOSCRIPT_PASSWORD` sets it at startup; `AMICOSCRIPT_AUTH=always` requires a + session even locally; `AMICOSCRIPT_AUTH=off` disables the layer for deployments + that put their own authentication in front. +- **Secrets are no longer echoed back to clients.** `GET /api/settings` reports + whether a Hugging Face token is stored and shows its last four characters; + `GET /api/llm/settings` reports whether an API key is set. Saving a form no + longer risks overwriting a stored credential with its own placeholder. +- Login attempts are throttled after repeated failures, and the loopback check + reads the direct peer address rather than `X-Forwarded-For`, which a caller + controls. + +### ✨ Library export and import + +- **Your library is now portable.** Export everything — recordings, transcripts, + analyses, folders and tags — as a single zip from **Backup** in the sidebar, and + import it on another machine or after a reinstall. Until now a library existed + only inside `~/.amicoscript` with no backup path and no way to move it. +- Import matches rows by id, so re-importing the same bundle is a no-op rather + than a duplicated library; `Overwrite` replaces existing rows instead. +- Bundles deliberately exclude `settings.json` — it holds your Hugging Face + token, LLM API key and password hash, none of which should travel in a file you + email to yourself. Imports reject path traversal entries and zip bombs. + +### 📤 New export formats + +- **WebVTT** (`.vtt`) — the subtitle format browsers accept in ``. + Speakers become `` voice spans rather than text baked into the caption. +- **CSV** (`.csv`) — one row per segment with both raw and human-readable + timestamps, speaker, text, translation and an edited flag. Written with a BOM + so Excel reads accents correctly, and leading `=`/`+`/`-`/`@` in transcript text + is defused so a spreadsheet cannot execute it as a formula. +- Both are available for single recordings, in the bulk-export menu, and from the + TUI's `/export` command. + +### 🤖 Long transcripts no longer get silently truncated + +- **AI analysis handles recordings larger than the model's context window.** A + one-hour meeting is roughly 12k tokens and Ollama defaults to 4096, so the + model was quietly dropping the *beginning* of the transcript and returning a + confident summary of the last few minutes. Anything over the configured budget + is now summarised in parts and merged, with progress reported per part. + Translation concatenates its parts instead of merging them, because merging + would rewrite the translation. +- The context budget is configurable under AI Analysis (default 8192 tokens). +- Analyses fall back to a non-streaming request when a server does not deliver + SSE, instead of completing with an empty result. + +### ✨ Automatic meeting summaries + +- Turn on **Summarise automatically** under Meeting auto-capture and every + finished call is summarised by your LLM without being asked. Fires only for + captured calls, only when an LLM is configured, and only once per recording. + +### 🔧 Reliability + +- **A restart no longer destroys work in progress.** Interrupted recordings used + to be flipped to `error` with no explanation — a two-hour meeting that was 90% + transcribed was simply lost. Anything whose audio is still on disk is requeued + automatically; anything that cannot be resumed is marked `interrupted` with a + reason the library shows on hover. `AMICOSCRIPT_RESUME_JOBS=0` restores the old + behaviour. +- **Finished jobs leave a tombstone** when they are evicted from memory after an + hour, so `/api/jobs/{id}/…` answers 410 with the recording id instead of a 404 + that looked like the job never existed. +- **URL imports download while the previous job is still transcribing.** Model + inference is still strictly one at a time, but fetching audio is network-bound + and no longer waits behind it — importing a playlist is roughly twice as fast. + Tune with `AMICOSCRIPT_DOWNLOAD_CONCURRENCY` (default 2). +- **Schema changes are versioned migrations.** They used to be ad-hoc `ALTER + TABLE` statements wrapped in `except: pass`, so a failed upgrade left a broken + database looking healthy. Steps are numbered, recorded in a `schema_version` + table, and fail loudly; a database from a newer build is refused rather than + guessed at. +- **Search no longer breaks on ordinary punctuation.** `covid-19`, `C++`, + `hello "world` and a bare `AND` were all FTS5 syntax errors that silently + downgraded the search to a slower, different query. Terms are now escaped + properly, quoted phrases are honoured, and the last word is treated as a prefix + so results narrow as you type. + +### 🧹 Maintenance + +- **The frontend is a set of ES modules.** `index.html` carried a single + 4,800-line ` + + + +
+ + LLM Settings + + + + +
+ +
+ + +

+ +
+ + +
+ + + +
+ + + + + +
+ + + +
+ +
+ +
+ + +
+ + +
+ +
+ +
+ + +
+
+ +
+ + +

+ Transcripts longer than this are summarised in parts and then merged, + instead of being silently truncated. Match your model's setting + (Ollama defaults to 4096). +

+
+ +
+ + +
+
+
+
@@ -1136,102 +1267,6 @@

P
- -
- - LLM Settings - - - - -
- -
- - -
- -
- -
- - -
- - -
- -
- -
- - -
-
- -
- - -

- Transcripts longer than this are summarised in parts and then merged, - instead of being silently truncated. Match your model's setting - (Ollama defaults to 4096). -

-
- -
- - -
-
-
- diff --git a/frontend/js/analysis.js b/frontend/js/analysis.js index 8f29a5e..ecded4d 100644 --- a/frontend/js/analysis.js +++ b/frontend/js/analysis.js @@ -5,6 +5,7 @@ import { state } from './state.js'; import { escHtml } from './transcript.js'; +import { currentProviderFields, showUrlNote } from './llm-setup.js'; import { clientLog } from './upload.js'; export function initAiAnalysis() { @@ -410,7 +411,7 @@ window._deleteAnalysis = async function (analysisId, recordingId) { export async function saveLlmSettings() { const fd = new FormData(); - const baseUrl = document.getElementById('llm-base-url').value.trim() || 'http://localhost:11434'; + const baseUrl = document.getElementById('llm-base-url').value.trim(); const model = document.getElementById('llm-model-input').value.trim(); fd.append('llm_base_url', baseUrl); fd.append('llm_model_name', model); @@ -422,8 +423,21 @@ export async function saveLlmSettings() { if (contextTokens && contextTokens.value.trim()) { fd.append('llm_context_tokens', contextTokens.value.trim()); } - clientLog(`LLM settings saved: url=${baseUrl}, model=${model || '(none)'}`); - try { await fetch('/api/llm/settings', { method: 'POST', body: fd }); } catch { /* ignore */ } + for (const [key, value] of Object.entries(currentProviderFields())) fd.append(key, value); + + clientLog(`LLM settings saved: url=${baseUrl || '(provider default)'}, model=${model || '(none)'}`); + try { + const res = await fetch('/api/llm/settings', { method: 'POST', body: fd }); + const body = await res.json().catch(() => ({})); + if (!res.ok) { + showUrlNote(body.detail || 'Could not save these settings.'); + return; + } + // The server cleans up the address (a trailing /v1, a container host). + // Show what it settled on rather than leaving a stale value on screen. + if (body.llm_base_url) document.getElementById('llm-base-url').value = body.llm_base_url; + showUrlNote(body.note ? `Adjusted: ${body.note}.` : ''); + } catch { /* offline; the value stays in the form */ } } const POPULAR_MODELS = [ diff --git a/frontend/js/llm-setup.js b/frontend/js/llm-setup.js new file mode 100644 index 0000000..834f1e8 --- /dev/null +++ b/frontend/js/llm-setup.js @@ -0,0 +1,219 @@ +// Choosing and configuring an LLM backend. +// +// The old flow was a single "Base URL" box, which assumed you knew that LM +// Studio listens on 1234, that Unsloth needs a key, and that "localhost" means +// the container when AmicoScript runs in Docker. This module turns that into: +// pick a provider, or press a button and let AmicoScript find what is already +// running. +// +// Part of the AmicoScript frontend. No build step: these are plain ES +// modules loaded directly by the browser via