From 1cbd6d1f99d518171c347b88e1944712a96f2e26 Mon Sep 17 00:00:00 2001 From: Pedro Loch Date: Tue, 28 Jul 2026 18:51:31 +0100 Subject: [PATCH 1/2] feat(voice): allow any OpenAI-compatible transcription endpoint The cloud backend hard-coded both the endpoint and the model name, so the only reachable provider was OpenAI itself even though the request is plain multipart against an OpenAI-shaped /audio/transcriptions route. Four optional variables, each defaulting to the previous behaviour, so an existing OPENAI_API_KEY install needs no change: TELEPI_TRANSCRIPTION_API_KEY falls back to OPENAI_API_KEY TELEPI_TRANSCRIPTION_URL default https://api.openai.com/v1/audio/transcriptions TELEPI_TRANSCRIPTION_MODEL default whisper-1 TELEPI_TRANSCRIPTION_AUTH_HEADER unset -> Authorization: Bearer The auth header is configurable because not every compatible provider accepts the Bearer form; SipPulse AI, for one, documents an `api-key` header. When the variable is unset nothing changes. Error messages now carry the endpoint, which is the one piece of context that was missing when a misconfigured URL returned a 401. --- .env.example | 9 ++++ README.md | 26 ++++++++++++ src/voice.ts | 47 ++++++++++++++++----- test/voice.test.ts | 101 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 173 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index cab16af..2390271 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,15 @@ TELEGRAM_ALLOWED_USER_IDS=123456789 # Optional: enable cloud-based voice transcription via OpenAI Whisper (~$0.006/min) # OPENAI_API_KEY=sk-... +# Optional: point cloud transcription at any OpenAI-compatible provider instead. +# Each variable falls back to the OpenAI default when unset, so an existing +# OPENAI_API_KEY setup keeps working untouched. +# TELEPI_TRANSCRIPTION_API_KEY=... # falls back to OPENAI_API_KEY +# TELEPI_TRANSCRIPTION_URL=https://api.openai.com/v1/audio/transcriptions +# TELEPI_TRANSCRIPTION_MODEL=whisper-1 +# Only needed when the provider does not authenticate with `Authorization: Bearer`. +# TELEPI_TRANSCRIPTION_AUTH_HEADER=api-key + # Optional: enable local/offline transcription via Sherpa-ONNX Parakeet # (primarily for Intel-based Macs, where parakeet-coreml is not supported) # SHERPA_ONNX_MODEL_DIR=/absolute/path/to/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8 diff --git a/README.md b/README.md index c37116b..f8de53a 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,7 @@ TelePi supports three transcription backends and picks the best one automaticall | **Parakeet CoreML** (local) | `npm install parakeet-coreml` + `brew install ffmpeg` | Free | On-device | | **Sherpa-ONNX Parakeet** (local, Intel Mac path) | `npm install sherpa-onnx-node` + download model + set `SHERPA_ONNX_MODEL_DIR` | Free | On-device | | **OpenAI Whisper** (cloud) | `OPENAI_API_KEY=sk-...` in your TelePi config file | ~$0.006/min | Cloud | +| **Any OpenAI-compatible provider** (cloud) | `TELEPI_TRANSCRIPTION_URL` + `TELEPI_TRANSCRIPTION_MODEL` + `TELEPI_TRANSCRIPTION_API_KEY` | Provider's | Cloud | TelePi tries backends in this order: @@ -303,6 +304,31 @@ OPENAI_API_KEY=sk-... No additional packages are required. Supports the same audio formats Telegram delivers (Ogg Opus, MP3, M4A, WAV, etc.). +#### Using another OpenAI-compatible provider + +The cloud backend sends a plain multipart request to an OpenAI-shaped +`/audio/transcriptions` endpoint, so any provider implementing that shape works. +Every variable below is optional and falls back to the OpenAI default, so an +existing `OPENAI_API_KEY` setup needs no changes. + +| Variable | Default | Purpose | +|----------|---------|---------| +| `TELEPI_TRANSCRIPTION_API_KEY` | `OPENAI_API_KEY` | Credential for the endpoint | +| `TELEPI_TRANSCRIPTION_URL` | `https://api.openai.com/v1/audio/transcriptions` | Full endpoint URL | +| `TELEPI_TRANSCRIPTION_MODEL` | `whisper-1` | Value sent in the `model` field | +| `TELEPI_TRANSCRIPTION_AUTH_HEADER` | unset | Header name for the key. Unset sends `Authorization: Bearer `; set sends `
: ` raw | + +Example, using SipPulse AI: + +``` +TELEPI_TRANSCRIPTION_API_KEY=... +TELEPI_TRANSCRIPTION_URL=https://api.sippulse.ai/openai/audio/transcriptions +TELEPI_TRANSCRIPTION_MODEL=pulse-precision-pro +``` + +`TELEPI_TRANSCRIPTION_AUTH_HEADER` exists for providers that do not accept the +Bearer form and expect their own header instead, such as `api-key`. + ## Session Tree Navigation Every prompt and response in Pi is stored as a tree of entries. TelePi exposes this tree so you can review history and jump back to any point to create a new branch. diff --git a/src/voice.ts b/src/voice.ts index de2c0e8..183b984 100644 --- a/src/voice.ts +++ b/src/voice.ts @@ -60,6 +60,21 @@ const SHERPA_ONNX_MODEL_DIR_ENV = "SHERPA_ONNX_MODEL_DIR"; const SHERPA_ONNX_NUM_THREADS_ENV = "SHERPA_ONNX_NUM_THREADS"; const SHERPA_MODEL_DOCS_URL = "https://k2-fsa.github.io/sherpa/onnx/pretrained_models/offline-transducer/nemo-transducer-models.html"; + +// The cloud backend talks plain multipart to an OpenAI-shaped endpoint, so any +// provider that implements /audio/transcriptions works once the URL, the model +// name and the credential are not hard-coded. Every default below reproduces +// the previous behaviour exactly, so an existing OPENAI_API_KEY setup keeps +// working with no new configuration. +const TRANSCRIPTION_URL_ENV = "TELEPI_TRANSCRIPTION_URL"; +const TRANSCRIPTION_MODEL_ENV = "TELEPI_TRANSCRIPTION_MODEL"; +const TRANSCRIPTION_API_KEY_ENV = "TELEPI_TRANSCRIPTION_API_KEY"; +// Some OpenAI-compatible providers authenticate with their own header instead +// of `Authorization: Bearer` (SipPulse uses `api-key`). When this is set the +// key is sent raw under that header; when it is unset the Bearer form is used. +const TRANSCRIPTION_AUTH_HEADER_ENV = "TELEPI_TRANSCRIPTION_AUTH_HEADER"; +const DEFAULT_TRANSCRIPTION_URL = "https://api.openai.com/v1/audio/transcriptions"; +const DEFAULT_TRANSCRIPTION_MODEL = "whisper-1"; const FFMPEG_INSTALL_MESSAGE = `ffmpeg not found. Install it with: ${getPlatformInstallHint("ffmpeg")}`; const NO_BACKEND_ERROR = `Voice messages require a transcription backend. @@ -75,7 +90,13 @@ Option 2: Install Sherpa-ONNX for local/offline Parakeet transcription on Intel- Set ${SHERPA_ONNX_MODEL_DIR_ENV}=/path/to/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8 Option 3: Set OPENAI_API_KEY for cloud transcription (~$0.006/min): - Add OPENAI_API_KEY=sk-... to your .env file`; + Add OPENAI_API_KEY=sk-... to your .env file + + Any OpenAI-compatible transcription endpoint works instead: + ${TRANSCRIPTION_API_KEY_ENV}=... (falls back to OPENAI_API_KEY) + ${TRANSCRIPTION_URL_ENV}=https://provider.example/v1/audio/transcriptions + ${TRANSCRIPTION_MODEL_ENV}=provider-model-name + ${TRANSCRIPTION_AUTH_HEADER_ENV}=api-key (only if the provider does not use Bearer)`; const _require = createRequire(import.meta.url); let _importModule: (specifier: string) => Promise = async (specifier) => _require(specifier); @@ -311,11 +332,15 @@ async function transcribeWithSherpaOnnx( } async function transcribeWithOpenAI(filePath: string): Promise { - const apiKey = process.env.OPENAI_API_KEY?.trim(); + const apiKey = resolveTranscriptionApiKey(); if (!apiKey) { throw new Error(NO_BACKEND_ERROR); } + const endpoint = process.env[TRANSCRIPTION_URL_ENV]?.trim() || DEFAULT_TRANSCRIPTION_URL; + const model = process.env[TRANSCRIPTION_MODEL_ENV]?.trim() || DEFAULT_TRANSCRIPTION_MODEL; + const authHeader = process.env[TRANSCRIPTION_AUTH_HEADER_ENV]?.trim(); + const startedAt = Date.now(); const audioBuffer = await readFile(filePath); const ext = (path.extname(filePath) || ".ogg").slice(1).toLowerCase(); @@ -327,26 +352,24 @@ async function transcribeWithOpenAI(filePath: string): Promise "")).trim(); throw new Error( - `OpenAI transcription failed (${response.status}): ${errorText || response.statusText || "Unknown error"}`, + `OpenAI transcription failed (${response.status}): ${errorText || response.statusText || "Unknown error"} [endpoint: ${endpoint}]`, ); } const payload = (await response.json()) as { text?: unknown }; if (typeof payload.text !== "string") { - throw new Error("OpenAI transcription response did not include a text field"); + throw new Error(`OpenAI transcription response did not include a text field [endpoint: ${endpoint}]`); } return { @@ -504,8 +527,12 @@ function decodeAudioToSamples(filePath: string): Promise { }); } +function resolveTranscriptionApiKey(): string | undefined { + return process.env[TRANSCRIPTION_API_KEY_ENV]?.trim() || process.env.OPENAI_API_KEY?.trim() || undefined; +} + function hasOpenAIApiKey(): boolean { - return Boolean(process.env.OPENAI_API_KEY?.trim()); + return Boolean(resolveTranscriptionApiKey()); } function extractTranscribedText(result: unknown): string | undefined { diff --git a/test/voice.test.ts b/test/voice.test.ts index 7705a93..157a556 100644 --- a/test/voice.test.ts +++ b/test/voice.test.ts @@ -13,8 +13,18 @@ import { transcribeAudio, } from "../src/voice.js"; +const TRANSCRIPTION_ENV_KEYS = [ + "TELEPI_TRANSCRIPTION_API_KEY", + "TELEPI_TRANSCRIPTION_URL", + "TELEPI_TRANSCRIPTION_MODEL", + "TELEPI_TRANSCRIPTION_AUTH_HEADER", +] as const; + describe("voice transcription", () => { const originalOpenAIKey = process.env.OPENAI_API_KEY; + const originalTranscriptionEnv = Object.fromEntries( + TRANSCRIPTION_ENV_KEYS.map((key) => [key, process.env[key]]), + ); const originalSherpaModelDir = process.env.SHERPA_ONNX_MODEL_DIR; const originalSherpaNumThreads = process.env.SHERPA_ONNX_NUM_THREADS; @@ -55,6 +65,9 @@ describe("voice transcription", () => { delete process.env.OPENAI_API_KEY; delete process.env.SHERPA_ONNX_MODEL_DIR; delete process.env.SHERPA_ONNX_NUM_THREADS; + for (const key of TRANSCRIPTION_ENV_KEYS) { + delete process.env[key]; + } _resetImportHook(); vi.unstubAllGlobals(); }); @@ -70,6 +83,15 @@ describe("voice transcription", () => { process.env.OPENAI_API_KEY = originalOpenAIKey; } + for (const key of TRANSCRIPTION_ENV_KEYS) { + const original = originalTranscriptionEnv[key]; + if (original === undefined) { + delete process.env[key]; + } else { + process.env[key] = original; + } + } + if (originalSherpaModelDir === undefined) { delete process.env.SHERPA_ONNX_MODEL_DIR; } else { @@ -324,6 +346,85 @@ describe("voice transcription", () => { ); }); + it("honours a custom transcription endpoint, model and auth header", async () => { + _setImportHook(async (specifier) => { + if (specifier === "parakeet-coreml") { + throw moduleNotFound("parakeet-coreml"); + } + throw new Error(`unexpected import: ${specifier}`); + }); + process.env.TELEPI_TRANSCRIPTION_API_KEY = "sp-test"; + process.env.TELEPI_TRANSCRIPTION_URL = "https://api.sippulse.ai/openai/audio/transcriptions"; + process.env.TELEPI_TRANSCRIPTION_MODEL = "pulse-precision-pro"; + process.env.TELEPI_TRANSCRIPTION_AUTH_HEADER = "api-key"; + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ text: "custom transcript" }), + }); + vi.stubGlobal("fetch", fetchMock); + + const result = await transcribeAudio(audioPath); + + expect(result).toMatchObject({ text: "custom transcript", backend: "openai" }); + expect(fetchMock).toHaveBeenCalledWith( + "https://api.sippulse.ai/openai/audio/transcriptions", + expect.objectContaining({ + method: "POST", + headers: { "api-key": "sp-test" }, + body: expect.any(FormData), + }), + ); + const [, init] = fetchMock.mock.calls[0] as [string, { body: FormData }]; + expect(init.body.get("model")).toBe("pulse-precision-pro"); + }); + + it("keeps the Bearer form when only the endpoint and model are overridden", async () => { + _setImportHook(async (specifier) => { + if (specifier === "parakeet-coreml") { + throw moduleNotFound("parakeet-coreml"); + } + throw new Error(`unexpected import: ${specifier}`); + }); + process.env.TELEPI_TRANSCRIPTION_API_KEY = "sp-test"; + process.env.TELEPI_TRANSCRIPTION_URL = "https://api.sippulse.ai/openai/audio/transcriptions"; + process.env.TELEPI_TRANSCRIPTION_MODEL = "pulse-precision-pro"; + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ text: "bearer transcript" }), + }); + vi.stubGlobal("fetch", fetchMock); + + await transcribeAudio(audioPath); + + expect(fetchMock).toHaveBeenCalledWith( + "https://api.sippulse.ai/openai/audio/transcriptions", + expect.objectContaining({ headers: { Authorization: "Bearer sp-test" } }), + ); + }); + + it("prefers TELEPI_TRANSCRIPTION_API_KEY over OPENAI_API_KEY", async () => { + _setImportHook(async (specifier) => { + if (specifier === "parakeet-coreml") { + throw moduleNotFound("parakeet-coreml"); + } + throw new Error(`unexpected import: ${specifier}`); + }); + process.env.OPENAI_API_KEY = "sk-test"; + process.env.TELEPI_TRANSCRIPTION_API_KEY = "sp-test"; + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ text: "precedence transcript" }), + }); + vi.stubGlobal("fetch", fetchMock); + + await transcribeAudio(audioPath); + + expect(fetchMock).toHaveBeenCalledWith( + "https://api.openai.com/v1/audio/transcriptions", + expect.objectContaining({ headers: { Authorization: "Bearer sp-test" } }), + ); + }); + it("throws a helpful error when no backend is available", async () => { _setImportHook(async (specifier) => { if (specifier === "parakeet-coreml") { From 3e0d635f6b4001f7c455ac60d62465b848711f5f Mon Sep 17 00:00:00 2001 From: Pedro Loch Date: Tue, 28 Jul 2026 19:32:46 +0100 Subject: [PATCH 2/2] feat(voice): forward an optional transcription prompt The OpenAI transcription API accepts a `prompt` field: free-form context that biases decoding toward names and jargon the model would otherwise miss. TELEPI_TRANSCRIPTION_PROMPT forwards it, and the field is omitted entirely when the variable is unset, so providers that reject it are unaffected. Measured against a real provider, same clip, same model (pulse-precision-pro), audio saying "Manda a Hipatia revisar o worktree do Maestro": without prompt Manda a Ipate a revisar o osc3 do maestro. with prompt Manda a Hipatia revisar o worktree do Maestro. Two things worth knowing, both found by testing rather than reading: Support varies by MODEL, not only by provider. The same provider's telephony model returned byte-identical output with and without the field - it silently ignores it - while its precision model honoured it. Nothing in either response says which happened. Phrasing matters. The identical vocabulary as a bare comma list left the name wrong; the same terms inside a context sentence fixed it. The README says so rather than leaving it to be rediscovered. --- .env.example | 2 ++ README.md | 13 +++++++++++++ src/voice.ts | 11 ++++++++++- test/voice.test.ts | 25 +++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 2390271..ee767a2 100644 --- a/.env.example +++ b/.env.example @@ -25,6 +25,8 @@ TELEGRAM_ALLOWED_USER_IDS=123456789 # TELEPI_TRANSCRIPTION_MODEL=whisper-1 # Only needed when the provider does not authenticate with `Authorization: Bearer`. # TELEPI_TRANSCRIPTION_AUTH_HEADER=api-key +# Context that biases transcription toward names and jargon you say often. +# TELEPI_TRANSCRIPTION_PROMPT=Conversation about software. Terms: Kubernetes, Grafana, ... # Optional: enable local/offline transcription via Sherpa-ONNX Parakeet # (primarily for Intel-based Macs, where parakeet-coreml is not supported) diff --git a/README.md b/README.md index f8de53a..8320d17 100644 --- a/README.md +++ b/README.md @@ -317,6 +317,19 @@ existing `OPENAI_API_KEY` setup needs no changes. | `TELEPI_TRANSCRIPTION_URL` | `https://api.openai.com/v1/audio/transcriptions` | Full endpoint URL | | `TELEPI_TRANSCRIPTION_MODEL` | `whisper-1` | Value sent in the `model` field | | `TELEPI_TRANSCRIPTION_AUTH_HEADER` | unset | Header name for the key. Unset sends `Authorization: Bearer `; set sends `
: ` raw | +| `TELEPI_TRANSCRIPTION_PROMPT` | unset | Sent as the API's `prompt` field. Free-form context that biases decoding toward names and jargon. Omitted entirely when unset | + +A prompt is worth setting if you dictate proper nouns the model does not know. +Write it as a **context sentence**, not a bare word list - the same wording as a +list performed noticeably worse in testing: + +``` +TELEPI_TRANSCRIPTION_PROMPT=Conversation about software development. Terms: Kubernetes, Grafana, worktree, ... +``` + +Support varies by provider and even by model within one provider, and a model +that ignores the field does so silently. Transcribe the same clip with and +without it before trusting it. Example, using SipPulse AI: diff --git a/src/voice.ts b/src/voice.ts index 183b984..64ed5f8 100644 --- a/src/voice.ts +++ b/src/voice.ts @@ -73,6 +73,10 @@ const TRANSCRIPTION_API_KEY_ENV = "TELEPI_TRANSCRIPTION_API_KEY"; // of `Authorization: Bearer` (SipPulse uses `api-key`). When this is set the // key is sent raw under that header; when it is unset the Bearer form is used. const TRANSCRIPTION_AUTH_HEADER_ENV = "TELEPI_TRANSCRIPTION_AUTH_HEADER"; +// Optional `prompt` field of the OpenAI transcription API: free-form context +// that biases decoding toward names and jargon the model would otherwise miss. +// Sent only when set, so providers that reject the field are unaffected. +const TRANSCRIPTION_PROMPT_ENV = "TELEPI_TRANSCRIPTION_PROMPT"; const DEFAULT_TRANSCRIPTION_URL = "https://api.openai.com/v1/audio/transcriptions"; const DEFAULT_TRANSCRIPTION_MODEL = "whisper-1"; const FFMPEG_INSTALL_MESSAGE = `ffmpeg not found. Install it with: ${getPlatformInstallHint("ffmpeg")}`; @@ -96,7 +100,8 @@ Option 3: Set OPENAI_API_KEY for cloud transcription (~$0.006/min): ${TRANSCRIPTION_API_KEY_ENV}=... (falls back to OPENAI_API_KEY) ${TRANSCRIPTION_URL_ENV}=https://provider.example/v1/audio/transcriptions ${TRANSCRIPTION_MODEL_ENV}=provider-model-name - ${TRANSCRIPTION_AUTH_HEADER_ENV}=api-key (only if the provider does not use Bearer)`; + ${TRANSCRIPTION_AUTH_HEADER_ENV}=api-key (only if the provider does not use Bearer) + ${TRANSCRIPTION_PROMPT_ENV}=... (context to bias names and jargon)`; const _require = createRequire(import.meta.url); let _importModule: (specifier: string) => Promise = async (specifier) => _require(specifier); @@ -353,6 +358,10 @@ async function transcribeWithOpenAI(filePath: string): Promise { @@ -425,6 +426,30 @@ describe("voice transcription", () => { ); }); + it("sends the transcription prompt only when it is set", async () => { + _setImportHook(async (specifier) => { + if (specifier === "parakeet-coreml") { + throw moduleNotFound("parakeet-coreml"); + } + throw new Error(`unexpected import: ${specifier}`); + }); + process.env.OPENAI_API_KEY = "sk-test"; + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ text: "t" }), + }); + vi.stubGlobal("fetch", fetchMock); + + await transcribeAudio(audioPath); + const [, withoutPrompt] = fetchMock.mock.calls[0] as [string, { body: FormData }]; + expect(withoutPrompt.body.get("prompt")).toBeNull(); + + process.env.TELEPI_TRANSCRIPTION_PROMPT = "Hipatia, worktree, herdr"; + await transcribeAudio(audioPath); + const [, withPrompt] = fetchMock.mock.calls[1] as [string, { body: FormData }]; + expect(withPrompt.body.get("prompt")).toBe("Hipatia, worktree, herdr"); + }); + it("throws a helpful error when no backend is available", async () => { _setImportHook(async (specifier) => { if (specifier === "parakeet-coreml") {